Add initial package instance creation

Adds the PackageInstance class and functions to group package manifests
and create a package instance out of them. Adds a sample example case for
python.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/__init__.py

@@ -110,28 +110,36 @@
     pubspec.PubspecLock
 ]
 
+PACKAGE_INSTANCE_TYPES = [
+    pypi.PythonPackageInstance
+]
+
 PACKAGE_MANIFESTS_BY_TYPE = {
-    (
-        cls.package_manifest_type
-        if isinstance(cls, models.PackageManifest)
-        else cls.default_type
-    ): cls
+    cls.default_type: cls
     for cls in PACKAGE_MANIFEST_TYPES
 }
-# We cannot have two package classes with the same type
-if len(PACKAGE_MANIFESTS_BY_TYPE) != len(PACKAGE_MANIFEST_TYPES):
-    seen_types = {}
-    for pmt in PACKAGE_MANIFEST_TYPES:
-        manifest = pmt()
-        assert manifest.package_manifest_type
-        seen = seen_types.get(manifest.package_manifest_type)
-        if seen:
-            msg = ('Invalid duplicated packagedcode.Package types: '
-                   '"{}:{}" and "{}:{}" have the same type.'
-                  .format(manifest.package_manifest_type, manifest.__name__, seen.package_manifest_type, seen.__name__,))
-            raise Exception(msg)
-        else:
-            seen_types[manifest.package_manifest_type] = manifest
+
+PACKAGE_INSTANCES_BY_TYPE = {
+    cls.default_type: cls
+    for cls in PACKAGE_INSTANCE_TYPES
+}
+
+def check_package_manifest_classes():
+
+    # We cannot have two package classes with the same type
+    if len(PACKAGE_MANIFESTS_BY_TYPE) != len(PACKAGE_MANIFEST_TYPES):
+        seen_types = {}
+        for pmt in PACKAGE_MANIFEST_TYPES:
+            manifest = pmt()
+            assert manifest.package_manifest_type
+            seen = seen_types.get(manifest.package_manifest_type)
+            if seen:
+                msg = ('Invalid duplicated packagedcode.Package types: '
+                    '"{}:{}" and "{}:{}" have the same type.'
+                    .format(manifest.package_manifest_type, manifest.__name__, seen.package_manifest_type, seen.__name__,))
+                raise Exception(msg)
+            else:
+                seen_types[manifest.package_manifest_type] = manifest
 
 
 def get_package_class(scan_data, default=models.Package):

src/packagedcode/models.py

@@ -765,6 +765,100 @@ def recognize(cls, location):
         raise NotImplementedError
 
 
+@attr.s()
+class PackageInstance:
+    """
+    A package instance mixin as represented by its package manifests, files and data
+    from its package manifests.
+
+    Subclasses must extend a Package subclass for a given ecosystem.
+    """
+
+    package_uuid = String(
+        label='Package instance UUID',
+        help='A unique ID for package instances in a codebase scan.'
+             'Consists of a pURL and an UUID field as a pURL qualifier.'
+    )
+
+    package_manifest_paths = List(
+        item_type=String,
+        label='Package manifest paths',
+        help='List of package manifest file paths for this package'
+    )
+
+    files = List(
+        item_type=PackageFile,
+        label='Provided files',
+        help='List of files provided by this package.'
+    )
+
+    def populate_instance_from_manifests(self, package_manifests_by_path, uuid):
+        """
+        Create a package instance object from one or multiple package manifests.
+        """
+        for path, package_manifest in package_manifests_by_path.items():
+            self.package_manifest_paths.append(path)
+
+        self.package_manifest_paths = tuple(self.package_manifest_paths)
+
+        # ToDo: This field would be pURL + UUID as a qualifier instead
+        self.package_uuid = str(uuid)
+
+    def get_package_files(self, resource, codebase):
+        """
+        Return a list of all the file paths for a package instance.
+ 
+        Sub-classes should override to implement their own package files finding methods.
+        """
+        files = []
+
+        parent = resource.parent(codebase)
+
+        for resource in parent.walk(codebase):
+            if resource.is_dir:
+                continue
+
+            files.append(resource.path)
+        
+        return files
+
+    def get_other_manifests_for_instance(self, resource, codebase):
+        """
+        Return a dictionary of other package manifests by their paths for a given package instance.
+
+        Sub-classes can override to implement their own package manifest finding methods.
+        """
+        package_manifests_by_path = {}
+
+        parent = resource.parent(codebase)
+
+        for resource in parent.walk(codebase):
+            if resource.is_dir:
+                continue
+
+            filename = file_name(resource.location)
+            file_patterns = self.get_file_patterns(manifests=self.manifests)
+            if any(fnmatch.fnmatchcase(filename, pattern) for pattern in file_patterns):
+                if not resource.package_manifests:
+                    continue # Raise Exception(?)
+
+                #ToDo: Implement for multiple package manifests per path
+                package_manifests_by_path[resource.path] = resource.package_manifests[0]
+
+        return package_manifests_by_path
+    
+    def get_file_patterns(self, manifests):
+        """
+        Return a list of all `file_patterns` for all the PackageManifest classes
+        in `manifests`.
+        """
+        manifest_file_patterns = []
+        for manifest in manifests:
+            manifest_file_patterns.extend(manifest.file_patterns)
+        
+        return manifest_file_patterns
+
+
 # Package types
 # NOTE: this is somewhat redundant with extractcode archive handlers
 # yet the purpose and semantics are rather different here

src/packagedcode/plugin_package.py

@@ -10,6 +10,9 @@
 
 import attr
 import click
+import os
+import sys
+import uuid
 
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
@@ -19,8 +22,30 @@
 
 from packagedcode import get_package_instance
 from packagedcode import PACKAGE_MANIFEST_TYPES
+from packagedcode import PACKAGE_INSTANCES_BY_TYPE
 
 
+TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
+
+if TRACE:
+
+    use_print = True
+
+    if use_print:
+        printer = print
+    else:
+        import logging
+
+        logger = logging.getLogger(__name__)
+        # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
+        logging.basicConfig(stream=sys.stdout)
+        logger.setLevel(logging.DEBUG)
+        printer = logger.debug
+
+    def logger_debug(*args):
+        return printer(' '.join(isinstance(a, str) and a or repr(a)
+                                     for a in args))
+
 def print_packages(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
@@ -51,8 +76,9 @@ class PackageScanner(ScanPlugin):
 
     resource_attributes = {}
     codebase_attributes = {}
-    resource_attributes['package_manifests'] = attr.ib(default=attr.Factory(list), repr=False)
     codebase_attributes['packages'] = attr.ib(default=attr.Factory(list), repr=False)
+    resource_attributes['package_manifests'] = attr.ib(default=attr.Factory(list), repr=False)
+    #resource_attributes['for_packages'] = attr.ib(default=attr.Factory(list), repr=False)
 
     sort_order = 6
 
@@ -102,13 +128,90 @@ def create_packages_from_manifests(codebase, **kwargs):
     Create package instances from package manifests present in the codebase.
     """
     package_manifests = []
+    package_instances_by_paths = {}
+    package_instance_by_identifiers = {}
 
     for resource in codebase.walk(topdown=False):
-        if resource.package_manifests:
+        if not resource.package_manifests:
+            continue
+
+        # continue if resource.path already in `package_instances_by_paths`
+        if resource.path in package_instances_by_paths:
+            continue
+
+        if TRACE:
+            logger_debug(
+                'create_packages_from_manifests:',
+                'location:', resource.location,
+            )
+
+        # Currently we assume there is only one PackageManifest 
+        # ToDo: Do this for multiple PackageManifests per resource
+        manifest = resource.package_manifests[0]
+
+        # Check if PackageInstance is implemented
+        pk_instance_class = PACKAGE_INSTANCES_BY_TYPE.get(manifest["type"])
+        if not pk_instance_class:
             package_manifests.extend(resource.package_manifests)
+            continue
 
+        # create a PackageInstance from the `default_type`
+        pk_instance = pk_instance_class()
+        pk_instance_uuid = uuid.uuid4()
+        package_instance_by_identifiers[pk_instance_uuid] = pk_instance
+
+        # use the get_other_manifests_for_instance to get other instances
+        package_manifests_by_path = pk_instance.get_other_manifests_for_instance(resource, codebase)
+        package_manifests_by_path[resource.path] = manifest
+
+        if TRACE:
+            logger_debug(
+                'create_packages_from_manifests:',
+                'package_manifests_by_path:', package_manifests_by_path,
+            )
+
+        # add `path: Instance` into `package_instances_by_paths` for all manifests
+        for path in package_manifests_by_path.keys():
+            print(f"Path: {path}")
+            package_instances_by_paths[path] = pk_instance
+
+        # populate PackageInstance with data from manifests
+        pk_instance.populate_instance_from_manifests(package_manifests_by_path, uuid=pk_instance_uuid)
+
+        # get files for this PackageInstance
+        pk_instance.files = tuple(pk_instance.get_package_files(resource, codebase))
+
+        # add instance uuid to `for_packages` for all manifests (and files ?)
+        update_files_with_package_instances(package_manifests_by_path, codebase, pk_instance)
+
+        if TRACE:
+            logger_debug(
+                'create_packages_from_manifests:',
+                'pk_instance:', pk_instance,
+            )
+
+    # ToDo: replace this with PackageInstance objects once basic implementation is complete
     codebase.attributes.packages.extend(package_manifests)
 
+    if TRACE:
+        logger_debug(
+            'create_packages_from_manifests:',
+            'package_instances_by_paths:', package_instances_by_paths,
+        )
+
+    # Get unique PackageInstance objects from `package_instances_by_paths`
+
+    package_instances = list(package_instance_by_identifiers.values())
+    codebase.attributes.packages.extend(package_instances)
+
+
+def update_files_with_package_instances(package_manifests_by_path, codebase, package_instance):
+
+    for path in package_manifests_by_path.keys():
+        # Update `for_packages` attribute for resource at path with
+        # reference to this package_instance
+        continue
+
 
 def set_packages_root(resource, codebase):
     """

src/packagedcode/pypi.py

@@ -48,7 +48,7 @@
 """
 Detect and collect Python packages information.
 """
-# TODO: add support for poetry and setup.cfg
+# TODO: add support for poetry and setup.cfg and metadata.json
 
 TRACE = False
 
@@ -373,6 +373,23 @@ def recognize(cls, location):
         yield cls(dependencies=dependent_packages)
 
 
+@attr.s()
+class PythonPackageInstance(PythonPackage, models.PackageInstance):
+    """
+    A Python PackageInstance that is created out of one/multiple python package
+    manifests.
+    """
+
+    @property
+    def manifests(self):
+        return [
+            MetadataFile,
+            RequirementsFile,
+            PipfileLock,
+            DependencyFile,
+            SetupPy
+        ]
+
 
 def get_attribute(metainfo, name, multiple=False):
     """

tests/packagedcode/data/instance/pypi/LICENSE.rst

@@ -0,0 +1,28 @@
+Copyright 2014 Pallets
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1.  Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+3.  Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tests/packagedcode/data/instance/pypi/MANIFEST.in

@@ -0,0 +1,10 @@
+include CHANGES.rst
+include tox.ini
+include requirements/*.txt
+graft artwork
+graft docs
+prune docs/_build
+graft examples
+graft tests
+include src/click/py.typed
+global-exclude *.pyc