Merge pull request #3036 from nexB/release-31-rc4-prep

Release 31 rc4 prep

Author: pombredanne

CHANGELOG.rst

@@ -34,6 +34,7 @@ Important API changes:
 
   - There is a a new top-level "dependencies" attribute that contains each
     dependency instance, these can be standalone or releated to a package.
+    These contain a new "extra_data" object.
 
   - There is a new resource-level attribute "for_packages" which refers to
     packages through package_uuids (pURL + uuid string).
@@ -172,6 +173,10 @@ Package detection:
   - The package_data attribute `dependencies` (which is a list of DependentPackages),
     now has a new attribute `resolved_package` with a package data mapping.
     Also the `requirement` attribute is renamed to `extracted_requirement`.
+    There is a new `extra_data` to collect extra data as needed.
+
+- For Pypi packages, python_requires is treated as a package dependency.
+
 
 
 License Clarity Scoring Update

src/packagedcode/alpine.py

@@ -58,6 +58,11 @@ def parse(cls, location):
             package_type=cls.default_package_type,
         )
 
+    @classmethod
+    def compute_normalized_license(cls, package):
+        _declared, detected = detect_declared_license(package.declared_license)
+        return detected
+
     @classmethod
     def assemble(cls, package_data, resource, codebase):
         # get the root resource of the rootfs
@@ -76,6 +81,7 @@ def assemble(cls, package_data, resource, codebase):
 
         package.license_expression = cls.compute_normalized_license(package)
 
+
         dependent_packages = package_data.dependencies
         if dependent_packages:
             yield from models.Dependency.from_dependent_packages(

src/packagedcode/debian.py

@@ -28,9 +28,11 @@
 
 TRACE = SCANCODE_DEBUG_PACKAGE_API
 
+
 def logger_debug(*args):
     pass
 
+
 logger = logging.getLogger(__name__)
 
 if TRACE:
@@ -43,7 +45,6 @@ def logger_debug(*args):
             ' '.join(isinstance(a, str) and a or repr(a) for a in args)
         )
 
-
 # TODO: add dependencies
 
 
@@ -282,14 +283,27 @@ def assemble(cls, package_data, resource, codebase):
         resources = []
         # TODO: keep track of missing files
         for res in root_resource.walk(codebase):
+            if TRACE:
+                logger_debug(f'   debian: assemble: root_walk: res: {res}')
             if not res.path.endswith(assemblable_paths):
                 continue
 
             for pkgdt in res.package_data:
                 package_data = models.PackageData.from_dict(pkgdt)
+                if TRACE:
+                    # logger_debug(f'     debian: assemble: root_walk: package_data: {package_data}')
+                    logger_debug(f'     debian: assemble: root_walk: package_data: {package_data.license_expression}')
+
+                # Most debian secondary files are only specific to a name. We
+                # have a few cases where the arch is included in the lists and
+                # md5sums.
                 package.update(
                     package_data=package_data,
                     datafile_path=res.path,
+                    replace=False,
+                    include_version=False,
+                    include_qualifiers=False,
+                    include_subpath=False,
                 )
                 package_file_references.extend(package_data.file_references)
 

src/packagedcode/models.py

@@ -7,10 +7,10 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import logging
 import os
 import uuid
 from fnmatch import fnmatchcase
-import logging
 
 import attr
 from packageurl import normalize_qualifiers
@@ -26,7 +26,15 @@
 from commoncode.datautils import String
 from commoncode.fileutils import as_posixpath
 from commoncode.resource import Resource
-from typecode import contenttype
+try:
+    from typecode import contenttype
+except ImportError:
+    contenttype = None
+
+try:
+    from packagedcode import licensing
+except ImportError:
+    licensing = None
 
 """
 This module contain data models for package and dependencies, abstracting and
@@ -366,6 +374,11 @@ class DependentPackage(ModelMixin):
              'lockfiles for Composer or Cargo contain extra dependency data.'
          )
 
+    extra_data = Mapping(
+        label='extra data',
+        help='A mapping of arbitrary extra data.',
+    )
+
 
 @attr.attributes(slots=True)
 class Dependency(DependentPackage):
@@ -771,7 +784,13 @@ def compute_normalized_license(declared_license, expression_symbols=None):
     if not declared_license:
         return
 
-    from packagedcode import licensing
+    if not licensing:
+        if TRACE:
+            logger_debug(
+                f'Failed to compute license for {declared_license!r}: '
+                'cannot import packagedcode.licensing')
+        return 'unknown'
+
     try:
         return licensing.get_normalized_expression(
             query_string=declared_license,
@@ -781,7 +800,6 @@ def compute_normalized_license(declared_license, expression_symbols=None):
         # we never fail just for this
         if TRACE:
             logger_debug(f'Failed to compute license for {declared_license!r}: {e!r}')
-        # FIXME: add logging
         return 'unknown'
 
 
@@ -852,7 +870,8 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False):
                 filetypes = filetypes or cls.filetypes
                 if not filetypes:
                     return True
-                else:
+                # we check for contenttype IFF this is available
+                if contenttype:
                     T = contenttype.get_type(location)
                     actual_type = T.filetype_file.lower()
                     return any(ft in actual_type for ft in filetypes)
@@ -1207,6 +1226,13 @@ def __attrs_post_init__(self, *args, **kwargs):
     def to_dict(self):
         return super().to_dict(with_details=False)
 
+    def to_package_data(self):
+        mapping = super().to_dict(with_details=True)
+        mapping.pop('package_uid', None)
+        mapping.pop('datafile_paths', None)
+        mapping.pop('datasource_ids', None)
+        return PackageData.from_dict(mapping)
+
     @classmethod
     def from_package_data(cls, package_data, datafile_path):
         """
@@ -1255,7 +1281,15 @@ def is_compatible(self, package_data, include_qualifiers=True):
             and self.primary_language == package_data.primary_language
         )
 
-    def update(self, package_data, datafile_path, replace=False):
+    def update(
+        self,
+        package_data,
+        datafile_path,
+        replace=False,
+        include_version=True,
+        include_qualifiers=False,
+        include_subpath=False,
+    ):
         """
         Update this Package with data from the ``package_data`` PackageData.
 
@@ -1281,9 +1315,15 @@ def update(self, package_data, datafile_path, replace=False):
         if isinstance(package_data, dict):
             package_data = PackageData.from_dict(package_data)
 
-        if not self.is_compatible(package_data, include_qualifiers=False):
+        if not is_compatible(
+            purl1=self,
+            purl2=package_data,
+            include_version=include_version,
+            include_qualifiers=include_qualifiers,
+            include_subpath=include_subpath,
+        ):
             if TRACE_UPDATE:
-                logger_debug(f'update: {self.purl} not compatible with: {package_data.purl}')
+                logger_debug(f'update: skipping: {self.purl} is not compatible with: {package_data.purl}')
             return False
 
         # always append these new items
@@ -1345,6 +1385,56 @@ def get_packages_files(self, codebase):
                     yield resource
 
 
+def is_compatible(
+    purl1,
+    purl2,
+    include_version=True,
+    include_qualifiers=True,
+    include_subpath=True,
+):
+    """
+    Return True if the ``purl1`` PackageURL-like object is compatible with
+    the ``purl2`` PackageURL-like object, e.g. it is about the same package.
+    PackageData objectys are PackageURL-like.
+
+    For example::
+    >>> p1 = PackageURL.from_string('pkg:deb/libncurses5@6.1-1ubuntu1.18.04?arch=arm64')
+    >>> p2 = PackageURL.from_string('pkg:deb/libncurses5@6.1-1ubuntu1.18.04')
+    >>> p3 = PackageURL.from_string('pkg:deb/libssl')
+    >>> p4 = PackageURL.from_string('pkg:deb/libncurses5')
+    >>> p5 = PackageURL.from_string('pkg:deb/libncurses5@6.1-1ubuntu1.18.04?arch=arm64#/sbin')
+    >>> is_compatible(p1, p2)
+    False
+    >>> is_compatible(p1, p2, include_qualifiers=False)
+    True
+    >>> is_compatible(p1, p4)
+    False
+    >>> is_compatible(p1, p4, include_version=False, include_qualifiers=False)
+    True
+    >>> is_compatible(p3, p4)
+    False
+    >>> is_compatible(p1, p5)
+    False
+    >>> is_compatible(p1, p5, include_subpath=False)
+    True
+    """
+    is_compatible = (
+        purl1.type == purl2.type
+        and purl1.namespace == purl2.namespace
+        and purl1.name == purl2.name
+    )
+    if include_version:
+        is_compatible = is_compatible and (purl1.version == purl2.version)
+
+    if include_qualifiers:
+        is_compatible = is_compatible and (purl1.qualifiers == purl2.qualifiers)
+
+    if include_subpath:
+        is_compatible = is_compatible and (purl1.subpath == purl2.subpath)
+
+    return is_compatible
+
+
 @attr.attributes(slots=True)
 class PackageWithResources(Package):
     """
@@ -1384,7 +1474,15 @@ def merge_sequences(list1, list2, **kwargs):
     merged = []
     existing = set()
     for item in list1 + list2:
-        key = item.to_tuple(**kwargs)
+        try:
+            if hasattr(item, 'to_tuple'):
+                key = item.to_tuple(**kwargs)
+            else:
+                key = to_tuple(kwargs)
+
+        except Exception as e:
+            raise Exception(f'Failed to merge sequences: {item}', f'kwargs: {kwargs}') from e
+
         if not key in existing:
             merged.append(item)
             existing.add(key)

src/packagedcode/plugin_package.py

@@ -156,7 +156,8 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
 
 def get_installed_packages(root_dir, processes=2, **kwargs):
     """
-    Yield Package and their Resources as they are found in `root_dir`
+    Detect and yield Package mappings with their assigned Resource in a ``resources``
+    attribute as they are found in `root_dir`.
     """
     from scancode import cli