Restore summarycode and API changes

Restores previously made changes in summarycode, and deprecates
package API with a warning, and adds other small bugfixes.

Signed-off-by: Ayan Sinha <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/plugin_package.py

@@ -80,8 +80,8 @@ def get_scanner(self, **kwargs):
         """
         Return a scanner callable to scan a Resource for packages.
         """
-        from scancode.api import get_package_manifest_info
-        return get_package_manifest_info
+        from scancode.api import get_package_manifests
+        return get_package_manifests
 
     def process_codebase(self, codebase, **kwargs):
         """
@@ -94,12 +94,12 @@ def process_codebase(self, codebase, **kwargs):
         for resource in codebase.walk(topdown=False):
             set_packages_root(resource, codebase)
 
-        summarize_codebase_package_manifests(codebase, **kwargs)
+        create_packages_from_manifests(codebase, **kwargs)
 
 
-def summarize_codebase_package_manifests(codebase, **kwargs):
+def create_packages_from_manifests(codebase, **kwargs):
     """
-    Summarize codebase by package manifests.
+    Create package instances from package manifests present in the codebase.
     """
     package_manifests = []
 

src/scancode/api.py

@@ -287,7 +287,43 @@ def _licenses_data_from_match(
 SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
 
 
-def get_package_manifest_info(location, **kwargs):
+def get_package_info(location, **kwargs):
+    """
+    Return a mapping of package information detected in the
+    file at `location`.
+
+    Note that all exceptions are caught if there are any errors while parsing a
+    package.
+    """
+    import warnings
+    warnings.warn(
+        "`get_package_info` is deprecated. Use `get_package_manifests` instead.",
+        DeprecationWarning,
+        stacklevel=1
+    )
+
+    from packagedcode.recognize import recognize_package_manifests
+    try:
+        recognized_package_manifests = recognize_package_manifests(location)
+        if recognized_package_manifests:
+            return dict(package_manifests=[
+                package_manifest.to_dict()
+                for package_manifest in recognized_package_manifests
+            ])
+    except Exception as e:
+        if TRACE:
+            logger.error('get_package_info: {}: Exception: {}'.format(location, e))
+
+        if SCANCODE_DEBUG_PACKAGE_API:
+            raise
+        else:
+            # attention: we are swallowing ALL exceptions here!
+            pass
+
+    return dict(package_manifests=[])
+
+
+def get_package_manifests(location, **kwargs):
     """
     Return a mapping of package manifest information detected in the
     file at `location`.
@@ -305,7 +341,7 @@ def get_package_manifest_info(location, **kwargs):
             ])
     except Exception as e:
         if TRACE:
-            logger.error('get_package_info: {}: Exception: {}'.format(location, e))
+            logger.error('get_package_manifests: {}: Exception: {}'.format(location, e))
 
         if SCANCODE_DEBUG_PACKAGE_API:
             raise

src/summarycode/plugin_consolidate.py

@@ -98,21 +98,21 @@ def to_dict(self, **kwargs):
 
 
 @attr.s
-class ConsolidatedPackageManifest(object):
-    package_manifest = attr.ib()
+class ConsolidatedPackage(object):
+    package = attr.ib()
     consolidation = attr.ib()
 
     def to_dict(self, **kwargs):
-        package_manifest = self.package_manifest.to_dict()
-        package_manifest.update(self.consolidation.to_dict())
-        return package_manifest
+        package = self.package.to_dict()
+        package.update(self.consolidation.to_dict())
+        return package
 
 
 @post_scan_impl
 class Consolidator(PostScanPlugin):
     """
     A ScanCode post-scan plugin to return consolidated components and consolidated
-    package_manifests for different types of codebase summarization.
+    packages for different types of codebase summarization.
 
     A consolidated component is a group of Resources that have the same origin.
     Currently, a ConsolidatedComponent is created for each detected copyright holder
@@ -127,7 +127,7 @@ class Consolidator(PostScanPlugin):
     """
     codebase_attributes = dict(
         consolidated_components=attr.ib(default=attr.Factory(list)),
-        consolidated_package_manifests=attr.ib(default=attr.Factory(list))
+        consolidated_packages=attr.ib(default=attr.Factory(list))
     )
 
     resource_attributes = dict(
@@ -140,10 +140,9 @@ class Consolidator(PostScanPlugin):
         PluggableCommandLineOption(('--consolidate',),
             is_flag=True, default=False,
             help='Group resources by Packages or license and copyright holder and '
-                 'return those groupings as a list of consolidated package_manifests and '
+                 'return those groupings as a list of consolidated packages and '
                  'a list of consolidated components. '
-                 'This requires the scan to have/be run with the copyright, license, and '
-                 'package options active',
+                 'This requires the scan to have/be run with the copyright, license, and package options active',
             help_group=POST_SCAN_GROUP
         )
     ]
@@ -152,12 +151,12 @@ def is_enabled(self, consolidate, **kwargs):
         return consolidate
 
     def process_codebase(self, codebase, **kwargs):
-        # Collect ConsolidatedPackageManifests and ConsolidatedComponents
+        # Collect ConsolidatedPackages and ConsolidatedComponents
         # TODO: Have a "catch-all" Component for the things that we haven't grouped
         consolidations = []
         root = codebase.root
-        if hasattr(root, 'package_manifests') and hasattr(root, 'copyrights') and hasattr(root, 'licenses'):
-            consolidations.extend(get_consolidated_package_manifests(codebase))
+        if hasattr(root, 'packages') and hasattr(root, 'copyrights') and hasattr(root, 'licenses'):
+            consolidations.extend(get_consolidated_packages(codebase))
         if hasattr(root, 'copyrights') and hasattr(root, 'licenses'):
             consolidations.extend(get_holders_consolidated_components(codebase))
 
@@ -167,24 +166,24 @@ def process_codebase(self, codebase, **kwargs):
         # Sort consolidations by holders for consistent ordering before enumeration
         consolidations = sorted(consolidations, key=lambda c: '_'.join(h.key for h in c.consolidation.core_holders))
 
-        # Add ConsolidatedPackageManifests and ConsolidatedComponents to top-level codebase attributes
-        codebase.attributes.consolidated_package_manifests = consolidated_package_manifests = []
+        # Add ConsolidatedPackages and ConsolidatedComponents to top-level codebase attributes
+        codebase.attributes.consolidated_packages = consolidated_packages = []
         codebase.attributes.consolidated_components = consolidated_components = []
         identifier_counts = Counter()
         for index, c in enumerate(consolidations, start=1):
             # Skip consolidation if it does not have any Files
             if c.consolidation.files_count == 0:
                 continue
-            if isinstance(c, ConsolidatedPackageManifest):
+            if isinstance(c, ConsolidatedPackage):
                 # We use the purl as the identifier for ConsolidatedPackages
-                purl = c.package_manifest.purl
+                purl = c.package.purl
                 identifier_counts[purl] += 1
                 identifier = python_safe_name('{}_{}'.format(purl, identifier_counts[purl]))
                 c.consolidation.identifier = identifier
                 for resource in c.consolidation.resources:
                     resource.consolidated_to.append(identifier)
                     resource.save(codebase)
-                consolidated_package_manifests.append(c.to_dict())
+                consolidated_packages.append(c.to_dict())
             elif isinstance(c, ConsolidatedComponent):
                 consolidation_identifier = c.consolidation.identifier
                 if consolidation_identifier:
@@ -219,20 +218,20 @@ def process_codebase(self, codebase, **kwargs):
             resource.save(codebase)
 
 
-def get_consolidated_package_manifests(codebase):
+def get_consolidated_packages(codebase):
     """
-    Yield a ConsolidatedPackageManifest for each detected package_manifest in the codebase
+    Yield a ConsolidatedPackage for each detected package in the codebase
     """
     for resource in codebase.walk(topdown=False):
-        for package_manifest_data in resource.package_manifests:
-            package_manifest = get_package_instance(package_manifest_data)
-            package_root = package_manifest.get_package_root(resource, codebase)
+        for package_data in resource.packages:
+            package = get_package_instance(package_data)
+            package_root = package.get_package_root(resource, codebase)
             package_root.extra_data['package_root'] = True
             package_root.save(codebase)
-            is_build_file = isinstance(package_manifest, BaseBuildManifestPackage)
-            package_resources = list(package_manifest.get_package_resources(package_root, codebase))
-            package_license_expression = package_manifest.license_expression
-            package_copyright = package_manifest.copyright
+            is_build_file = isinstance(package, BaseBuildManifestPackage)
+            package_resources = list(package.get_package_resources(package_root, codebase))
+            package_license_expression = package.license_expression
+            package_copyright = package.copyright
 
             package_holders = []
             if package_copyright:
@@ -274,14 +273,14 @@ def get_consolidated_package_manifests(codebase):
                 resources=package_resources,
             )
             if is_build_file:
-                c.identifier = package_manifest.name
+                c.identifier = package.name
                 yield ConsolidatedComponent(
                     type='build',
                     consolidation=c
                 )
             else:
-                yield ConsolidatedPackageManifest(
-                    package_manifest=package_manifest,
+                yield ConsolidatedPackage(
+                    package=package,
                     consolidation=c
                 )
 

src/summarycode/score.py

@@ -166,31 +166,31 @@ def compute_license_score(codebase):
 
 def get_declared_license_keys(codebase):
     """
-    Return a list of declared license keys found in package_manifests and key files.
+    Return a list of declared license keys found in packages and key files.
     """
     return (
         get_declared_license_keys_in_key_files(codebase) +
-        get_declared_license_keys_in_package_manifests(codebase)
+        get_declared_license_keys_in_packages(codebase)
     )
 
 
-def get_declared_license_keys_in_package_manifests(codebase):
+def get_declared_license_keys_in_packages(codebase):
     """
-    Return a list of declared license keys found in package_manifests.
+    Return a list of declared license keys found in packages.
 
     A package manifest (such as Maven POM file or an npm package.json file)
     contains structured declared license information. This is further normalized
     as a license_expression. We extract the list of licenses from the normalized
     license expressions.
     """
-    package_manifests = chain.from_iterable(
-        getattr(res, 'package_manifests', []) or []
+    packages = chain.from_iterable(
+        getattr(res, 'packages', []) or []
         for res in codebase.walk(topdown=True))
 
     licensing = Licensing()
     detected_good_licenses = []
-    for package_manifest in package_manifests:
-        expression = package_manifest.get('license_expression')
+    for package in packages:
+        expression = package.get('license_expression')
         if expression:
             exp = licensing.parse(
                 expression, validate=False, strict=False, simple=True)

src/summarycode/summarizer.py

@@ -112,7 +112,7 @@ def summarize_codebase(codebase, keep_details, **kwargs):
         ('holders', holder_summarizer),
         ('authors', author_summarizer),
         ('programming_language', language_summarizer),
-        ('package_manifests', package_summarizer),
+        ('packages', package_summarizer),
     ]
 
     # find which attributes are available for summarization by checking the root
@@ -232,7 +232,7 @@ def summarize_languages(languages):
     'holders',
     'authors',
     'programming_language',
-    # 'package_manifests',
+    # 'packages',
 ])
 
 
@@ -293,7 +293,7 @@ def summarize_codebase_key_files(codebase, **kwargs):
     summarizables = codebase.attributes.summary.keys()
     if TRACE: logger_debug('summarizables:', summarizables)
 
-    # TODO: we cannot summarize package_manifests with "key files" for now
+    # TODO: we cannot summarize packages with "key files" for now
     summarizables = [k for k in summarizables if k in SUMMARIZABLE_ATTRS]
 
     # create one counter for each summarized attribute
@@ -408,53 +408,53 @@ def summarize_codebase_by_facet(codebase, **kwargs):
     if TRACE: logger_debug('codebase summary_by_facet:', final_summaries)
 
 
-def add_files(package_manifests, resource):
+def add_files(packages, resource):
     """
-    Update in-place every package mapping in the `package_manifests` list
-    by updating or creatig the the "files" attribute from the `resource`.
-    Yield back the package_manifests.
+    Update in-place every package mapping in the `packages` list by updating or
+    creating the the "files" attribute from the `resource`. Yield back the
+    packages.
     """
-    for package_manifest in package_manifests:
-        files = package_manifest['files'] = package_manifest.get('files') or []
+    for package in packages:
+        files = package['files'] = package.get('files') or []
         fil = resource.to_dict(skinny=True)
         if fil not in files:
             files.append(fil)
-        yield package_manifest
+        yield package
 
 
 def package_summarizer(resource, children, keep_details=False):
     """
-    Populate a package_manifests summary list of package_manifests mappings.
+    Populate a packages summary list of packages mappings.
 
     Note: `keep_details` is never used, as we are not keeping details of
-    package_manifests as this has no value.
+    packages as this has no value.
     """
-    package_manifests = []
+    packages = []
 
     # Collect current data
-    current_package_manifests = getattr(resource, 'package_manifests') or []
+    current_packages = getattr(resource, 'packages') or []
 
-    if TRACE_LIGHT and current_package_manifests:
+    if TRACE_LIGHT and current_packages:
         from packagedcode.models import Package
-        packs = [Package.create(**p) for p in current_package_manifests]
+        packs = [Package.create(**p) for p in current_packages]
         logger_debug('package_summarizer: for:', resource,
-                     'current_package_manifests are:', packs)
+                     'current_packages are:', packs)
 
-    current_package_manifests = add_files(current_package_manifests, resource)
-    package_manifests.extend(current_package_manifests)
+    current_packages = add_files(current_packages, resource)
+    packages.extend(current_packages)
 
-    if TRACE_LIGHT and package_manifests:
+    if TRACE_LIGHT and packages:
         logger_debug()
         from packagedcode.models import Package  # NOQA
-        packs = [Package.create(**p) for p in package_manifests]
+        packs = [Package.create(**p) for p in packages]
         logger_debug('package_summarizer: for:', resource,
-                     'package_manifests are:', packs)
+                     'packages are:', packs)
 
-    # Collect direct children package_manifests summary
+    # Collect direct children packages summary
     for child in children:
-        child_summaries = get_resource_summary(child, key='package_manifests', as_attribute=False) or []
-        package_manifests.extend(child_summaries)
+        child_summaries = get_resource_summary(child, key='packages', as_attribute=False) or []
+        packages.extend(child_summaries)
 
     # summarize proper
-    set_resource_summary(resource, key='package_manifests', value=package_manifests, as_attribute=False)
-    return package_manifests
+    set_resource_summary(resource, key='packages', value=packages, as_attribute=False)
+    return packages