Make summary work with new package code

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/packagedcode/cargo.py

@@ -90,7 +90,7 @@ def assemble(cls, package_data, resource, codebase):
         Assemble Cargo.toml and possible Cargo.lock datafiles
         """
         yield from cls.assemble_from_many_datafiles(
-            datafile_name_patterns=('Cargo.toml', 'Cargo.lock',),
+            datafile_name_patterns=('Cargo.toml', 'cargo.toml', 'Cargo.lock', 'cargo.lock'),
             directory=resource.parent(codebase),
             codebase=codebase,
         )

src/packagedcode/plugin_package.py

@@ -11,7 +11,6 @@
 import click
 import os
 import logging
-import sys
 
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import DOC_GROUP

src/summarycode/classify.py

@@ -234,6 +234,8 @@ def set_classification_flags(resource,
 
     resource.is_legal = is_legal = check_resource_name_start_and_end(resource, _LEGAL)
     resource.is_readme = is_readme = check_resource_name_start_and_end(resource, _README)
-    resource.is_manifest = is_manifest = path.endswith(_MANIF) or getattr(resource, 'package_data', None)
+    # FIXME: this will never be picked up as this is NOT available in a pre-scan plugin
+    has_package_data = bool(getattr(resource, 'package_data', False))
+    resource.is_manifest = is_manifest = path.endswith(_MANIF) or has_package_data
     resource.is_key_file = (resource.is_top_level and (is_readme or is_legal or is_manifest))
     return resource

src/summarycode/score.py

@@ -9,14 +9,13 @@
 
 import attr
 
-from packagedcode.utils import combine_expressions
-from plugincode.post_scan import PostScanPlugin
-from plugincode.post_scan import post_scan_impl
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import POST_SCAN_GROUP
-
 from license_expression import Licensing
+from plugincode.post_scan import PostScanPlugin
+from plugincode.post_scan import post_scan_impl
 
+from packagedcode.utils import combine_expressions
 
 # Tracing flags
 TRACE = False
@@ -48,12 +47,14 @@ class LicenseClarityScore(PostScanPlugin):
     """
     Compute a License clarity score at the codebase level.
     """
+
     codebase_attributes = dict(summary=attr.ib(default=attr.Factory(dict)))
 
     sort_order = 5
 
     options = [
-        PluggableCommandLineOption(('--license-clarity-score',),
+        PluggableCommandLineOption(
+            ('--license-clarity-score',),
             is_flag=True,
             default=False,
             help='Compute a summary license clarity score at the codebase level.',
@@ -75,7 +76,6 @@ def process_codebase(self, codebase, license_clarity_score, **kwargs):
         codebase.attributes.summary['license_clarity_score'] = scoring_elements.to_dict()
 
 
-
 def compute_license_score(codebase):
     """
     Return a mapping of scoring elements and a license clarity score computed at
@@ -124,12 +124,25 @@ def compute_license_score(codebase):
     """
 
     scoring_elements = ScoringElements()
-    declared_licenses = get_field_values_from_codebase_resources(codebase, 'licenses', key_files_only=True)
-    declared_license_expressions = get_field_values_from_codebase_resources(codebase, 'license_expressions', key_files_only=True)
+    declared_licenses = get_field_values_from_codebase_resources(
+        codebase=codebase,
+        field_name='licenses',
+        key_files_only=True,
+    )
+    declared_license_expressions = get_field_values_from_codebase_resources(
+        codebase=codebase, field_name='license_expressions', key_files_only=True
+    )
+
     unique_declared_license_expressions = unique(declared_license_expressions)
     declared_license_categories = get_license_categories(declared_licenses)
-    copyrights = get_field_values_from_codebase_resources(codebase, 'copyrights', key_files_only=True)
-    other_licenses = get_field_values_from_codebase_resources(codebase, 'licenses', key_files_only=False)
+
+    copyrights = get_field_values_from_codebase_resources(
+        codebase=codebase, field_name='copyrights', key_files_only=True
+    )
+
+    other_licenses = get_field_values_from_codebase_resources(
+        codebase=codebase, field_name='licenses', key_files_only=False
+    )
 
     scoring_elements.declared_license = bool(declared_licenses)
     if scoring_elements.declared_license:
@@ -149,20 +162,23 @@ def compute_license_score(codebase):
 
     is_permissively_licensed = check_declared_license_categories(declared_license_categories)
     if is_permissively_licensed:
-        scoring_elements.conflicting_license_categories = check_for_conflicting_licenses(other_licenses)
-        if (
-            scoring_elements.conflicting_license_categories
-            and scoring_elements.score > 0
-        ):
+        scoring_elements.conflicting_license_categories = check_for_conflicting_licenses(
+            other_licenses
+        )
+        if scoring_elements.conflicting_license_categories and scoring_elements.score > 0:
             scoring_elements.score -= 20
 
     declared_license_expression = get_primary_license(unique_declared_license_expressions)
 
     if not declared_license_expression:
         # If we cannot get a single primary license, then we combine and simplify the license expressions from key files
-        combined_declared_license_expression = combine_expressions(unique_declared_license_expressions)
+        combined_declared_license_expression = combine_expressions(
+            unique_declared_license_expressions
+        )
         if combined_declared_license_expression:
-            declared_license_expression = str(Licensing().parse(combined_declared_license_expression).simplify())
+            declared_license_expression = str(
+                Licensing().parse(combined_declared_license_expression).simplify()
+            )
         scoring_elements.ambigous_compound_licensing = True
         if scoring_elements.score > 0:
             scoring_elements.score -= 10
@@ -201,7 +217,7 @@ def to_dict(self):
             'has_license_text': self.has_license_text,
             'declared_copyrights': self.declared_copyrights,
             'conflicting_license_categories': self.conflicting_license_categories,
-            'ambigous_compound_licensing': self.ambigous_compound_licensing
+            'ambigous_compound_licensing': self.ambigous_compound_licensing,
         }
 
 
@@ -235,13 +251,15 @@ def is_good_license(detected_license):
     rule = detected_license['matched_rule']
     coverage = rule.get('match_coverage') or 0
     relevance = rule.get('rule_relevance') or 0
-    match_types = dict([
-        ('is_license_text', rule['is_license_text']),
-        ('is_license_notice', rule['is_license_notice']),
-        ('is_license_reference', rule['is_license_reference']),
-        ('is_license_tag', rule['is_license_tag']),
-        ('is_license_intro', rule['is_license_intro']),
-    ])
+    match_types = dict(
+        [
+            ('is_license_text', rule['is_license_text']),
+            ('is_license_notice', rule['is_license_notice']),
+            ('is_license_reference', rule['is_license_reference']),
+            ('is_license_tag', rule['is_license_tag']),
+            ('is_license_intro', rule['is_license_intro']),
+        ]
+    )
     matched = False
     for match_type, mval in match_types.items():
         if mval:
@@ -256,9 +274,11 @@ def is_good_license(detected_license):
         if score >= thresholds.min_score:
             return True
     else:
-        if (score >= thresholds.min_score
-        and coverage >= thresholds.min_coverage
-        and relevance >= thresholds.min_relevance):
+        if (
+            score >= thresholds.min_score
+            and coverage >= thresholds.min_coverage
+            and relevance >= thresholds.min_relevance
+        ):
             return True
 
     return False
@@ -270,11 +290,7 @@ def check_declared_licenses(declared_licenses):
 
     If so, return True. Otherwise, return False.
     """
-    return any(
-        is_good_license(declared_license)
-        for declared_license
-        in declared_licenses
-    )
+    return any(is_good_license(declared_license) for declared_license in declared_licenses)
 
 
 def get_field_values_from_codebase_resources(codebase, field_name, key_files_only=False):
@@ -324,10 +340,12 @@ def check_for_license_texts(declared_licenses):
     """
     for declared_license in declared_licenses:
         matched_rule = declared_license.get('matched_rule', {})
-        if any([
-            matched_rule.get('is_license_text', False),
-            matched_rule.get('is_license_notice', False),
-        ]):
+        if any(
+            [
+                matched_rule.get('is_license_text', False),
+                matched_rule.get('is_license_notice', False),
+            ]
+        ):
             return True
     return False
 
@@ -362,10 +380,7 @@ def check_for_conflicting_licenses(other_licenses):
     If so, return True. Otherwise, return False.
     """
     for license_info in other_licenses:
-        if (
-            license_info.get('category', '')
-            in CONFLICTING_LICENSE_CATEGORIES
-        ):
+        if license_info.get('category', '') in CONFLICTING_LICENSE_CATEGORIES:
             return True
     return False
 
@@ -402,10 +417,7 @@ def group_license_expressions(unique_license_expressions):
                 break
             for j1 in joined_expressions[i:]:
                 if licensing.is_equivalent(j, j1):
-                    if (
-                        j not in unique_joined_expressions
-                        and j not in seen_joined_expression
-                    ):
+                    if j not in unique_joined_expressions and j not in seen_joined_expression:
                         unique_joined_expressions.append(j)
                         seen_joined_expression.append(j1)
     else:
@@ -445,9 +457,7 @@ def get_primary_license(declared_license_expressions):
     # Group single expressions to joined expressions to see if single
     # expressions are accounted for in a joined expression
     single_expressions_by_joined_expressions = {
-        joined_expression: []
-        for joined_expression
-        in unique_joined_expressions
+        joined_expression: [] for joined_expression in unique_joined_expressions
     }
     not_in_joined_expressions = []
     # Check to see if the single expression is in the joined expression