Improve License Detection reporting

src/formattedcode/output_csv.py

@@ -146,7 +146,7 @@ def collect_keys(mapping, key_group):
 
         for detection in scanned_file.get('license_detections', []):
             license_expression = detection["license_expression"]
-            detection_log = detection["detection_log"]
+            detection_log = detection.get("detection_log", []) or []
             detection_log = '\n'.join(detection_log)
             license_matches = detection["matches"]
             for match in license_matches:

src/licensedcode/detection.py

@@ -114,7 +114,6 @@ class DetectionRule(Enum):
 
     These are logged in LicenseDetection.detection_log for verbosity.
     """
-    NOT_COMBINED = 'not-combined'
     UNKNOWN_MATCH = 'unknown-match'
     LICENSE_CLUES = 'license-clues'
     FALSE_POSITIVE = 'possible-false-positive'
@@ -159,6 +158,13 @@ class LicenseDetection:
             'using the SPDX license expression syntax and ScanCode license keys.')
     )
 
+    matches = attr.ib(
+        default=attr.Factory(list),
+        metadata=dict(
+            help='List of license matches combined in this detection.'
+        )
+    )
+
     detection_log = attr.ib(
         repr=False,
         default=attr.Factory(list),
@@ -168,13 +174,14 @@ class LicenseDetection:
         )
     )
 
-    matches = attr.ib(
-        default=attr.Factory(list),
+    identifier = attr.ib(
+        default=None,
         metadata=dict(
-            help='List of license matches combined in this detection.'
-        )
+            help='An identifier unique for a license detection, containing the license '
+            'expression and a UUID crafted from the match contents.')
     )
 
+
     # Only used in unique detection calculation and referencing
     file_region = attr.ib(
         default=attr.Factory(dict),
@@ -222,11 +229,13 @@ def from_matches(
                 detection_log=detection_log,
             )
 
-        return cls(
+        detection = cls(
             matches=matches,
             license_expression=str(license_expression),
             detection_log=detection_log,
         )
+        detection.identifier = detection.identifier_with_expression
+        return detection
 
     def __eq__(self, other):
         return (
@@ -259,25 +268,23 @@ def get_file_region(self, path):
         )
 
     @property
-    def identifier(self):
+    def _identifier(self):
         """
         Return an unique identifier for a license detection, based on it's
         underlying license matches with the tokenized matched_text.
         """
         data = []
         for match in self.matches:
-            tokenized_matched_text = tuple(query_tokenizer(match.matched_text))
+            tokenized_matched_text = tuple(query_tokenizer(match.matched_text()))
             identifier = (
-                match.identifier,
+                match.rule.identifier,
                 match.score(),
                 tokenized_matched_text,
             )
             data.append(identifier)
 
         # Return a uuid generated from the contents of the matches
-        identifier_string = repr(tuple(data))
-        md_hash = sha1(identifier_string.encode('utf-8'))
-        return str(uuid.UUID(hex=md_hash.hexdigest()[:32]))
+        return get_uuid_on_content(content=data)
 
     @property
     def identifier_with_expression(self):
@@ -286,7 +293,7 @@ def identifier_with_expression(self):
         and an UUID created from the detection contents.
         """
         id_safe_expression = python_safe_name(s=str(self.license_expression))
-        return "{}-{}".format(id_safe_expression, self.identifier)
+        return "{}-{}".format(id_safe_expression, self._identifier)
 
     def get_start_end_line(self):
         """
@@ -416,6 +423,7 @@ def to_dict(
         self,
         include_text=False,
         license_text_diagnostics=False,
+        license_diagnostics=False,
         whole_lines=True,
     ):
         """
@@ -426,6 +434,9 @@ def dict_fields(attr, value):
             if attr.name == 'file_region':
                 return False
 
+            if attr.name == 'detection_log' and not license_diagnostics:
+                return False
+
             return True
 
         data_matches = []
@@ -445,6 +456,16 @@ def dict_fields(attr, value):
         return detection
 
 
+def get_uuid_on_content(content):
+    """
+    Return an UUID based on the contents of a list, which should be
+    a list of hashable elements.
+    """
+    identifier_string = repr(tuple(content))
+    md_hash = sha1(identifier_string.encode('utf-8'))
+    return str(uuid.UUID(hex=md_hash.hexdigest()[:32]))
+
+
 @attr.s
 class LicenseDetectionFromResult(LicenseDetection):
     """
@@ -471,7 +492,8 @@ def from_license_detection_mapping(
 
         detection = cls(
             license_expression=license_detection_mapping["license_expression"],
-            detection_log=license_detection_mapping["detection_log"],
+            detection_log=license_detection_mapping.get("detection_log", []) or None,
+            identifier=license_detection_mapping["identifier"],
             matches=matches,
             file_region=None,
         )
@@ -500,6 +522,20 @@ def detections_from_license_detection_mappings(
     return license_detections
 
 
+def get_new_identifier_from_detections(initial_detection, detections_added):
+    """
+    Return a new UUID based on two sets of detections: `initial_detection` is
+    the detection being modified with a list of detections (from another file region)
+    `detections_added`.
+    """
+    identifiers = [
+        detection_mapping["identifier"]
+        for detection_mapping in detections_added
+    ]
+    identifiers.append(initial_detection["identifier"])
+    return get_uuid_on_content(content=sorted(identifiers))
+
+
 @attr.s
 class LicenseMatchFromResult(LicenseMatch):
     """
@@ -584,9 +620,9 @@ class UniqueDetection:
     """
     identifier = attr.ib(default=None)
     license_expression = attr.ib(default=None)
-    count = attr.ib(default=None)
-    detection_log = attr.ib(default=attr.Factory(list))
+    detection_count = attr.ib(default=None)
     matches = attr.ib(default=attr.Factory(list))
+    detection_log = attr.ib(default=attr.Factory(list))
     files = attr.ib(factory=list)
 
     @classmethod
@@ -608,23 +644,30 @@ def get_unique_detections(cls, license_detections):
             detection_mapping = detection.to_dict()
             unique_license_detections.append(
                 cls(
-                    identifier=detection.identifier_with_expression,
+                    identifier=detection_mapping["identifier"],
                     license_expression=detection_mapping["license_expression"],
-                    detection_log=detection_mapping["detection_log"],
+                    detection_log=detection_mapping.get("detection_log", []) or [],
                     matches=detection_mapping["matches"],
-                    count=len(file_regions),
+                    detection_count=len(file_regions),
                     files=file_regions,
                 )
             )
 
         return unique_license_detections
 
-    def to_dict(self):
+    def to_dict(self, license_diagnostics):
 
         def dict_fields(attr, value):
+
             if attr.name == 'files':
                 return False
 
+            if attr.name == 'matches':
+                return False
+
+            if attr.name == 'detection_log' and not license_diagnostics:
+                return False
+
             return True
 
         return attr.asdict(self, filter=dict_fields)
@@ -1049,9 +1092,8 @@ def get_detected_license_expression(
 
     else:
         if TRACE_ANALYSIS:
-            logger_debug(f'analysis {DetectionRule.NOT_COMBINED.value}')
+            logger_debug(f'analysis not-combined')
         matches_for_expression = license_matches
-        detection_log.append(DetectionRule.NOT_COMBINED.value)
 
     if TRACE:
         logger_debug(f'matches_for_expression: {matches_for_expression}', f'detection_log: {detection_log}')
@@ -1355,6 +1397,7 @@ def process_detections(detections, licensing=Licensing()):
                         licensing=licensing,
                     ))
                     detection.detection_log.append(DetectionRule.NOT_LICENSE_CLUES.value)
+                    detection.identifier = detection.identifier_with_expression
 
             yield detection
 

src/licensedcode/licenses_reference.py

@@ -14,6 +14,8 @@
 from licensedcode.models import Rule
 from plugincode.post_scan import PostScanPlugin
 from plugincode.post_scan import post_scan_impl
+from commoncode.cliutils import PluggableCommandLineOption
+from commoncode.cliutils import POST_SCAN_GROUP
 import attr
 
 TRACE = os.environ.get('SCANCODE_DEBUG_LICENSE_REFERENCE', False)
@@ -48,8 +50,18 @@ class LicenseReference(PostScanPlugin):
     # TODO: send to the tail of the scan, after files
     sort_order = 1000
 
-    def is_enabled(self, **kwargs):
-        return kwargs.get('license') or kwargs.get('package')
+    options = [
+        PluggableCommandLineOption(('--license-references',),
+            is_flag=True,
+            help='Return reference data for all licenses and license rules'
+                 'present in detections.',
+            help_group=POST_SCAN_GROUP,
+            sort_order=100,
+        )
+    ]
+
+    def is_enabled(self, license_references, **kwargs):
+        return license_references
 
     def process_codebase(self, codebase, **kwargs):
         """

src/licensedcode/match.py

@@ -790,6 +790,7 @@ def to_dict(
         # LicenseDB Level Information (Rule that was matched)
         result['license_expression'] = self.rule.license_expression
         result['rule_identifier'] = self.rule.identifier
+        result['rule_relevance'] = self.rule.relevance
         result['rule_url'] = self.rule.rule_url
 
         if include_text: