Add support for license clues in --todo

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -279,7 +279,10 @@ def _identifier(self):
         """
         data = []
         for match in self.matches:
-            tokenized_matched_text = tuple(query_tokenizer(match.matched_text()))
+            if isinstance(match.matched_text, str):
+                tokenized_matched_text = tuple(query_tokenizer(match.matched_text))
+            else:
+                tokenized_matched_text = tuple(query_tokenizer(match.matched_text()))
             identifier = (
                 match.rule.identifier,
                 match.score(),
@@ -750,6 +753,16 @@ def get_unique_detections(cls, license_detections):
                 if detection.detection_log:
                     detection_log.extend(detection.detection_log)
 
+            if not detection.license_expression:
+                detection.license_expression = str(combine_expressions(
+                    expressions=[
+                        match.rule.license_expression
+                        for match in detection.matches
+                    ]
+                ))
+                detection.identifier = detection.identifier_with_expression 
+
+
             unique_license_detections.append(
                 cls(
                     identifier=detection.identifier,
@@ -1341,7 +1354,10 @@ def get_ambiguous_license_detections_by_type(unique_license_detections):
     ambi_license_detections = {}
 
     for detection in unique_license_detections:
-        if is_undetected_license_matches(license_matches=detection.matches):
+        if not detection.license_expression:
+            ambi_license_detections[DetectionCategory.MATCH_FRAGMENTS.value] = detection
+
+        elif is_undetected_license_matches(license_matches=detection.matches):
             ambi_license_detections[DetectionCategory.UNDETECTED_LICENSE.value] = detection
 
         elif "unknown" in detection.license_expression:

src/summarycode/todo.py

@@ -368,6 +368,9 @@ def get_review_comments(detection_log):
     if LicenseDetectionCategory.UNKNOWN_MATCH.value in detection_log:
         review_comments[LicenseDetectionCategory.UNKNOWN_MATCH.value] = ReviewComments.UNKNOWN_MATCH.value
 
+    if LicenseDetectionCategory.MATCH_FRAGMENTS.value in detection_log:
+        review_comments[LicenseDetectionCategory.MATCH_FRAGMENTS.value] = ReviewComments.MATCH_FRAGMENTS.value
+
     if LicenseDetectionCategory.LICENSE_CLUES.value in detection_log:
         review_comments[LicenseDetectionCategory.LICENSE_CLUES.value] = ReviewComments.LICENSE_CLUES.value
 

…todo/base64-arraybuffer-0.1.4/.npmignore …todo/base64-arraybuffer-0.1.4/.npmignoretests/summarycode/data/review/no_todo/base64-arraybuffer-0.1.4/.npmignore renamed to tests/summarycode/data/todo/no_todo/base64-arraybuffer-0.1.4/.npmignore tests/summarycode/data/review/no_todo/base64-arraybuffer-0.1.4/.npmignore renamed to tests/summarycode/data/todo/no_todo/base64-arraybuffer-0.1.4/.npmignore

@@ -0,0 +1,7 @@
+This src:package consists of various tarballs.
+
+This README is a dummy file for creating the base tarball of the name
+
+  fusiondirectory_<upstream-version>.orig.tar.gz
+
+ -- Mike Gabriel <sunweaver@debian.org>  Tue, 01 Apr 2014 16:36:31 +0200

…odo/base64-arraybuffer-0.1.4/LICENSE-MIT …odo/base64-arraybuffer-0.1.4/LICENSE-MITtests/summarycode/data/review/no_todo/base64-arraybuffer-0.1.4/LICENSE-MIT renamed to tests/summarycode/data/todo/no_todo/base64-arraybuffer-0.1.4/LICENSE-MIT tests/summarycode/data/review/no_todo/base64-arraybuffer-0.1.4/LICENSE-MIT renamed to tests/summarycode/data/todo/no_todo/base64-arraybuffer-0.1.4/LICENSE-MIT

@@ -0,0 +1,62 @@
+{
+  "license_detections": [],
+  "todo": [
+    {
+      "detection_id": "borceux-3c39742c-edef-82b7-0cdd-fc4d9ff8b044",
+      "review_comments": {
+        "imperfect-match-coverage": "The license detection likely is not conslusive as there was license matches with low score or coverage, and so this needs review. scancode would likely benifit from a license rule addition from this case, so please report this to scancode-toolkit github issues."
+      },
+      "detection": {
+        "license_expression": "borceux",
+        "matches": [
+          {
+            "score": 4.71,
+            "start_line": 1,
+            "end_line": 3,
+            "matched_length": 4,
+            "match_coverage": 4.71,
+            "matcher": "3-seq",
+            "license_expression": "borceux",
+            "rule_identifier": "borceux.LICENSE",
+            "rule_relevance": 100,
+            "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/borceux.LICENSE",
+            "matched_text": "package consists of [various] [tarballs].\n\n[This] README"
+          }
+        ],
+        "detection_log": [
+          "license-clues"
+        ],
+        "identifier": "borceux-3c39742c-edef-82b7-0cdd-fc4d9ff8b044"
+      }
+    }
+  ],
+  "files": [
+    {
+      "path": "README.multi-orig-tarball-package",
+      "type": "file",
+      "detected_license_expression": null,
+      "detected_license_expression_spdx": null,
+      "license_detections": [],
+      "license_clues": [
+        {
+          "score": 4.71,
+          "start_line": 1,
+          "end_line": 3,
+          "matched_length": 4,
+          "match_coverage": 4.71,
+          "matcher": "3-seq",
+          "license_expression": "borceux",
+          "rule_identifier": "borceux.LICENSE",
+          "rule_relevance": 100,
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/borceux.LICENSE",
+          "matched_text": "package consists of [various] [tarballs].\n\n[This] README"
+        }
+      ],
+      "percentage_of_license_text": 10.53,
+      "for_todo": [
+        "borceux-3c39742c-edef-82b7-0cdd-fc4d9ff8b044"
+      ],
+      "scan_errors": []
+    }
+  ]
+}