More License Detection changes

src/formattedcode/output_debian.py

@@ -17,7 +17,7 @@
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from licensedcode.detection import get_matches_from_detection_mappings
-
+from licensedcode.plugin_licenses_reference import get_matched_text_from_reference_data
 from scancode import notice
 
 """
@@ -106,7 +106,7 @@ def build_copyright_paragraphs(codebase, **kwargs):
         if scanned_file['type'] == 'directory':
             continue
         dfiles = scanned_file['path']
-        dlicense = build_license(scanned_file)
+        dlicense = build_license(codebase, scanned_file)
         dcopyright = build_copyright_field(scanned_file)
 
         file_para = CopyrightFilesParagraph.from_dict(dict(
@@ -132,7 +132,7 @@ def build_copyright_field(scanned_file):
     return '\n'.join(statements)
 
 
-def build_license(scanned_file):
+def build_license(codebase, scanned_file):
     """
     Return Debian-like text where the first line is the expression and the
     remaining lines are the license text from licenses detected in
@@ -146,11 +146,11 @@ def build_license(scanned_file):
         return
 
     licenses = scanned_file.get('license_detections', [])
-    text = '\n'.join(get_texts(licenses))
+    text = '\n'.join(get_texts(codebase, licenses))
     return f'{expression}\n{text}'
 
 
-def get_texts(detected_licenses):
+def get_texts(codebase, detected_licenses):
     """
     Yield license texts detected in this file.
 
@@ -179,8 +179,12 @@ def get_texts(detected_licenses):
     # set of (start line, end line, matched_rule identifier)
     seen = set()
     for lic in get_matches_from_detection_mappings(detected_licenses):
+        matched_text = get_matched_text_from_reference_data(
+            codebase=codebase,
+            rule_identifier=lic['rule_identifier']
+        )
         key = lic['start_line'], lic['end_line'], lic['rule_identifier']
         if key not in seen:
-            yield lic['matched_text']
+            if matched_text != None:
+                yield matched_text
             seen.add(key)
-

src/formattedcode/output_spdx.py

@@ -23,13 +23,15 @@
 from spdx.utils import SPDXNone
 from spdx.version import Version
 
+from license_expression import Licensing
 from commoncode.cliutils import OUTPUT_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.fileutils import file_name
 from commoncode.fileutils import parent_directory
 from commoncode.text import python_safe_name
 from formattedcode import FileOptionType
 from licensedcode.detection import get_matches_from_detection_mappings
+from licensedcode.plugin_licenses_reference import get_matched_text_from_reference_data
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 import scancode_config
@@ -170,6 +172,7 @@ def _process_codebase(
     package_name = build_package_name(input_path)
 
     write_spdx(
+        codebase=codebase,
         output_file=output_file,
         files=files,
         tool_name=tool_name,
@@ -208,6 +211,7 @@ def check_sha1(codebase):
 
 
 def write_spdx(
+    codebase,
     output_file,
     files,
     tool_name,
@@ -229,6 +233,10 @@ def write_spdx(
     producing this SPDX document.
     Use ``package_name`` as a Package name and as a namespace prefix base.
     """
+    from licensedcode import cache
+    licenses = cache.get_licenses_db()
+    licensing = Licensing()
+
     as_rdf = not as_tagvalue
     _patch_license_list()
 
@@ -282,11 +290,20 @@ def write_spdx(
         if license_matches:
             all_files_have_no_license = False
             for match in license_matches:
-                file_licenses = match["licenses"]
-                for file_license in file_licenses:
-                    license_key = file_license.get('key')
-
-                    spdx_id = file_license.get('spdx_license_key')
+                file_license_expression = match["license_expression"]
+                file_license_keys = licensing.license_keys(
+                    expression=file_license_expression,
+                    unique=True
+                )
+                matched_text = get_matched_text_from_reference_data(
+                    codebase=codebase,
+                    rule_identifier=match["rule_identifier"],
+                )
+                for license_key in file_license_keys:
+                    file_license = licenses.get(license_key)
+                    license_key = file_license.key
+
+                    spdx_id = file_license.spdx_license_key
                     if not spdx_id:
                         spdx_id = f'LicenseRef-scancode-{license_key}'
                     is_license_ref = spdx_id.lower().startswith('licenseref-')
@@ -295,7 +312,7 @@ def write_spdx(
                         spdx_license = License.from_identifier(spdx_id)
                     else:
                         spdx_license = ExtractedLicense(spdx_id)
-                        spdx_license.name = file_license.get('short_name')
+                        spdx_license.name = file_license.short_name
                         # FIXME: replace this with the licensedb URL
                         comment = (
                             f'See details at https://github.com/nexB/scancode-toolkit'