Use license/notice files from About data

Reference: #1004

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

scanpipe/pipes/d2d.py

@@ -921,6 +921,26 @@ def map_deployed_to_devel_using_about(self, to_resources):
 
         return mapped_to_resources
 
+    def get_about_file_companions(self, about_path):
+        """
+        Given an ``about_path`` path string to an About file,
+        get CodebaseResource objects for the companion license
+        and notice files.
+        """
+        about_file_resource = self.about_resources_by_path.get(about_path)
+        about_file_extra_data = self.about_pkgdata_by_path.get(about_path).get(
+            "extra_data"
+        )
+
+        about_file_companion_names = [
+            about_file_extra_data.get("license_file"),
+            about_file_extra_data.get("notice_file"),
+        ]
+        about_file_companions = about_file_resource.siblings().filter(
+            name__in=about_file_companion_names
+        )
+        return about_file_companions
+
     def create_about_packages_relations(self, project):
         """
         Create packages using About file package data, if the About file
@@ -932,11 +952,11 @@ def create_about_packages_relations(self, project):
 
         for about_path, mapped_resources in self.mapped_resources_by_aboutpath.items():
             about_file_resource = self.about_resources_by_path[about_path]
-            package_data = self.about_pkgdata_by_path[about_file_resource.path]
+            package_data = self.about_pkgdata_by_path[about_path]
 
             if not mapped_resources:
                 error_message_details = {
-                    "path": about_file_resource.path,
+                    "path": about_path,
                     "package_data": package_data,
                 }
                 project.add_warning(
@@ -968,12 +988,7 @@ def create_about_packages_relations(self, project):
 
             about_file_resource.update(status=flag.ABOUT_MAPPED)
 
-        for about_file_resource in self.about_resources_by_path.values():
-            about_file_companions = (
-                about_file_resource.siblings()
-                .filter(name__startswith=about_file_resource.name_without_extension)
-                .filter(extension__in=[".LICENSE", ".NOTICE"])
-            )
+            about_file_companions = self.get_about_file_companions(about_path)
             about_file_companions.update(status=flag.ABOUT_MAPPED)
 
         return about_purls, mapped_about_resources

scanpipe/pipes/resolve.py

@@ -86,18 +86,15 @@ def resolve_about_package(input_location):
             if value:
                 package_data[field_name] = value
 
+    package_data["extra_data"] = {}
+
     if about_resource := about_data.get("about_resource"):
         package_data["filename"] = list(about_resource.keys())[0]
 
     if ignored_resources := about_data.get("ignored_resources"):
-        extra_data = {"ignored_resources": list(ignored_resources.keys())}
-        package_data["extra_data"] = extra_data
-
-    if license_expression := about_data.get("license_expression"):
-        package_data["declared_license_expression"] = license_expression
+        package_data["extra_data"]["ignored_resources"] = list(ignored_resources.keys())
 
-    if notice_dict := about_data.get("notice_file"):
-        package_data["notice_text"] = list(notice_dict.values())[0]
+    populate_license_notice_fields_about(package_data, about_data)
 
     for field_name, value in about_data.items():
         if field_name.startswith("checksum_"):
@@ -107,6 +104,23 @@ def resolve_about_package(input_location):
     return package_data
 
 
+def populate_license_notice_fields_about(package_data, about_data):
+    """
+    Populate ``package_data`` with license and notice attributes
+    from ``about_data``.
+    """
+    if license_expression := about_data.get("license_expression"):
+        package_data["declared_license_expression"] = license_expression
+
+    if notice_dict := about_data.get("notice_file"):
+        package_data["notice_text"] = list(notice_dict.values())[0]
+        package_data["extra_data"]["notice_file"] = list(notice_dict.keys())[0]
+
+    if license_dict := about_data.get("license_file"):
+        package_data["extra_data"]["license_file"] = list(license_dict.keys())[0]
+        package_data["extracted_license_statement"] = list(license_dict.values())[0]
+
+
 def resolve_about_packages(input_location):
     """
     Wrap ``resolve_about_package`` to return a list as expected by the

scanpipe/tests/data/d2d/about_files/expected.json

@@ -111,10 +111,12 @@
       "other_license_expression": "",
       "other_license_expression_spdx": "",
       "other_license_detections": [],
-      "extracted_license_statement": "",
+      "extracted_license_statement": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   ",
       "notice_text": "notice",
       "source_packages": [],
       "extra_data": {
+        "notice_file": "flume-ng-node-1.9.0-sources.NOTICE",
+        "license_file": "flume-ng-node-1.9.0-sources.LICENSE",
         "ignored_resources": [
           "*flume-ng-node-*.jar-extract/org/apache/flume/node/ConfigurationProvider.class"
         ]

scanpipe/tests/pipes/test_resolve.py

@@ -92,6 +92,8 @@ def test_scanpipe_pipes_resolve_resolve_packages(self):
             "filename": "Django-4.0.8-py3-none-any.whl",
             "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl",
             "declared_license_expression": "bsd-new",
+            "extra_data": {"license_file": "bsd-new.LICENSE"},
+            "extracted_license_statement": None,
             "md5": "386349753c386e574dceca5067e2788a",
             "name": "django",
             "sha1": "4cc6f7abda928a0b12cd1f1cd8ad3677519ca04e",
@@ -114,6 +116,8 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self):
             "filename": "Django-4.0.8-py3-none-any.whl",
             "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl",
             "declared_license_expression": "bsd-new",
+            "extra_data": {"license_file": "bsd-new.LICENSE"},
+            "extracted_license_statement": None,
             "md5": "386349753c386e574dceca5067e2788a",
             "name": "django",
             "sha1": "4cc6f7abda928a0b12cd1f1cd8ad3677519ca04e",
@@ -124,7 +128,7 @@ def test_scanpipe_pipes_resolve_resolve_about_packages(self):
 
         input_location = self.manifest_location / "poor_values.ABOUT"
         package = resolve.resolve_about_packages(str(input_location))
-        expected = {"name": "project"}
+        expected = {"extra_data": {}, "name": "project"}
         self.assertEqual([expected], package)
 
     def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self):