Update inspect_packages pipeline

* Split package/dependencies creation in a seperate step
* Only create packages/dependencies from Assemblable PackageData

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

scanpipe/pipelines/inspect_packages.py

@@ -49,6 +49,7 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_ignored_resources,
             cls.scan_for_application_packages,
+            cls.create_packages_and_dependencies,
         )
 
     def scan_for_application_packages(self):
@@ -60,5 +61,7 @@ def scan_for_application_packages(self):
         # only detect package_data in resources and create
         # Package/Dependency instances directly instead of assembling
         # the packages and assigning files to them
-        scancode.scan_for_application_packages(self.project, assemble=False)
+        scancode.scan_for_application_packages(project=self.project, assemble=False)
+
+    def create_packages_and_dependencies(self):
         scancode.process_package_data(self.project)

scanpipe/pipes/scancode.py

@@ -39,6 +39,7 @@
 from extractcode import api as extractcode_api
 from packagedcode import get_package_handler
 from packagedcode import models as packagedcode_models
+from packagedcode import HANDLER_BY_DATASOURCE_ID
 from scancode import Scanner
 from scancode import api as scancode_api
 from scancode import cli as scancode_cli
@@ -461,6 +462,9 @@ def process_package_data(project):
         logger.info(f"  Processing: {resource.path}")
         for package_mapping in resource.package_data:
             pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping)
+            if not pd.can_assemble:
+                continue
+
             logger.info(f"  Package data: {pd.purl}")
 
             package_data = pd.to_dict()

setup.cfg

@@ -70,7 +70,7 @@ install_requires =
     # Docker
     container-inspector==32.0.1
     # ScanCode-toolkit
-    scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@c8f854ae1aabf78b19d3fe52e5caca08cc634037
+    scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@86e4e9059e05f53ca099ab68346486f781838270
     extractcode[full]==31.0.0
     commoncode==31.0.3
     # FetchCode