Restructure pipelines for verbosity

scanpipe/apps.py

@@ -178,6 +178,7 @@ def get_new_pipeline_name(pipeline_name):
             "inspect_manifest": "inspect_packages",
             "deploy_to_develop": "map_deploy_to_develop",
             "scan_package": "scan_single_package",
+            "scan_codebase_packages": "inspect_packages",
         }
         if new_name := pipeline_old_names_mapping.get(pipeline_name):
             warnings.warn(

scanpipe/migrations/0053_restructure_pipelines_data.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.0.1 on 2024-02-09 15:05
+
+from django.db import migrations
+
+
+pipeline_old_names_mapping = {
+    "scan_codebase_packages": "inspect_packages",
+}
+
+
+def rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=old_name).update(pipeline_name=new_name)
+
+
+def reverse_rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=new_name).update(pipeline_name=old_name)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0052_run_selected_groups"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            rename_pipelines_data,
+            reverse_code=reverse_rename_pipelines_data,
+        ),
+    ]

scanpipe/pipelines/inspect_packages.py

@@ -21,20 +21,14 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from scanpipe.pipelines.scan_codebase import ScanCodebase
-from scanpipe.pipes import resolve
-from scanpipe.pipes import update_or_create_package
+from scanpipe.pipes import scancode
 
 
 class InspectPackages(ScanCodebase):
     """
-    Inspect a codebase manifest files and resolve their associated packages.
-
-    Supports resolved packages for:
-    - Python: using nexB/python-inspector, supports requirements.txt and
-    setup.py manifests as input
+    Inspect a codebase for manifest files and gets all associated packages.
 
     Supports:
-    - BOM: SPDX document, CycloneDX BOM, AboutCode ABOUT file
     - Python: requirements.txt, setup.py, setup.cfg, Pipfile.lock
     - JavaScript: yarn.lock lockfile, npm package-lock.json lockfile
     - Java: Java JAR MANIFEST.MF, Gradle build script
@@ -45,8 +39,8 @@ class InspectPackages(ScanCodebase):
     - Dart: pubspec manifest, pubspec lockfile
     - OS: FreeBSD compact package manifest, Debian installed packages database
 
-    Full list available at https://scancode-toolkit.readthedocs.io/en/
-    doc-update-licenses/reference/available_package_parsers.html
+    Full list available at https://scancode-toolkit.readthedocs.io/en/stable/
+    reference/available_package_parsers.html
     """
 
     @classmethod
@@ -55,46 +49,18 @@ def steps(cls):
             cls.copy_inputs_to_codebase_directory,
             cls.extract_archives,
             cls.collect_and_create_codebase_resources,
+            cls.flag_empty_files,
             cls.flag_ignored_resources,
-            cls.get_manifest_inputs,
-            cls.get_packages_from_manifest,
-            cls.create_resolved_packages,
+            cls.scan_for_application_packages,
         )
 
-    def get_manifest_inputs(self):
-        """Locate all the manifest files from the project's input/ directory."""
-        self.manifest_resources = resolve.get_manifest_resources(self.project)
-
-    def get_packages_from_manifest(self):
-        """Get packages data from manifest files."""
-        self.resolved_packages = []
-
-        if not self.manifest_resources.exists():
-            self.project.add_warning(
-                description="No manifests found for resolving packages",
-                model="get_packages_from_manifest",
-            )
-            return
-
-        for resource in self.manifest_resources:
-            if packages := resolve.resolve_packages(resource.location):
-                self.resolved_packages.extend(packages)
-            else:
-                self.project.add_error(
-                    description="No packages could be resolved for",
-                    model="get_packages_from_manifest",
-                    details={"path": resource.path},
-                )
-
-    def create_resolved_packages(self):
-        """Create the resolved packages and their dependencies in the database."""
-        for package_data in self.resolved_packages:
-            package_data = resolve.set_license_expression(package_data)
-            dependencies = package_data.pop("dependencies", [])
-            update_or_create_package(self.project, package_data)
-
-            for dependency_data in dependencies:
-                resolved_package = dependency_data.get("resolved_package")
-                if resolved_package:
-                    resolved_package.pop("dependencies", [])
-                    update_or_create_package(self.project, resolved_package)
+    def scan_for_application_packages(self):
+        """
+        Scan resources for package information to add DiscoveredPackage
+        and DiscoveredDependency objects from detected package data.
+        """
+        # `assemble` is set to False because here in this pipeline we
+        # only detect package_data in resources without creating
+        # Package/Dependency instances, to get all the purls from a codebase.
+        scancode.scan_for_application_packages(self.project, assemble=False)
+        scancode.process_package_data(self.project)

scanpipe/pipelines/load_sbom.py

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from scanpipe.pipelines.scan_codebase import ScanCodebase
+from scanpipe.pipes import resolve
+from scanpipe.pipes import update_or_create_package
+
+
+class LoadSBOM(ScanCodebase):
+    """
+    Inspect a codebase for SBOMs and gets all associated packages.
+
+    Supported BOMs:
+    - SPDX document
+    - CycloneDX BOM
+    - AboutCode ABOUT file
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.copy_inputs_to_codebase_directory,
+            cls.extract_archives,
+            cls.collect_and_create_codebase_resources,
+            cls.flag_empty_files,
+            cls.flag_ignored_resources,
+            cls.get_sbom_inputs,
+            cls.get_packages_from_sboms,
+        )
+
+    def get_sbom_inputs(self):
+        """Locate all the SBOMs from the project's input/ directory."""
+        self.manifest_resources = resolve.get_manifest_resources(self.project)
+
+    def get_packages_from_sboms(self):
+        """Get packages data from manifest files."""
+        self.resolved_packages = []
+
+        if not self.manifest_resources.exists():
+            self.project.add_warning(
+                description="No SBOMs found for resolving packages",
+                model="get_packages_from_sboms",
+            )
+            return
+
+        for resource in self.manifest_resources:
+            if packages := resolve.resolve_packages(
+                input_location=resource.location,
+                package_registry=resolve.sbom_registry,
+            ):
+                for package_data in packages:
+                    package_data = resolve.set_license_expression(package_data)
+                    update_or_create_package(self.project, package_data)
+            else:
+                self.project.add_error(
+                    description="No packages could be resolved for SBOM",
+                    model="get_packages_from_sboms",
+                    details={"path": resource.path},
+                )

scanpipe/pipelines/populate_purldb.py

@@ -22,7 +22,6 @@
 
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import purldb
-from scanpipe.pipes import scancode
 
 
 class PopulatePurlDB(Pipeline):
@@ -36,7 +35,6 @@ def steps(cls):
         return (
             cls.populate_purldb_with_discovered_packages,
             cls.populate_purldb_with_discovered_dependencies,
-            cls.populate_purldb_with_detected_purls,
         )
 
     def populate_purldb_with_discovered_packages(self):
@@ -50,26 +48,3 @@ def populate_purldb_with_discovered_dependencies(self):
         purldb.populate_purldb_with_discovered_dependencies(
             project=self.project, logger=self.log
         )
-
-    def populate_purldb_with_detected_purls(self):
-        """Add DiscoveredPackage to PurlDB."""
-        no_packages_and_no_dependencies = all(
-            [
-                not self.project.discoveredpackages.exists(),
-                not self.project.discovereddependencies.exists(),
-            ]
-        )
-        # Even when there are no packages/dependencies, resource level
-        # package data could be detected (i.e. when we detect packages,
-        # but skip the assembly step that creates
-        # package/dependency instances)
-        if no_packages_and_no_dependencies:
-            packages = scancode.get_packages_with_purl_from_resources(self.project)
-            purls = [{"purl": package.purl} for package in packages]
-
-            self.log(f"Populating PurlDB with {len(purls):,d} " "detected PURLs"),
-            purldb.feed_purldb(
-                packages=purls,
-                chunk_size=100,
-                logger=self.log,
-            )

scanpipe/pipelines/resolve_dependencies.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from scanpipe.pipelines.scan_codebase import ScanCodebase
+from scanpipe.pipes import resolve
+from scanpipe.pipes import update_or_create_package
+
+
+class ResolveDependencies(ScanCodebase):
+    """
+    Inspect a codebase manifest files and resolve their associated packages.
+
+    Supports resolved packages for:
+    - Python: using nexB/python-inspector, supports requirements.txt and
+    setup.py manifests as input
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.copy_inputs_to_codebase_directory,
+            cls.extract_archives,
+            cls.collect_and_create_codebase_resources,
+            cls.flag_ignored_resources,
+            cls.get_manifest_inputs,
+            cls.get_packages_from_manifest,
+            cls.create_resolved_packages,
+        )
+
+    def get_manifest_inputs(self):
+        """Locate all the manifest files from the project's input/ directory."""
+        self.manifest_resources = resolve.get_manifest_resources(self.project)
+
+    def get_packages_from_manifest(self):
+        """Get packages data from manifest files."""
+        self.resolved_packages = []
+
+        if not self.manifest_resources.exists():
+            self.project.add_warning(
+                description="No manifests found for resolving packages",
+                model="get_packages_from_manifest",
+            )
+            return
+
+        for resource in self.manifest_resources:
+            if packages := resolve.resolve_packages(
+                input_location=resource.location,
+                package_registry=resolve.resolver_registry,
+            ):
+                self.resolved_packages.extend(packages)
+            else:
+                self.project.add_error(
+                    description="No packages could be resolved for",
+                    model="get_packages_from_manifest",
+                    details={"path": resource.path},
+                )
+
+    def create_resolved_packages(self):
+        """Create the resolved packages and their dependencies in the database."""
+        for package_data in self.resolved_packages:
+            package_data = resolve.set_license_expression(package_data)
+            dependencies = package_data.pop("dependencies", [])
+            update_or_create_package(self.project, package_data)
+
+            for dependency_data in dependencies:
+                resolved_package = dependency_data.get("resolved_package")
+                if resolved_package:
+                    resolved_package.pop("dependencies", [])
+                    update_or_create_package(self.project, resolved_package)