From 079d9e16c46333e6d302b3a25e8b11c6386c309b Mon Sep 17 00:00:00 2001
From: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Date: Fri, 9 Feb 2024 23:34:43 +0530
Subject: [PATCH 1/5] Restructure pipelines for verbosity

Remove scan_codebase_packages pipeline, and restructure inspect_packages
pipeline into load_sbom and resolve_packages pipelines.

Reference: https://github.com/nexB/scancode.io/discussions/1035
Reference: https://github.com/nexB/scancode.io/issues/1034
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
---
 scanpipe/apps.py                              |  1 +
 .../0053_restructure_pipelines_data.py        | 33 +++++++
 scanpipe/pipelines/inspect_packages.py        | 66 ++++----------
 scanpipe/pipelines/load_sbom.py               | 78 ++++++++++++++++
 scanpipe/pipelines/populate_purldb.py         | 25 ------
 scanpipe/pipelines/resolve_dependencies.py    | 88 +++++++++++++++++++
 scanpipe/pipelines/scan_codebase_packages.py  | 51 -----------
 scanpipe/pipes/resolve.py                     | 24 ++---
 scanpipe/pipes/scancode.py                    | 31 ++++++-
 scanpipe/tests/pipes/test_resolve.py          | 13 +--
 scanpipe/tests/test_pipelines.py              | 30 ++++---
 setup.cfg                                     |  3 +-
 12 files changed, 282 insertions(+), 161 deletions(-)
 create mode 100644 scanpipe/migrations/0053_restructure_pipelines_data.py
 create mode 100644 scanpipe/pipelines/load_sbom.py
 create mode 100644 scanpipe/pipelines/resolve_dependencies.py
 delete mode 100644 scanpipe/pipelines/scan_codebase_packages.py

diff --git a/scanpipe/apps.py b/scanpipe/apps.py
index 8f46096a96..cddfac36a9 100644
--- a/scanpipe/apps.py
+++ b/scanpipe/apps.py
@@ -178,6 +178,7 @@ def get_new_pipeline_name(pipeline_name):
             "inspect_manifest": "inspect_packages",
             "deploy_to_develop": "map_deploy_to_develop",
             "scan_package": "scan_single_package",
+            "scan_codebase_packages": "inspect_packages",
         }
         if new_name := pipeline_old_names_mapping.get(pipeline_name):
             warnings.warn(
diff --git a/scanpipe/migrations/0053_restructure_pipelines_data.py b/scanpipe/migrations/0053_restructure_pipelines_data.py
new file mode 100644
index 0000000000..e5b9e14dcc
--- /dev/null
+++ b/scanpipe/migrations/0053_restructure_pipelines_data.py
@@ -0,0 +1,33 @@
+# Generated by Django 5.0.1 on 2024-02-09 15:05
+
+from django.db import migrations
+
+
+pipeline_old_names_mapping = {
+    "scan_codebase_packages": "inspect_packages",
+}
+
+
+def rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=old_name).update(pipeline_name=new_name)
+
+
+def reverse_rename_pipelines_data(apps, schema_editor):
+    Run = apps.get_model("scanpipe", "Run")
+    for old_name, new_name in pipeline_old_names_mapping.items():
+        Run.objects.filter(pipeline_name=new_name).update(pipeline_name=old_name)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("scanpipe", "0052_run_selected_groups"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            rename_pipelines_data,
+            reverse_code=reverse_rename_pipelines_data,
+        ),
+    ]
diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py
index f5a8568924..5e8abca275 100644
--- a/scanpipe/pipelines/inspect_packages.py
+++ b/scanpipe/pipelines/inspect_packages.py
@@ -21,20 +21,14 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from scanpipe.pipelines.scan_codebase import ScanCodebase
-from scanpipe.pipes import resolve
-from scanpipe.pipes import update_or_create_package
+from scanpipe.pipes import scancode
 
 
 class InspectPackages(ScanCodebase):
     """
-    Inspect a codebase manifest files and resolve their associated packages.
-
-    Supports resolved packages for:
-    - Python: using nexB/python-inspector, supports requirements.txt and
-    setup.py manifests as input
+    Inspect a codebase for manifest files and gets all associated packages.
 
     Supports:
-    - BOM: SPDX document, CycloneDX BOM, AboutCode ABOUT file
     - Python: requirements.txt, setup.py, setup.cfg, Pipfile.lock
     - JavaScript: yarn.lock lockfile, npm package-lock.json lockfile
     - Java: Java JAR MANIFEST.MF, Gradle build script
@@ -45,8 +39,8 @@ class InspectPackages(ScanCodebase):
     - Dart: pubspec manifest, pubspec lockfile
     - OS: FreeBSD compact package manifest, Debian installed packages database
 
-    Full list available at https://scancode-toolkit.readthedocs.io/en/
-    doc-update-licenses/reference/available_package_parsers.html
+    Full list available at https://scancode-toolkit.readthedocs.io/en/stable/
+    reference/available_package_parsers.html
     """
 
     @classmethod
@@ -55,46 +49,18 @@ def steps(cls):
             cls.copy_inputs_to_codebase_directory,
             cls.extract_archives,
             cls.collect_and_create_codebase_resources,
+            cls.flag_empty_files,
             cls.flag_ignored_resources,
-            cls.get_manifest_inputs,
-            cls.get_packages_from_manifest,
-            cls.create_resolved_packages,
+            cls.scan_for_application_packages,
         )
 
-    def get_manifest_inputs(self):
-        """Locate all the manifest files from the project's input/ directory."""
-        self.manifest_resources = resolve.get_manifest_resources(self.project)
-
-    def get_packages_from_manifest(self):
-        """Get packages data from manifest files."""
-        self.resolved_packages = []
-
-        if not self.manifest_resources.exists():
-            self.project.add_warning(
-                description="No manifests found for resolving packages",
-                model="get_packages_from_manifest",
-            )
-            return
-
-        for resource in self.manifest_resources:
-            if packages := resolve.resolve_packages(resource.location):
-                self.resolved_packages.extend(packages)
-            else:
-                self.project.add_error(
-                    description="No packages could be resolved for",
-                    model="get_packages_from_manifest",
-                    details={"path": resource.path},
-                )
-
-    def create_resolved_packages(self):
-        """Create the resolved packages and their dependencies in the database."""
-        for package_data in self.resolved_packages:
-            package_data = resolve.set_license_expression(package_data)
-            dependencies = package_data.pop("dependencies", [])
-            update_or_create_package(self.project, package_data)
-
-            for dependency_data in dependencies:
-                resolved_package = dependency_data.get("resolved_package")
-                if resolved_package:
-                    resolved_package.pop("dependencies", [])
-                    update_or_create_package(self.project, resolved_package)
+    def scan_for_application_packages(self):
+        """
+        Scan resources for package information to add DiscoveredPackage
+        and DiscoveredDependency objects from detected package data.
+        """
+        # `assemble` is set to False because here in this pipeline we
+        # only detect package_data in resources without creating
+        # Package/Dependency instances, to get all the purls from a codebase.
+        scancode.scan_for_application_packages(self.project, assemble=False)
+        scancode.process_package_data(self.project)
diff --git a/scanpipe/pipelines/load_sbom.py b/scanpipe/pipelines/load_sbom.py
new file mode 100644
index 0000000000..f2ca4845d4
--- /dev/null
+++ b/scanpipe/pipelines/load_sbom.py
@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from scanpipe.pipelines.scan_codebase import ScanCodebase
+from scanpipe.pipes import resolve
+from scanpipe.pipes import update_or_create_package
+
+
+class LoadSBOM(ScanCodebase):
+    """
+    Inspect a codebase for SBOMs and gets all associated packages.
+
+    Supported BOMs:
+    - SPDX document
+    - CycloneDX BOM
+    - AboutCode ABOUT file
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.copy_inputs_to_codebase_directory,
+            cls.extract_archives,
+            cls.collect_and_create_codebase_resources,
+            cls.flag_empty_files,
+            cls.flag_ignored_resources,
+            cls.get_sbom_inputs,
+            cls.get_packages_from_sboms,
+        )
+
+    def get_sbom_inputs(self):
+        """Locate all the SBOMs from the project's input/ directory."""
+        self.manifest_resources = resolve.get_manifest_resources(self.project)
+
+    def get_packages_from_sboms(self):
+        """Get packages data from manifest files."""
+        self.resolved_packages = []
+
+        if not self.manifest_resources.exists():
+            self.project.add_warning(
+                description="No SBOMs found for resolving packages",
+                model="get_packages_from_sboms",
+            )
+            return
+
+        for resource in self.manifest_resources:
+            if packages := resolve.resolve_packages(
+                input_location=resource.location,
+                package_registry=resolve.sbom_registry,
+            ):
+                for package_data in packages:
+                    package_data = resolve.set_license_expression(package_data)
+                    update_or_create_package(self.project, package_data)
+            else:
+                self.project.add_error(
+                    description="No packages could be resolved for SBOM",
+                    model="get_packages_from_sboms",
+                    details={"path": resource.path},
+                )
diff --git a/scanpipe/pipelines/populate_purldb.py b/scanpipe/pipelines/populate_purldb.py
index b8edcd3fd7..41c098c248 100644
--- a/scanpipe/pipelines/populate_purldb.py
+++ b/scanpipe/pipelines/populate_purldb.py
@@ -22,7 +22,6 @@
 
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import purldb
-from scanpipe.pipes import scancode
 
 
 class PopulatePurlDB(Pipeline):
@@ -36,7 +35,6 @@ def steps(cls):
         return (
             cls.populate_purldb_with_discovered_packages,
             cls.populate_purldb_with_discovered_dependencies,
-            cls.populate_purldb_with_detected_purls,
         )
 
     def populate_purldb_with_discovered_packages(self):
@@ -50,26 +48,3 @@ def populate_purldb_with_discovered_dependencies(self):
         purldb.populate_purldb_with_discovered_dependencies(
             project=self.project, logger=self.log
         )
-
-    def populate_purldb_with_detected_purls(self):
-        """Add DiscoveredPackage to PurlDB."""
-        no_packages_and_no_dependencies = all(
-            [
-                not self.project.discoveredpackages.exists(),
-                not self.project.discovereddependencies.exists(),
-            ]
-        )
-        # Even when there are no packages/dependencies, resource level
-        # package data could be detected (i.e. when we detect packages,
-        # but skip the assembly step that creates
-        # package/dependency instances)
-        if no_packages_and_no_dependencies:
-            packages = scancode.get_packages_with_purl_from_resources(self.project)
-            purls = [{"purl": package.purl} for package in packages]
-
-            self.log(f"Populating PurlDB with {len(purls):,d} " "detected PURLs"),
-            purldb.feed_purldb(
-                packages=purls,
-                chunk_size=100,
-                logger=self.log,
-            )
diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py
new file mode 100644
index 0000000000..b7e74972b2
--- /dev/null
+++ b/scanpipe/pipelines/resolve_dependencies.py
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from scanpipe.pipelines.scan_codebase import ScanCodebase
+from scanpipe.pipes import resolve
+from scanpipe.pipes import update_or_create_package
+
+
+class ResolveDependencies(ScanCodebase):
+    """
+    Inspect a codebase manifest files and resolve their associated packages.
+
+    Supports resolved packages for:
+    - Python: using nexB/python-inspector, supports requirements.txt and
+    setup.py manifests as input
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.copy_inputs_to_codebase_directory,
+            cls.extract_archives,
+            cls.collect_and_create_codebase_resources,
+            cls.flag_ignored_resources,
+            cls.get_manifest_inputs,
+            cls.get_packages_from_manifest,
+            cls.create_resolved_packages,
+        )
+
+    def get_manifest_inputs(self):
+        """Locate all the manifest files from the project's input/ directory."""
+        self.manifest_resources = resolve.get_manifest_resources(self.project)
+
+    def get_packages_from_manifest(self):
+        """Get packages data from manifest files."""
+        self.resolved_packages = []
+
+        if not self.manifest_resources.exists():
+            self.project.add_warning(
+                description="No manifests found for resolving packages",
+                model="get_packages_from_manifest",
+            )
+            return
+
+        for resource in self.manifest_resources:
+            if packages := resolve.resolve_packages(
+                input_location=resource.location,
+                package_registry=resolve.resolver_registry,
+            ):
+                self.resolved_packages.extend(packages)
+            else:
+                self.project.add_error(
+                    description="No packages could be resolved for",
+                    model="get_packages_from_manifest",
+                    details={"path": resource.path},
+                )
+
+    def create_resolved_packages(self):
+        """Create the resolved packages and their dependencies in the database."""
+        for package_data in self.resolved_packages:
+            package_data = resolve.set_license_expression(package_data)
+            dependencies = package_data.pop("dependencies", [])
+            update_or_create_package(self.project, package_data)
+
+            for dependency_data in dependencies:
+                resolved_package = dependency_data.get("resolved_package")
+                if resolved_package:
+                    resolved_package.pop("dependencies", [])
+                    update_or_create_package(self.project, resolved_package)
diff --git a/scanpipe/pipelines/scan_codebase_packages.py b/scanpipe/pipelines/scan_codebase_packages.py
deleted file mode 100644
index 1442ec20dd..0000000000
--- a/scanpipe/pipelines/scan_codebase_packages.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# http://nexb.com and https://github.com/nexB/scancode.io
-# The ScanCode.io software is licensed under the Apache License version 2.0.
-# Data generated with ScanCode.io is provided as-is without warranties.
-# ScanCode is a trademark of nexB Inc.
-#
-# You may not use this software except in compliance with the License.
-# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-# CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
-# ScanCode.io should be considered or used as legal advice. Consult an Attorney
-# for any legal advice.
-#
-# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
-# Visit https://github.com/nexB/scancode.io for support and download.
-
-from scanpipe.pipelines.scan_codebase import ScanCodebase
-from scanpipe.pipes import scancode
-
-
-class ScanCodebasePackages(ScanCodebase):
-    """
-    Scan a codebase for PURLs without assembling full packages/dependencies.
-
-    This Pipeline is intended for gathering PURL information from a
-    codebase without the overhead of full package assembly.
-    """
-
-    @classmethod
-    def steps(cls):
-        return (
-            cls.copy_inputs_to_codebase_directory,
-            cls.extract_archives,
-            cls.collect_and_create_codebase_resources,
-            cls.flag_empty_files,
-            cls.flag_ignored_resources,
-            cls.scan_for_application_packages,
-        )
-
-    def scan_for_application_packages(self):
-        """Scan unknown resources for packages information."""
-        # `assemble` is set to False because here in this pipeline we
-        # only detect package_data in resources without creating
-        # Package/Dependency instances, to get all the purls from a codebase.
-        scancode.scan_for_application_packages(self.project, assemble=False)
diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py
index 8072b3977b..73776fc3ea 100644
--- a/scanpipe/pipes/resolve.py
+++ b/scanpipe/pipes/resolve.py
@@ -31,7 +31,6 @@
 from packagedcode.licensing import get_license_detections_and_expression
 from packageurl import PackageURL
 from python_inspector.api import resolve_dependencies
-from scancode.api import get_package_data
 
 from scanpipe.models import DiscoveredPackage
 from scanpipe.pipes import cyclonedx
@@ -43,7 +42,7 @@
 """
 
 
-def resolve_packages(input_location):
+def resolve_packages(input_location, package_registry=None):
     """Resolve the packages from manifest file."""
     default_package_type = get_default_package_type(input_location)
     # we only try to resolve packages if file at input_location is
@@ -51,15 +50,11 @@ def resolve_packages(input_location):
     if not default_package_type:
         return
 
-    # The ScanCode.io resolvers take precedence over the ScanCode-toolkit ones.
-    resolver = resolver_registry.get(default_package_type)
+    # Get resolvers for available packages/SBOMs in the registry
+    resolver = package_registry.get(default_package_type)
     if resolver:
         resolved_packages = resolver(input_location=input_location)
-    else:
-        package_data = get_package_data(location=input_location)
-        resolved_packages = package_data.get("package_data", [])
-
-    return resolved_packages
+        return resolved_packages
 
 
 def get_manifest_resources(project):
@@ -280,10 +275,17 @@ def get_default_package_type(input_location):
                 return "spdx"
 
 
-# Mapping between the `default_package_type` its related resolver function
+# Mapping between `default_package_type` its related resolver functions
+# for package dependency resolvers
 resolver_registry = {
-    "about": resolve_about_packages,
     "pypi": resolve_pypi_packages,
+}
+
+
+# Mapping between `default_package_type` its related resolver functions
+# for SBOMs and About files
+sbom_registry = {
+    "about": resolve_about_packages,
     "spdx": resolve_spdx_packages,
     "cyclonedx": resolve_cyclonedx_packages,
 }
diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py
index 6854d56818..35bf91c173 100644
--- a/scanpipe/pipes/scancode.py
+++ b/scanpipe/pipes/scancode.py
@@ -404,7 +404,8 @@ def add_resource_to_package(package_uid, resource, project):
 def assemble_packages(project):
     """
     Create instances of DiscoveredPackage and DiscoveredDependency for `project`
-    from the parsed package data present in the CodebaseResources of `project`.
+    from the parsed package data present in the CodebaseResources of `project`,
+    using the respective package handlers for each package manifest type.
     """
     logger.info(f"Project {project} assemble_packages:")
     seen_resource_paths = set()
@@ -442,6 +443,34 @@ def assemble_packages(project):
                     logger.info(f"Unknown Package assembly item type: {item!r}")
 
 
+def process_package_data(project):
+    """
+    Create instances of DiscoveredPackage and DiscoveredDependency for `project`
+    from the parsed package data present in the CodebaseResources of `project`.
+
+    Here package assembly though package handlers are not performed, instead
+    package/dependency objects are created directly from package data.
+    """
+    logger.info(f"Project {project} process_package_data:")
+    seen_resource_paths = set()
+
+    for resource in project.codebaseresources.has_package_data():
+        if resource.path in seen_resource_paths:
+            continue
+
+        logger.info(f"  Processing: {resource.path}")
+        for package_mapping in resource.package_data:
+            pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping)
+            logger.info(f"  Package data: {pd.purl}")
+
+            package_data = pd.to_dict()
+            dependencies = package_data.pop("dependencies")
+            pipes.update_or_create_package(project, package_data)
+
+            for dep in dependencies:
+                pipes.update_or_create_dependency(project, dep)
+
+
 def get_packages_with_purl_from_resources(project):
     """
     Yield Dependency or PackageData objects created from detected package_data
diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py
index f8802cd4d1..52787ece16 100644
--- a/scanpipe/tests/pipes/test_resolve.py
+++ b/scanpipe/tests/pipes/test_resolve.py
@@ -20,7 +20,6 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-import json
 from pathlib import Path
 
 from django.test import TestCase
@@ -87,7 +86,10 @@ def test_scanpipe_pipes_resolve_convert_spdx_expression(self):
     def test_scanpipe_pipes_resolve_resolve_packages(self):
         # ScanCode.io resolvers
         input_location = self.manifest_location / "Django-4.0.8-py3-none-any.whl.ABOUT"
-        packages = resolve.resolve_packages(str(input_location))
+        packages = resolve.resolve_packages(
+            input_location=str(input_location),
+            package_registry=resolve.sbom_registry,
+        )
         expected = {
             "filename": "Django-4.0.8-py3-none-any.whl",
             "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl",
@@ -102,13 +104,6 @@ def test_scanpipe_pipes_resolve_resolve_packages(self):
         }
         self.assertEqual([expected], packages)
 
-        # ScanCode-toolkit resolvers
-        input_location = self.manifest_location / "package.json"
-        packages = resolve.resolve_packages(str(input_location))
-        expected_location = self.manifest_location / "package.expected.json"
-        expected = json.loads(expected_location.read_text())
-        self.assertEqual(expected, packages)
-
     def test_scanpipe_pipes_resolve_resolve_about_packages(self):
         input_location = self.manifest_location / "Django-4.0.8-py3-none-any.whl.ABOUT"
         package = resolve.resolve_about_packages(str(input_location))
diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py
index 1045b0ad96..bdae2ff41f 100644
--- a/scanpipe/tests/test_pipelines.py
+++ b/scanpipe/tests/test_pipelines.py
@@ -586,8 +586,8 @@ def test_scanpipe_scan_codebase_pipeline_integration(self):
         expected_file = self.data_location / "is-npm-1.0.0_scan_codebase.json"
         self.assertPipelineResultEqual(expected_file, result_file)
 
-    def test_scanpipe_scan_codebase_packages_does_not_create_packages(self):
-        pipeline_name = "scan_codebase_packages"
+    def test_scanpipe_inspect_packages_creates_packages(self):
+        pipeline_name = "inspect_packages"
         project1 = Project.objects.create(name="Analysis")
 
         filename = "is-npm-1.0.0.tgz"
@@ -601,8 +601,8 @@ def test_scanpipe_scan_codebase_packages_does_not_create_packages(self):
         self.assertEqual(0, exitcode, msg=out)
 
         self.assertEqual(6, project1.codebaseresources.count())
-        self.assertEqual(0, project1.discoveredpackages.count())
-        self.assertEqual(0, project1.discovereddependencies.count())
+        self.assertEqual(1, project1.discoveredpackages.count())
+        self.assertEqual(1, project1.discovereddependencies.count())
 
     def test_scanpipe_scan_codebase_can_process_wheel(self):
         pipeline_name = "scan_codebase"
@@ -862,7 +862,7 @@ def test_scanpipe_find_vulnerabilities_pipeline_integration(
         self.assertEqual(expected, package1.affected_by_vulnerabilities)
 
     def test_scanpipe_inspect_manifest_pipeline_integration(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
         run = project1.add_pipeline(pipeline_name)
@@ -877,7 +877,7 @@ def test_scanpipe_inspect_manifest_pipeline_integration(self):
         self.assertIn(expected, message.description)
 
     def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
         run = project1.add_pipeline(pipeline_name)
@@ -892,7 +892,7 @@ def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self):
         self.assertIn(expected, message.description)
 
     def test_scanpipe_inspect_manifest_pipeline_integration_misc(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
         input_location = (
@@ -911,7 +911,7 @@ def test_scanpipe_inspect_manifest_pipeline_integration_misc(self):
     def test_scanpipe_inspect_manifest_pipeline_pypi_integration(
         self, resolve_dependencies
     ):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
         run = project1.add_pipeline(pipeline_name)
@@ -930,7 +930,7 @@ def test_scanpipe_inspect_manifest_pipeline_pypi_integration(
                 self.assertEqual(value, getattr(discoveredpackage, field_name))
 
     def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 
         input_location = (
@@ -952,7 +952,7 @@ def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self):
         self.assertEqual("bsd-new", discoveredpackage.declared_license_expression)
 
     def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 
         input_location = self.data_location / "manifests" / "toml.spdx.json"
@@ -974,7 +974,7 @@ def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self):
         self.assertEqual("mit", discoveredpackage.declared_license_expression)
 
     def test_scanpipe_inspect_manifest_pipeline_cyclonedx_integration(self):
-        pipeline_name = "inspect_packages"
+        pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 
         input_location = self.data_location / "cyclonedx/nested.cdx.json"
@@ -1176,6 +1176,7 @@ def mock_request_post_return(url, data, headers, timeout):
         pipes.collect_and_create_codebase_resources(project1)
 
         scancode.scan_for_application_packages(project1, assemble=False)
+        scancode.process_package_data(project1)
 
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()
@@ -1183,7 +1184,10 @@ def mock_request_post_return(url, data, headers, timeout):
         exitcode, out = pipeline.execute()
         self.assertEqual(0, exitcode, msg=out)
 
-        self.assertIn("Populating PurlDB with 7 detected PURLs", run.log)
-        self.assertIn("Successfully queued 7 PURLs for indexing in PurlDB", run.log)
+        self.assertIn("Populating PurlDB with 1 PURLs from DiscoveredPackage", run.log)
+        self.assertIn(
+            "Populating PurlDB with 6 unresolved PURLs from DiscoveredDependency",
+            run.log,
+        )
         self.assertIn("1 PURLs were already present in PurlDB index queue", run.log)
         self.assertIn("Couldn't index 1 unsupported PURLs", run.log)
diff --git a/setup.cfg b/setup.cfg
index 7c442883ca..3477ff1129 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -128,11 +128,12 @@ scancodeio_pipelines =
     find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities
     inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages
     load_inventory = scanpipe.pipelines.load_inventory:LoadInventory
+    load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM
     map_deploy_to_develop = scanpipe.pipelines.deploy_to_develop:DeployToDevelop
     match_to_purldb = scanpipe.pipelines.match_to_purldb:MatchToPurlDB
     populate_purldb = scanpipe.pipelines.populate_purldb:PopulatePurlDB
+    resolve_dependencies = scanpipe.pipelines.resolve_dependencies:ResolveDependencies
     scan_codebase = scanpipe.pipelines.scan_codebase:ScanCodebase
-    scan_codebase_packages = scanpipe.pipelines.scan_codebase_packages:ScanCodebasePackages
     scan_single_package = scanpipe.pipelines.scan_single_package:ScanSinglePackage
 
 [isort]

From 930b66cf75adf3733f8442db074134f1d19b844c Mon Sep 17 00:00:00 2001
From: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Date: Mon, 12 Feb 2024 18:02:11 +0530
Subject: [PATCH 2/5] Refactor functions and improve docstrings

Reference: https://github.com/nexB/scancode.io/pull/1074
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
---
 scanpipe/pipelines/inspect_packages.py     | 28 +++++-----
 scanpipe/pipelines/load_sbom.py            | 42 ++++++---------
 scanpipe/pipelines/resolve_dependencies.py | 59 +++++++++------------
 scanpipe/pipelines/scan_codebase.py        | 12 ++---
 scanpipe/pipelines/scan_single_package.py  |  5 +-
 scanpipe/pipes/resolve.py                  | 61 +++++++++++++++++++++-
 scanpipe/tests/pipes/test_resolve.py       |  4 +-
 scanpipe/tests/test_pipelines.py           | 36 +++++++++----
 8 files changed, 149 insertions(+), 98 deletions(-)

diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py
index 5e8abca275..af59d737d4 100644
--- a/scanpipe/pipelines/inspect_packages.py
+++ b/scanpipe/pipelines/inspect_packages.py
@@ -26,21 +26,18 @@
 
 class InspectPackages(ScanCodebase):
     """
-    Inspect a codebase for manifest files and gets all associated packages.
+    Inspect a codebase for package and pre-resolved dependencies.
 
-    Supports:
-    - Python: requirements.txt, setup.py, setup.cfg, Pipfile.lock
-    - JavaScript: yarn.lock lockfile, npm package-lock.json lockfile
-    - Java: Java JAR MANIFEST.MF, Gradle build script
-    - Ruby: RubyGems gemspec manifest, RubyGems Bundler Gemfile.lock
-    - Rust: Rust Cargo.lock dependencies lockfile, Rust Cargo.toml package manifest
-    - PHP: PHP composer lockfile, PHP composer manifest
-    - NuGet: nuspec package manifest
-    - Dart: pubspec manifest, pubspec lockfile
-    - OS: FreeBSD compact package manifest, Debian installed packages database
+    This pipeline inspects a codebase for application packages
+    and their dependencies using package manifests and dependency
+    lockfiles. It does not resolve dependencies, it does instead
+    collect already pre-resolved dependencies from lockfiles, and
+    direct dependencies (possibly not resolved) as found in
+    package manifests' dependency sections.
 
-    Full list available at https://scancode-toolkit.readthedocs.io/en/stable/
-    reference/available_package_parsers.html
+    See documentation for the list of supported package manifests and
+    dependency lockfiles:
+    https://scancode-toolkit.readthedocs.io/en/stable/reference/available_package_parsers.html
     """
 
     @classmethod
@@ -60,7 +57,8 @@ def scan_for_application_packages(self):
         and DiscoveredDependency objects from detected package data.
         """
         # `assemble` is set to False because here in this pipeline we
-        # only detect package_data in resources without creating
-        # Package/Dependency instances, to get all the purls from a codebase.
+        # only detect package_data in resources and create
+        # Package/Dependency instances directly instead of assembling
+        # the packages and assigning files to them
         scancode.scan_for_application_packages(self.project, assemble=False)
         scancode.process_package_data(self.project)
diff --git a/scanpipe/pipelines/load_sbom.py b/scanpipe/pipelines/load_sbom.py
index f2ca4845d4..492722a438 100644
--- a/scanpipe/pipelines/load_sbom.py
+++ b/scanpipe/pipelines/load_sbom.py
@@ -22,12 +22,11 @@
 
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
-from scanpipe.pipes import update_or_create_package
 
 
 class LoadSBOM(ScanCodebase):
     """
-    Inspect a codebase for SBOMs and gets all associated packages.
+    Detects and loads all associated packages from SBOMs.
 
     Supported BOMs:
     - SPDX document
@@ -45,34 +44,25 @@ def steps(cls):
             cls.flag_ignored_resources,
             cls.get_sbom_inputs,
             cls.get_packages_from_sboms,
+            cls.create_packages_from_sboms,
         )
 
     def get_sbom_inputs(self):
-        """Locate all the SBOMs from the project's input/ directory."""
+        """Locate all the SBOMs among the codebase resources."""
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
     def get_packages_from_sboms(self):
-        """Get packages data from manifest files."""
-        self.resolved_packages = []
-
-        if not self.manifest_resources.exists():
-            self.project.add_warning(
-                description="No SBOMs found for resolving packages",
-                model="get_packages_from_sboms",
-            )
-            return
+        """Get packages data from SBOMs."""
+        self.packages = resolve.get_packages(
+            project=self.project,
+            package_registry=resolve.sbom_registry,
+            manifest_resources=self.manifest_resources,
+            model="get_packages_from_sboms",
+        )
 
-        for resource in self.manifest_resources:
-            if packages := resolve.resolve_packages(
-                input_location=resource.location,
-                package_registry=resolve.sbom_registry,
-            ):
-                for package_data in packages:
-                    package_data = resolve.set_license_expression(package_data)
-                    update_or_create_package(self.project, package_data)
-            else:
-                self.project.add_error(
-                    description="No packages could be resolved for SBOM",
-                    model="get_packages_from_sboms",
-                    details={"path": resource.path},
-                )
+    def create_packages_from_sboms(self):
+        """Create the packages and dependencies from the SBOM, in the database."""
+        resolve.create_packages_and_dependencies(
+            project=self.project,
+            packages=self.packages,
+        )
diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py
index b7e74972b2..6efac05269 100644
--- a/scanpipe/pipelines/resolve_dependencies.py
+++ b/scanpipe/pipelines/resolve_dependencies.py
@@ -22,12 +22,15 @@
 
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipes import resolve
-from scanpipe.pipes import update_or_create_package
 
 
 class ResolveDependencies(ScanCodebase):
     """
-    Inspect a codebase manifest files and resolve their associated packages.
+    Resolve dependencies from package manifests/lockfiles.
+
+    This pipeline inspects codebase lockfiles/manifest files
+    which contain package requirements and resolve their
+    associated packages.
 
     Supports resolved packages for:
     - Python: using nexB/python-inspector, supports requirements.txt and
@@ -47,42 +50,28 @@ def steps(cls):
         )
 
     def get_manifest_inputs(self):
-        """Locate all the manifest files from the project's input/ directory."""
+        """
+        Locate all the package manifest files for which package resolvers are
+        supported, in the codebase resources.
+        """
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
     def get_packages_from_manifest(self):
-        """Get packages data from manifest files."""
-        self.resolved_packages = []
-
-        if not self.manifest_resources.exists():
-            self.project.add_warning(
-                description="No manifests found for resolving packages",
-                model="get_packages_from_manifest",
-            )
-            return
-
-        for resource in self.manifest_resources:
-            if packages := resolve.resolve_packages(
-                input_location=resource.location,
-                package_registry=resolve.resolver_registry,
-            ):
-                self.resolved_packages.extend(packages)
-            else:
-                self.project.add_error(
-                    description="No packages could be resolved for",
-                    model="get_packages_from_manifest",
-                    details={"path": resource.path},
-                )
+        """
+        Resolve package data from lockfiles/requirement files with package
+        requirements/dependenices.
+        """
+        self.resolved_packages = resolve.get_packages(
+            project=self.project,
+            package_registry=resolve.resolver_registry,
+            manifest_resources=self.manifest_resources,
+            model="get_packages_from_manifest",
+        )
 
     def create_resolved_packages(self):
         """Create the resolved packages and their dependencies in the database."""
-        for package_data in self.resolved_packages:
-            package_data = resolve.set_license_expression(package_data)
-            dependencies = package_data.pop("dependencies", [])
-            update_or_create_package(self.project, package_data)
-
-            for dependency_data in dependencies:
-                resolved_package = dependency_data.get("resolved_package")
-                if resolved_package:
-                    resolved_package.pop("dependencies", [])
-                    update_or_create_package(self.project, resolved_package)
+        resolve.create_packages_and_dependencies(
+            project=self.project,
+            packages=self.resolved_packages,
+            resolved=True,
+        )
diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py
index b754606baa..e86867019c 100644
--- a/scanpipe/pipelines/scan_codebase.py
+++ b/scanpipe/pipelines/scan_codebase.py
@@ -28,15 +28,11 @@
 
 class ScanCodebase(Pipeline):
     """
-    Scan a codebase with ScanCode-toolkit.
+    Scan a codebase for application packages, licenses, and copyrights.
 
-    If the codebase consists of several packages and dependencies, it will try to
-    resolve and scan those too.
-
-    Input files are copied to the project's codebase/ directory and are extracted
-    in place before running the scan.
-    Alternatively, the code can be manually copied to the project codebase/
-    directory.
+    This pipeline does not further scan the files contained in a package
+    for license and copyrights and only considers the declared license
+    of a package.
     """
 
     @classmethod
diff --git a/scanpipe/pipelines/scan_single_package.py b/scanpipe/pipelines/scan_single_package.py
index b28550101e..ae88295127 100644
--- a/scanpipe/pipelines/scan_single_package.py
+++ b/scanpipe/pipelines/scan_single_package.py
@@ -36,7 +36,10 @@
 
 class ScanSinglePackage(Pipeline):
     """
-    Scan a single package file or package archive with ScanCode-toolkit.
+    Scan a single package file or package archive.
+
+    This pipeline scans a single package for package
+    metadata, declared dependencies, licenses, and copyrights.
 
     The output is a summary of the scan results in JSON format.
     """
diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py
index 73776fc3ea..6c192b94ba 100644
--- a/scanpipe/pipes/resolve.py
+++ b/scanpipe/pipes/resolve.py
@@ -36,14 +36,71 @@
 from scanpipe.pipes import cyclonedx
 from scanpipe.pipes import flag
 from scanpipe.pipes import spdx
+from scanpipe.pipes import update_or_create_dependency
+from scanpipe.pipes import update_or_create_package
 
 """
 Resolve packages from manifest, lockfile, and SBOM.
 """
 
 
-def resolve_packages(input_location, package_registry=None):
-    """Resolve the packages from manifest file."""
+def get_packages(project, package_registry, manifest_resources, model):
+    """
+    Get package data from package manifests/lockfiles/SBOMs or
+    get package data for resolved packages from package requirements.
+    """
+    resolved_packages = []
+
+    if not manifest_resources.exists():
+        project.add_warning(
+            description="No resources found with package data",
+            model=model,
+        )
+        return
+
+    for resource in manifest_resources:
+        if packages := get_packages_from_manifest(
+            input_location=resource.location,
+            package_registry=package_registry,
+        ):
+            resolved_packages.extend(packages)
+        else:
+            project.add_error(
+                description="No packages could be resolved for",
+                model=model,
+                details={"path": resource.path},
+            )
+
+    return resolved_packages
+
+
+def create_packages_and_dependencies(project, packages, resolved=False):
+    """
+    Create DiscoveredPackage and DiscoveredDependency objects for
+    packages detected in a package manifest, lockfile or SBOM.
+
+    If resolved, create packages out of resolved dependencies,
+    otherwise create dependencies.
+    """
+    for package_data in packages:
+        package_data = set_license_expression(package_data)
+        dependencies = package_data.pop("dependencies", [])
+        update_or_create_package(project, package_data)
+
+        for dependency_data in dependencies:
+            if resolved:
+                if resolved_package := dependency_data.get("resolved_package"):
+                    resolved_package.pop("dependencies", [])
+                    update_or_create_package(project, resolved_package)
+            else:
+                update_or_create_dependency(project, dependency_data)
+
+
+def get_packages_from_manifest(input_location, package_registry=None):
+    """
+    Resolve packages or get packages data from a package manifest file/
+    lockfile/SBOM at `input_location`.
+    """
     default_package_type = get_default_package_type(input_location)
     # we only try to resolve packages if file at input_location is
     # a package manifest, and ignore for other files
diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py
index 52787ece16..2e908d971f 100644
--- a/scanpipe/tests/pipes/test_resolve.py
+++ b/scanpipe/tests/pipes/test_resolve.py
@@ -83,10 +83,10 @@ def test_scanpipe_pipes_resolve_convert_spdx_expression(self):
         scancode_expression = "mit OR gpl-2.0 WITH generic-exception"
         self.assertEqual(scancode_expression, resolve.convert_spdx_expression(spdx))
 
-    def test_scanpipe_pipes_resolve_resolve_packages(self):
+    def test_scanpipe_pipes_resolve_get_packages_from_manifest(self):
         # ScanCode.io resolvers
         input_location = self.manifest_location / "Django-4.0.8-py3-none-any.whl.ABOUT"
-        packages = resolve.resolve_packages(
+        packages = resolve.get_packages_from_manifest(
             input_location=str(input_location),
             package_registry=resolve.sbom_registry,
         )
diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py
index bdae2ff41f..8c915f8720 100644
--- a/scanpipe/tests/test_pipelines.py
+++ b/scanpipe/tests/test_pipelines.py
@@ -586,7 +586,7 @@ def test_scanpipe_scan_codebase_pipeline_integration(self):
         expected_file = self.data_location / "is-npm-1.0.0_scan_codebase.json"
         self.assertPipelineResultEqual(expected_file, result_file)
 
-    def test_scanpipe_inspect_packages_creates_packages(self):
+    def test_scanpipe_inspect_packages_creates_packages_npm(self):
         pipeline_name = "inspect_packages"
         project1 = Project.objects.create(name="Analysis")
 
@@ -604,6 +604,24 @@ def test_scanpipe_inspect_packages_creates_packages(self):
         self.assertEqual(1, project1.discoveredpackages.count())
         self.assertEqual(1, project1.discovereddependencies.count())
 
+    def test_scanpipe_inspect_packages_creates_packages_pypi(self):
+        pipeline_name = "inspect_packages"
+        project1 = Project.objects.create(name="Analysis")
+
+        input_location = (
+            self.data_location / "manifests" / "python-inspector-0.10.0.zip"
+        )
+        project1.copy_input_from(input_location)
+
+        run = project1.add_pipeline(pipeline_name)
+        pipeline = run.make_pipeline_instance()
+
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
+        self.assertEqual(6, project1.codebaseresources.count())
+        self.assertEqual(0, project1.discoveredpackages.count())
+        self.assertEqual(26, project1.discovereddependencies.count())
+
     def test_scanpipe_scan_codebase_can_process_wheel(self):
         pipeline_name = "scan_codebase"
         project1 = Project.objects.create(name="Analysis")
@@ -861,7 +879,7 @@ def test_scanpipe_find_vulnerabilities_pipeline_integration(
         expected = vulnerability_data[0]["affected_by_vulnerabilities"]
         self.assertEqual(expected, package1.affected_by_vulnerabilities)
 
-    def test_scanpipe_inspect_manifest_pipeline_integration(self):
+    def test_scanpipe_resolve_dependencies_pipeline_integration(self):
         pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
@@ -873,10 +891,10 @@ def test_scanpipe_inspect_manifest_pipeline_integration(self):
         self.assertEqual(1, project1.projectmessages.count())
         message = project1.projectmessages.get()
         self.assertEqual("get_packages_from_manifest", message.model)
-        expected = "No manifests found for resolving packages"
+        expected = "No resources found with package data"
         self.assertIn(expected, message.description)
 
-    def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self):
+    def test_scanpipe_resolve_dependencies_pipeline_integration_empty_manifest(self):
         pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
@@ -891,7 +909,7 @@ def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self):
         expected = "No packages could be resolved for"
         self.assertIn(expected, message.description)
 
-    def test_scanpipe_inspect_manifest_pipeline_integration_misc(self):
+    def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self):
         pipeline_name = "resolve_dependencies"
         project1 = Project.objects.create(name="Analysis")
 
@@ -908,7 +926,7 @@ def test_scanpipe_inspect_manifest_pipeline_integration_misc(self):
         self.assertEqual(26, project1.discoveredpackages.count())
 
     @mock.patch("scanpipe.pipes.resolve.resolve_dependencies")
-    def test_scanpipe_inspect_manifest_pipeline_pypi_integration(
+    def test_scanpipe_resolve_dependencies_pipeline_pypi_integration(
         self, resolve_dependencies
     ):
         pipeline_name = "resolve_dependencies"
@@ -929,7 +947,7 @@ def test_scanpipe_inspect_manifest_pipeline_pypi_integration(
             if value and field_name not in exclude_fields:
                 self.assertEqual(value, getattr(discoveredpackage, field_name))
 
-    def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self):
+    def test_scanpipe_load_sbom_pipeline_aboutfile_integration(self):
         pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 
@@ -951,7 +969,7 @@ def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self):
         self.assertEqual("4.0.8", discoveredpackage.version)
         self.assertEqual("bsd-new", discoveredpackage.declared_license_expression)
 
-    def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self):
+    def test_scanpipe_load_sbom_pipeline_spdx_integration(self):
         pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 
@@ -973,7 +991,7 @@ def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self):
         self.assertEqual("MIT", discoveredpackage.extracted_license_statement)
         self.assertEqual("mit", discoveredpackage.declared_license_expression)
 
-    def test_scanpipe_inspect_manifest_pipeline_cyclonedx_integration(self):
+    def test_scanpipe_load_sbom_pipeline_cyclonedx_integration(self):
         pipeline_name = "load_sbom"
         project1 = Project.objects.create(name="Analysis")
 

From 03c21a8fa48ed1993d83133360001510580b573e Mon Sep 17 00:00:00 2001
From: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Date: Mon, 12 Feb 2024 19:32:51 +0530
Subject: [PATCH 3/5] Add unittests for new functions

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
---
 scanpipe/pipes/resolve.py             |  2 +-
 scanpipe/tests/pipes/test_resolve.py  | 57 +++++++++++++++++++++++++++
 scanpipe/tests/pipes/test_scancode.py | 12 ++++++
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py
index 6c192b94ba..7006484cc7 100644
--- a/scanpipe/pipes/resolve.py
+++ b/scanpipe/pipes/resolve.py
@@ -44,7 +44,7 @@
 """
 
 
-def get_packages(project, package_registry, manifest_resources, model):
+def get_packages(project, package_registry, manifest_resources, model=None):
     """
     Get package data from package manifests/lockfiles/SBOMs or
     get package data for resolved packages from package requirements.
diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py
index 2e908d971f..a36007b61c 100644
--- a/scanpipe/tests/pipes/test_resolve.py
+++ b/scanpipe/tests/pipes/test_resolve.py
@@ -27,6 +27,8 @@
 from scanpipe import pipes
 from scanpipe.models import Project
 from scanpipe.pipes import resolve
+from scanpipe.pipes.input import copy_inputs
+from scanpipe.pipes.scancode import extract_archives
 from scanpipe.tests import package_data1
 
 
@@ -154,3 +156,58 @@ def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self):
             "md5": "76cf50f29e47676962645632737365a7",
         }
         self.assertEqual(expected, package_data)
+
+    def test_scanpipe_resolve_get_manifest_resources(self):
+        project1 = Project.objects.create(name="Analysis")
+        input_location = (
+            self.data_location / "manifests" / "python-inspector-0.10.0.zip"
+        )
+        project1.copy_input_from(input_location)
+        copy_inputs(project1.inputs(), project1.codebase_path)
+
+        extract_archives(project1.codebase_path, recurse=True)
+        pipes.collect_and_create_codebase_resources(project1)
+
+        resources = resolve.get_manifest_resources(project1)
+        self.assertTrue(resources.exists())
+        requirements_resource = project1.codebaseresources.get(
+            path=(
+                "python-inspector-0.10.0.zip-extract/"
+                "python-inspector-0.10.0/requirements.txt"
+            )
+        )
+        self.assertIn(requirements_resource, resources)
+
+    def test_scanpipe_resolve_get_packages_from_sbom(self):
+        project1 = Project.objects.create(name="Analysis")
+        input_location = self.data_location / "manifests" / "toml.spdx.json"
+
+        project1.copy_input_from(input_location)
+        copy_inputs(project1.inputs(), project1.codebase_path)
+        pipes.collect_and_create_codebase_resources(project1)
+        resources = resolve.get_manifest_resources(project1)
+
+        packages = resolve.get_packages(
+            project1,
+            resolve.sbom_registry,
+            resources,
+        )
+        self.assertEqual(1, len(packages))
+        self.assertEqual("toml", packages[0]["name"])
+
+    def test_scanpipe_resolve_create_packages_and_dependencies(self):
+        project1 = Project.objects.create(name="Analysis")
+        input_location = self.data_location / "manifests" / "toml.spdx.json"
+
+        project1.copy_input_from(input_location)
+        copy_inputs(project1.inputs(), project1.codebase_path)
+        pipes.collect_and_create_codebase_resources(project1)
+        resources = resolve.get_manifest_resources(project1)
+        packages = resolve.get_packages(
+            project1,
+            resolve.sbom_registry,
+            resources,
+        )
+        resolve.create_packages_and_dependencies(project1, packages)
+        self.assertEqual(1, project1.discoveredpackages.count())
+        self.assertEqual(0, project1.discovereddependencies.count())
diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py
index 4f0b68c0b4..3164890991 100644
--- a/scanpipe/tests/pipes/test_scancode.py
+++ b/scanpipe/tests/pipes/test_scancode.py
@@ -40,6 +40,7 @@
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
+from scanpipe.pipes import collect_and_create_codebase_resources
 from scanpipe.pipes import input
 from scanpipe.pipes import scancode
 from scanpipe.pipes.input import copy_input
@@ -542,3 +543,14 @@ def test_scanpipe_pipes_scancode_get_detection_data(self):
 
         results = scancode.get_detection_data(detection_entry)
         self.assertEqual(expected, results)
+
+    def test_scanpipe_scancode_process_package_data(self):
+        project1 = Project.objects.create(name="Utility: PurlDB")
+        package_json_location = self.data_location / "manifests" / "package.json"
+        copy_input(package_json_location, project1.codebase_path)
+        collect_and_create_codebase_resources(project1)
+        scancode.scan_for_application_packages(project1, assemble=False)
+        scancode.process_package_data(project1)
+
+        self.assertEqual(1, project1.discoveredpackages.count())
+        self.assertEqual(6, project1.discovereddependencies.count())

From b9da877e8b6fe03e47aadc1e87c92013b3f491d1 Mon Sep 17 00:00:00 2001
From: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Date: Mon, 12 Feb 2024 20:59:23 +0530
Subject: [PATCH 4/5] Update docs and add CHANGELOG entry

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
---
 CHANGELOG.rst               | 14 ++++++++++++++
 docs/automation.rst         |  2 +-
 docs/built-in-pipelines.rst | 24 ++++++++++++++++--------
 docs/faq.rst                | 19 ++++++++++++++-----
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 71f1bd88be..daa9564905 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -17,6 +17,20 @@ v33.2.0 (unreleased)
 
    https://github.com/nexB/scancode.io/issues/1071
 
+- Rename pipeline for consistency and precision:
+  * scan_codebase_packages: inspect_packages
+
+  Restructure the inspect_manifest pipeline into:
+  * load_sbom: for loading SPDX/CycloneDX SBOMs and ABOUT files
+  * resolve_dependencies: for resolving package dependencies
+  * inspect_packages: gets package data from package manifests/lockfiles 
+
+  A data migration is included to facilitate the migration of existing data.
+  Only the new names are available in the web UI but the REST API and CLI are backward
+  compatible with the old names.
+  https://github.com/nexB/scancode.io/issues/1034
+  https://github.com/nexB/scancode.io/discussions/1035
+
 v33.1.0 (2024-02-02)
 --------------------
 
diff --git a/docs/automation.rst b/docs/automation.rst
index 203f2895a0..54efae725b 100644
--- a/docs/automation.rst
+++ b/docs/automation.rst
@@ -27,7 +27,7 @@ automation methods such as a cron job or a git hook::
         "https://github.com/nexB/scancode.io/archive/refs/tags/v32.4.0.zip",
     ]
     PIPELINES = [
-        "scan_codebase_package",
+        "inspect_packages",
         "find_vulnerabilities",
     ]
     EXECUTE_NOW = True
diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst
index d5da9b71ab..f2e8a98109 100644
--- a/docs/built-in-pipelines.rst
+++ b/docs/built-in-pipelines.rst
@@ -72,6 +72,22 @@ Load Inventory
     :members:
     :member-order: bysource
 
+.. _pipeline_load_sbom:
+
+Load SBOM
+---------
+.. autoclass:: scanpipe.pipelines.load_sbom.LoadSBOM()
+    :members:
+    :member-order: bysource
+
+.. _pipeline_resolve_dependencies:
+
+Resolve Dependencies
+--------------------
+.. autoclass:: scanpipe.pipelines.resolve_dependencies.ResolveDependencies()
+    :members:
+    :member-order: bysource
+
 .. _pipeline_map_deploy_to_develop:
 
 Map Deploy To Develop
@@ -126,14 +142,6 @@ Scan Codebase
     :members:
     :member-order: bysource
 
-.. _pipeline_scan_codebase_package:
-
-Scan Codebase Package
----------------------
-.. autoclass:: scanpipe.pipelines.scan_codebase_packages.ScanCodebasePackages()
-    :members:
-    :member-order: bysource
-
 .. _pipeline_scan_single_package:
 
 Scan Single Package
diff --git a/docs/faq.rst b/docs/faq.rst
index 1111bf8c45..1b854b3413 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -25,18 +25,27 @@ Here are some general guidelines based on different input scenarios:
 
 - If you have a **Docker image** as input, use the
   :ref:`analyze_docker_image <pipeline_analyze_docker_image>` pipeline.
-- For a full **codebase compressed as an archive**, choose the
+- For a full **codebase compressed as an archive**, optionally also with
+  it's **pre-resolved dependenices**, and want to detect all the packages
+  present linked with their respective files, use the
   :ref:`scan_codebase <pipeline_scan_codebase>` pipeline.
-- If you have a **single package archive**, opt for the
+- If you have a **single package archive**, and you want to get information
+  on licenses, copyrights and package metadata for it, opt for the
   :ref:`scan_single_package <pipeline_scan_single_package>` pipeline.
 - When dealing with a **Linux root filesystem** (rootfs), the
   :ref:`analyze_root_filesystem_or_vm_image <pipeline_analyze_root_filesystem>` pipeline
   is the appropriate choice.
 - For processing the results of a **ScanCode-toolkit scan** or **ScanCode.io scan**,
   use the :ref:`load_inventory <pipeline_load_inventory>` pipeline.
-- When you have **manifest files**, such as a
-  **CycloneDX BOM, SPDX document, lockfile**, etc.,
-  use the :ref:`inspect_packages <pipeline_inspect_packages>` pipeline.
+- When you want to import **SPDX/CycloneDX SBOMs or ABOUT files** into a project,
+  use the :ref:`load_sbom <pipeline_load_sbom>` pipeline.
+- When you have **lockfiles or other package manifests** in a codebase and you want to
+  resolve packages from their package requirements, use the
+  :ref:`resolve_dependencies <pipeline_resolve_dependencies>` pipeline.
+- When you have application **package archives/codebases** and optionally also
+  their **pre-resolved dependenices** and you want to **inspect packages**
+  present in the package manifests and dependency, use the
+  :ref:`inspect_packages <pipeline_inspect_packages>` pipeline.
 - For scenarios involving both a **development and deployment codebase**, consider using
   the :ref:`map_deploy_to_develop <pipeline_map_deploy_to_develop>` pipeline.
 

From d92df4dea93d68d5a099618dacf9438069465030 Mon Sep 17 00:00:00 2001
From: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
Date: Tue, 13 Feb 2024 22:28:24 +0530
Subject: [PATCH 5/5] Improve docstrings for pipelines

Suggested-by: Philippe Ombredanne <pombredanne@nexb.com>
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
---
 scanpipe/pipelines/inspect_packages.py     |  2 +-
 scanpipe/pipelines/load_sbom.py            |  7 ++++---
 scanpipe/pipelines/resolve_dependencies.py | 19 ++++++++-----------
 scanpipe/pipelines/scan_codebase.py        |  2 +-
 scanpipe/pipelines/scan_single_package.py  |  6 +++---
 5 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py
index af59d737d4..702ae4d8b5 100644
--- a/scanpipe/pipelines/inspect_packages.py
+++ b/scanpipe/pipelines/inspect_packages.py
@@ -26,7 +26,7 @@
 
 class InspectPackages(ScanCodebase):
     """
-    Inspect a codebase for package and pre-resolved dependencies.
+    Inspect a codebase for packages and pre-resolved dependencies.
 
     This pipeline inspects a codebase for application packages
     and their dependencies using package manifests and dependency
diff --git a/scanpipe/pipelines/load_sbom.py b/scanpipe/pipelines/load_sbom.py
index 492722a438..727f027d5d 100644
--- a/scanpipe/pipelines/load_sbom.py
+++ b/scanpipe/pipelines/load_sbom.py
@@ -26,12 +26,13 @@
 
 class LoadSBOM(ScanCodebase):
     """
-    Detects and loads all associated packages from SBOMs.
+    Load package data from one or more SBOMs.
 
-    Supported BOMs:
+    Supported SBOMs:
     - SPDX document
     - CycloneDX BOM
-    - AboutCode ABOUT file
+    Other formats:
+    - AboutCode .ABOUT files for package curations.
     """
 
     @classmethod
diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py
index 6efac05269..d2597e8eec 100644
--- a/scanpipe/pipelines/resolve_dependencies.py
+++ b/scanpipe/pipelines/resolve_dependencies.py
@@ -26,15 +26,15 @@
 
 class ResolveDependencies(ScanCodebase):
     """
-    Resolve dependencies from package manifests/lockfiles.
+    Resolve dependencies from package manifests and lockfiles.
 
-    This pipeline inspects codebase lockfiles/manifest files
-    which contain package requirements and resolve their
-    associated packages.
+    This pipeline collects lockfiles and manifest files
+    that contain dependency requirements, and resolves these
+    to a concrete set of package versions.
 
-    Supports resolved packages for:
-    - Python: using nexB/python-inspector, supports requirements.txt and
-    setup.py manifests as input
+    Supports resolving packages for:
+    - Python: using python-inspector, using requirements.txt and
+    setup.py manifests as inputs
     """
 
     @classmethod
@@ -50,10 +50,7 @@ def steps(cls):
         )
 
     def get_manifest_inputs(self):
-        """
-        Locate all the package manifest files for which package resolvers are
-        supported, in the codebase resources.
-        """
+        """Locate package manifest files with a supported package resolver."""
         self.manifest_resources = resolve.get_manifest_resources(self.project)
 
     def get_packages_from_manifest(self):
diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py
index e86867019c..6535424913 100644
--- a/scanpipe/pipelines/scan_codebase.py
+++ b/scanpipe/pipelines/scan_codebase.py
@@ -32,7 +32,7 @@ class ScanCodebase(Pipeline):
 
     This pipeline does not further scan the files contained in a package
     for license and copyrights and only considers the declared license
-    of a package.
+    of a package. It does not scan for system (Linux distro) packages.
     """
 
     @classmethod
diff --git a/scanpipe/pipelines/scan_single_package.py b/scanpipe/pipelines/scan_single_package.py
index ae88295127..7fd4351704 100644
--- a/scanpipe/pipelines/scan_single_package.py
+++ b/scanpipe/pipelines/scan_single_package.py
@@ -36,10 +36,10 @@
 
 class ScanSinglePackage(Pipeline):
     """
-    Scan a single package file or package archive.
+    Scan a single package archive (or package manifest file).
 
-    This pipeline scans a single package for package
-    metadata, declared dependencies, licenses, and copyrights.
+    This pipeline scans a single package for package metadata,
+    declared dependencies, licenses, license clarity score and copyrights.
 
     The output is a summary of the scan results in JSON format.
     """