diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 71f1bd88be..daa9564905 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,20 @@ v33.2.0 (unreleased) https://github.com/nexB/scancode.io/issues/1071 +- Rename pipeline for consistency and precision: + * scan_codebase_packages: inspect_packages + + Restructure the inspect_manifest pipeline into: + * load_sbom: for loading SPDX/CycloneDX SBOMs and ABOUT files + * resolve_dependencies: for resolving package dependencies + * inspect_packages: gets package data from package manifests/lockfiles + + A data migration is included to facilitate the migration of existing data. + Only the new names are available in the web UI but the REST API and CLI are backward + compatible with the old names. + https://github.com/nexB/scancode.io/issues/1034 + https://github.com/nexB/scancode.io/discussions/1035 + v33.1.0 (2024-02-02) -------------------- diff --git a/docs/automation.rst b/docs/automation.rst index 203f2895a0..54efae725b 100644 --- a/docs/automation.rst +++ b/docs/automation.rst @@ -27,7 +27,7 @@ automation methods such as a cron job or a git hook:: "https://github.com/nexB/scancode.io/archive/refs/tags/v32.4.0.zip", ] PIPELINES = [ - "scan_codebase_package", + "inspect_packages", "find_vulnerabilities", ] EXECUTE_NOW = True diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index d5da9b71ab..f2e8a98109 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -72,6 +72,22 @@ Load Inventory :members: :member-order: bysource +.. _pipeline_load_sbom: + +Load SBOM +--------- +.. autoclass:: scanpipe.pipelines.load_sbom.LoadSBOM() + :members: + :member-order: bysource + +.. _pipeline_resolve_dependencies: + +Resolve Dependencies +-------------------- +.. autoclass:: scanpipe.pipelines.resolve_dependencies.ResolveDependencies() + :members: + :member-order: bysource + .. _pipeline_map_deploy_to_develop: Map Deploy To Develop @@ -126,14 +142,6 @@ Scan Codebase :members: :member-order: bysource -.. _pipeline_scan_codebase_package: - -Scan Codebase Package ---------------------- -.. autoclass:: scanpipe.pipelines.scan_codebase_packages.ScanCodebasePackages() - :members: - :member-order: bysource - .. _pipeline_scan_single_package: Scan Single Package diff --git a/docs/faq.rst b/docs/faq.rst index 1111bf8c45..1b854b3413 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -25,18 +25,27 @@ Here are some general guidelines based on different input scenarios: - If you have a **Docker image** as input, use the :ref:`analyze_docker_image ` pipeline. -- For a full **codebase compressed as an archive**, choose the +- For a full **codebase compressed as an archive**, optionally also with + it's **pre-resolved dependenices**, and want to detect all the packages + present linked with their respective files, use the :ref:`scan_codebase ` pipeline. -- If you have a **single package archive**, opt for the +- If you have a **single package archive**, and you want to get information + on licenses, copyrights and package metadata for it, opt for the :ref:`scan_single_package ` pipeline. - When dealing with a **Linux root filesystem** (rootfs), the :ref:`analyze_root_filesystem_or_vm_image ` pipeline is the appropriate choice. - For processing the results of a **ScanCode-toolkit scan** or **ScanCode.io scan**, use the :ref:`load_inventory ` pipeline. -- When you have **manifest files**, such as a - **CycloneDX BOM, SPDX document, lockfile**, etc., - use the :ref:`inspect_packages ` pipeline. +- When you want to import **SPDX/CycloneDX SBOMs or ABOUT files** into a project, + use the :ref:`load_sbom ` pipeline. +- When you have **lockfiles or other package manifests** in a codebase and you want to + resolve packages from their package requirements, use the + :ref:`resolve_dependencies ` pipeline. +- When you have application **package archives/codebases** and optionally also + their **pre-resolved dependenices** and you want to **inspect packages** + present in the package manifests and dependency, use the + :ref:`inspect_packages ` pipeline. - For scenarios involving both a **development and deployment codebase**, consider using the :ref:`map_deploy_to_develop ` pipeline. diff --git a/scanpipe/apps.py b/scanpipe/apps.py index 8f46096a96..cddfac36a9 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -178,6 +178,7 @@ def get_new_pipeline_name(pipeline_name): "inspect_manifest": "inspect_packages", "deploy_to_develop": "map_deploy_to_develop", "scan_package": "scan_single_package", + "scan_codebase_packages": "inspect_packages", } if new_name := pipeline_old_names_mapping.get(pipeline_name): warnings.warn( diff --git a/scanpipe/migrations/0053_restructure_pipelines_data.py b/scanpipe/migrations/0053_restructure_pipelines_data.py new file mode 100644 index 0000000000..e5b9e14dcc --- /dev/null +++ b/scanpipe/migrations/0053_restructure_pipelines_data.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.1 on 2024-02-09 15:05 + +from django.db import migrations + + +pipeline_old_names_mapping = { + "scan_codebase_packages": "inspect_packages", +} + + +def rename_pipelines_data(apps, schema_editor): + Run = apps.get_model("scanpipe", "Run") + for old_name, new_name in pipeline_old_names_mapping.items(): + Run.objects.filter(pipeline_name=old_name).update(pipeline_name=new_name) + + +def reverse_rename_pipelines_data(apps, schema_editor): + Run = apps.get_model("scanpipe", "Run") + for old_name, new_name in pipeline_old_names_mapping.items(): + Run.objects.filter(pipeline_name=new_name).update(pipeline_name=old_name) + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0052_run_selected_groups"), + ] + + operations = [ + migrations.RunPython( + rename_pipelines_data, + reverse_code=reverse_rename_pipelines_data, + ), + ] diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index f5a8568924..702ae4d8b5 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -21,32 +21,23 @@ # Visit https://github.com/nexB/scancode.io for support and download. from scanpipe.pipelines.scan_codebase import ScanCodebase -from scanpipe.pipes import resolve -from scanpipe.pipes import update_or_create_package +from scanpipe.pipes import scancode class InspectPackages(ScanCodebase): """ - Inspect a codebase manifest files and resolve their associated packages. + Inspect a codebase for packages and pre-resolved dependencies. - Supports resolved packages for: - - Python: using nexB/python-inspector, supports requirements.txt and - setup.py manifests as input + This pipeline inspects a codebase for application packages + and their dependencies using package manifests and dependency + lockfiles. It does not resolve dependencies, it does instead + collect already pre-resolved dependencies from lockfiles, and + direct dependencies (possibly not resolved) as found in + package manifests' dependency sections. - Supports: - - BOM: SPDX document, CycloneDX BOM, AboutCode ABOUT file - - Python: requirements.txt, setup.py, setup.cfg, Pipfile.lock - - JavaScript: yarn.lock lockfile, npm package-lock.json lockfile - - Java: Java JAR MANIFEST.MF, Gradle build script - - Ruby: RubyGems gemspec manifest, RubyGems Bundler Gemfile.lock - - Rust: Rust Cargo.lock dependencies lockfile, Rust Cargo.toml package manifest - - PHP: PHP composer lockfile, PHP composer manifest - - NuGet: nuspec package manifest - - Dart: pubspec manifest, pubspec lockfile - - OS: FreeBSD compact package manifest, Debian installed packages database - - Full list available at https://scancode-toolkit.readthedocs.io/en/ - doc-update-licenses/reference/available_package_parsers.html + See documentation for the list of supported package manifests and + dependency lockfiles: + https://scancode-toolkit.readthedocs.io/en/stable/reference/available_package_parsers.html """ @classmethod @@ -55,46 +46,19 @@ def steps(cls): cls.copy_inputs_to_codebase_directory, cls.extract_archives, cls.collect_and_create_codebase_resources, + cls.flag_empty_files, cls.flag_ignored_resources, - cls.get_manifest_inputs, - cls.get_packages_from_manifest, - cls.create_resolved_packages, + cls.scan_for_application_packages, ) - def get_manifest_inputs(self): - """Locate all the manifest files from the project's input/ directory.""" - self.manifest_resources = resolve.get_manifest_resources(self.project) - - def get_packages_from_manifest(self): - """Get packages data from manifest files.""" - self.resolved_packages = [] - - if not self.manifest_resources.exists(): - self.project.add_warning( - description="No manifests found for resolving packages", - model="get_packages_from_manifest", - ) - return - - for resource in self.manifest_resources: - if packages := resolve.resolve_packages(resource.location): - self.resolved_packages.extend(packages) - else: - self.project.add_error( - description="No packages could be resolved for", - model="get_packages_from_manifest", - details={"path": resource.path}, - ) - - def create_resolved_packages(self): - """Create the resolved packages and their dependencies in the database.""" - for package_data in self.resolved_packages: - package_data = resolve.set_license_expression(package_data) - dependencies = package_data.pop("dependencies", []) - update_or_create_package(self.project, package_data) - - for dependency_data in dependencies: - resolved_package = dependency_data.get("resolved_package") - if resolved_package: - resolved_package.pop("dependencies", []) - update_or_create_package(self.project, resolved_package) + def scan_for_application_packages(self): + """ + Scan resources for package information to add DiscoveredPackage + and DiscoveredDependency objects from detected package data. + """ + # `assemble` is set to False because here in this pipeline we + # only detect package_data in resources and create + # Package/Dependency instances directly instead of assembling + # the packages and assigning files to them + scancode.scan_for_application_packages(self.project, assemble=False) + scancode.process_package_data(self.project) diff --git a/scanpipe/pipelines/scan_codebase_packages.py b/scanpipe/pipelines/load_sbom.py similarity index 57% rename from scanpipe/pipelines/scan_codebase_packages.py rename to scanpipe/pipelines/load_sbom.py index 1442ec20dd..727f027d5d 100644 --- a/scanpipe/pipelines/scan_codebase_packages.py +++ b/scanpipe/pipelines/load_sbom.py @@ -21,15 +21,18 @@ # Visit https://github.com/nexB/scancode.io for support and download. from scanpipe.pipelines.scan_codebase import ScanCodebase -from scanpipe.pipes import scancode +from scanpipe.pipes import resolve -class ScanCodebasePackages(ScanCodebase): +class LoadSBOM(ScanCodebase): """ - Scan a codebase for PURLs without assembling full packages/dependencies. + Load package data from one or more SBOMs. - This Pipeline is intended for gathering PURL information from a - codebase without the overhead of full package assembly. + Supported SBOMs: + - SPDX document + - CycloneDX BOM + Other formats: + - AboutCode .ABOUT files for package curations. """ @classmethod @@ -40,12 +43,27 @@ def steps(cls): cls.collect_and_create_codebase_resources, cls.flag_empty_files, cls.flag_ignored_resources, - cls.scan_for_application_packages, + cls.get_sbom_inputs, + cls.get_packages_from_sboms, + cls.create_packages_from_sboms, ) - def scan_for_application_packages(self): - """Scan unknown resources for packages information.""" - # `assemble` is set to False because here in this pipeline we - # only detect package_data in resources without creating - # Package/Dependency instances, to get all the purls from a codebase. - scancode.scan_for_application_packages(self.project, assemble=False) + def get_sbom_inputs(self): + """Locate all the SBOMs among the codebase resources.""" + self.manifest_resources = resolve.get_manifest_resources(self.project) + + def get_packages_from_sboms(self): + """Get packages data from SBOMs.""" + self.packages = resolve.get_packages( + project=self.project, + package_registry=resolve.sbom_registry, + manifest_resources=self.manifest_resources, + model="get_packages_from_sboms", + ) + + def create_packages_from_sboms(self): + """Create the packages and dependencies from the SBOM, in the database.""" + resolve.create_packages_and_dependencies( + project=self.project, + packages=self.packages, + ) diff --git a/scanpipe/pipelines/populate_purldb.py b/scanpipe/pipelines/populate_purldb.py index b8edcd3fd7..41c098c248 100644 --- a/scanpipe/pipelines/populate_purldb.py +++ b/scanpipe/pipelines/populate_purldb.py @@ -22,7 +22,6 @@ from scanpipe.pipelines import Pipeline from scanpipe.pipes import purldb -from scanpipe.pipes import scancode class PopulatePurlDB(Pipeline): @@ -36,7 +35,6 @@ def steps(cls): return ( cls.populate_purldb_with_discovered_packages, cls.populate_purldb_with_discovered_dependencies, - cls.populate_purldb_with_detected_purls, ) def populate_purldb_with_discovered_packages(self): @@ -50,26 +48,3 @@ def populate_purldb_with_discovered_dependencies(self): purldb.populate_purldb_with_discovered_dependencies( project=self.project, logger=self.log ) - - def populate_purldb_with_detected_purls(self): - """Add DiscoveredPackage to PurlDB.""" - no_packages_and_no_dependencies = all( - [ - not self.project.discoveredpackages.exists(), - not self.project.discovereddependencies.exists(), - ] - ) - # Even when there are no packages/dependencies, resource level - # package data could be detected (i.e. when we detect packages, - # but skip the assembly step that creates - # package/dependency instances) - if no_packages_and_no_dependencies: - packages = scancode.get_packages_with_purl_from_resources(self.project) - purls = [{"purl": package.purl} for package in packages] - - self.log(f"Populating PurlDB with {len(purls):,d} " "detected PURLs"), - purldb.feed_purldb( - packages=purls, - chunk_size=100, - logger=self.log, - ) diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py new file mode 100644 index 0000000000..d2597e8eec --- /dev/null +++ b/scanpipe/pipelines/resolve_dependencies.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from scanpipe.pipelines.scan_codebase import ScanCodebase +from scanpipe.pipes import resolve + + +class ResolveDependencies(ScanCodebase): + """ + Resolve dependencies from package manifests and lockfiles. + + This pipeline collects lockfiles and manifest files + that contain dependency requirements, and resolves these + to a concrete set of package versions. + + Supports resolving packages for: + - Python: using python-inspector, using requirements.txt and + setup.py manifests as inputs + """ + + @classmethod + def steps(cls): + return ( + cls.copy_inputs_to_codebase_directory, + cls.extract_archives, + cls.collect_and_create_codebase_resources, + cls.flag_ignored_resources, + cls.get_manifest_inputs, + cls.get_packages_from_manifest, + cls.create_resolved_packages, + ) + + def get_manifest_inputs(self): + """Locate package manifest files with a supported package resolver.""" + self.manifest_resources = resolve.get_manifest_resources(self.project) + + def get_packages_from_manifest(self): + """ + Resolve package data from lockfiles/requirement files with package + requirements/dependenices. + """ + self.resolved_packages = resolve.get_packages( + project=self.project, + package_registry=resolve.resolver_registry, + manifest_resources=self.manifest_resources, + model="get_packages_from_manifest", + ) + + def create_resolved_packages(self): + """Create the resolved packages and their dependencies in the database.""" + resolve.create_packages_and_dependencies( + project=self.project, + packages=self.resolved_packages, + resolved=True, + ) diff --git a/scanpipe/pipelines/scan_codebase.py b/scanpipe/pipelines/scan_codebase.py index b754606baa..6535424913 100644 --- a/scanpipe/pipelines/scan_codebase.py +++ b/scanpipe/pipelines/scan_codebase.py @@ -28,15 +28,11 @@ class ScanCodebase(Pipeline): """ - Scan a codebase with ScanCode-toolkit. + Scan a codebase for application packages, licenses, and copyrights. - If the codebase consists of several packages and dependencies, it will try to - resolve and scan those too. - - Input files are copied to the project's codebase/ directory and are extracted - in place before running the scan. - Alternatively, the code can be manually copied to the project codebase/ - directory. + This pipeline does not further scan the files contained in a package + for license and copyrights and only considers the declared license + of a package. It does not scan for system (Linux distro) packages. """ @classmethod diff --git a/scanpipe/pipelines/scan_single_package.py b/scanpipe/pipelines/scan_single_package.py index b28550101e..7fd4351704 100644 --- a/scanpipe/pipelines/scan_single_package.py +++ b/scanpipe/pipelines/scan_single_package.py @@ -36,7 +36,10 @@ class ScanSinglePackage(Pipeline): """ - Scan a single package file or package archive with ScanCode-toolkit. + Scan a single package archive (or package manifest file). + + This pipeline scans a single package for package metadata, + declared dependencies, licenses, license clarity score and copyrights. The output is a summary of the scan results in JSON format. """ diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 8072b3977b..7006484cc7 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -31,35 +31,87 @@ from packagedcode.licensing import get_license_detections_and_expression from packageurl import PackageURL from python_inspector.api import resolve_dependencies -from scancode.api import get_package_data from scanpipe.models import DiscoveredPackage from scanpipe.pipes import cyclonedx from scanpipe.pipes import flag from scanpipe.pipes import spdx +from scanpipe.pipes import update_or_create_dependency +from scanpipe.pipes import update_or_create_package """ Resolve packages from manifest, lockfile, and SBOM. """ -def resolve_packages(input_location): - """Resolve the packages from manifest file.""" +def get_packages(project, package_registry, manifest_resources, model=None): + """ + Get package data from package manifests/lockfiles/SBOMs or + get package data for resolved packages from package requirements. + """ + resolved_packages = [] + + if not manifest_resources.exists(): + project.add_warning( + description="No resources found with package data", + model=model, + ) + return + + for resource in manifest_resources: + if packages := get_packages_from_manifest( + input_location=resource.location, + package_registry=package_registry, + ): + resolved_packages.extend(packages) + else: + project.add_error( + description="No packages could be resolved for", + model=model, + details={"path": resource.path}, + ) + + return resolved_packages + + +def create_packages_and_dependencies(project, packages, resolved=False): + """ + Create DiscoveredPackage and DiscoveredDependency objects for + packages detected in a package manifest, lockfile or SBOM. + + If resolved, create packages out of resolved dependencies, + otherwise create dependencies. + """ + for package_data in packages: + package_data = set_license_expression(package_data) + dependencies = package_data.pop("dependencies", []) + update_or_create_package(project, package_data) + + for dependency_data in dependencies: + if resolved: + if resolved_package := dependency_data.get("resolved_package"): + resolved_package.pop("dependencies", []) + update_or_create_package(project, resolved_package) + else: + update_or_create_dependency(project, dependency_data) + + +def get_packages_from_manifest(input_location, package_registry=None): + """ + Resolve packages or get packages data from a package manifest file/ + lockfile/SBOM at `input_location`. + """ default_package_type = get_default_package_type(input_location) # we only try to resolve packages if file at input_location is # a package manifest, and ignore for other files if not default_package_type: return - # The ScanCode.io resolvers take precedence over the ScanCode-toolkit ones. - resolver = resolver_registry.get(default_package_type) + # Get resolvers for available packages/SBOMs in the registry + resolver = package_registry.get(default_package_type) if resolver: resolved_packages = resolver(input_location=input_location) - else: - package_data = get_package_data(location=input_location) - resolved_packages = package_data.get("package_data", []) - - return resolved_packages + return resolved_packages def get_manifest_resources(project): @@ -280,10 +332,17 @@ def get_default_package_type(input_location): return "spdx" -# Mapping between the `default_package_type` its related resolver function +# Mapping between `default_package_type` its related resolver functions +# for package dependency resolvers resolver_registry = { - "about": resolve_about_packages, "pypi": resolve_pypi_packages, +} + + +# Mapping between `default_package_type` its related resolver functions +# for SBOMs and About files +sbom_registry = { + "about": resolve_about_packages, "spdx": resolve_spdx_packages, "cyclonedx": resolve_cyclonedx_packages, } diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 6854d56818..35bf91c173 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -404,7 +404,8 @@ def add_resource_to_package(package_uid, resource, project): def assemble_packages(project): """ Create instances of DiscoveredPackage and DiscoveredDependency for `project` - from the parsed package data present in the CodebaseResources of `project`. + from the parsed package data present in the CodebaseResources of `project`, + using the respective package handlers for each package manifest type. """ logger.info(f"Project {project} assemble_packages:") seen_resource_paths = set() @@ -442,6 +443,34 @@ def assemble_packages(project): logger.info(f"Unknown Package assembly item type: {item!r}") +def process_package_data(project): + """ + Create instances of DiscoveredPackage and DiscoveredDependency for `project` + from the parsed package data present in the CodebaseResources of `project`. + + Here package assembly though package handlers are not performed, instead + package/dependency objects are created directly from package data. + """ + logger.info(f"Project {project} process_package_data:") + seen_resource_paths = set() + + for resource in project.codebaseresources.has_package_data(): + if resource.path in seen_resource_paths: + continue + + logger.info(f" Processing: {resource.path}") + for package_mapping in resource.package_data: + pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) + logger.info(f" Package data: {pd.purl}") + + package_data = pd.to_dict() + dependencies = package_data.pop("dependencies") + pipes.update_or_create_package(project, package_data) + + for dep in dependencies: + pipes.update_or_create_dependency(project, dep) + + def get_packages_with_purl_from_resources(project): """ Yield Dependency or PackageData objects created from detected package_data diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index f8802cd4d1..a36007b61c 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -20,7 +20,6 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -import json from pathlib import Path from django.test import TestCase @@ -28,6 +27,8 @@ from scanpipe import pipes from scanpipe.models import Project from scanpipe.pipes import resolve +from scanpipe.pipes.input import copy_inputs +from scanpipe.pipes.scancode import extract_archives from scanpipe.tests import package_data1 @@ -84,10 +85,13 @@ def test_scanpipe_pipes_resolve_convert_spdx_expression(self): scancode_expression = "mit OR gpl-2.0 WITH generic-exception" self.assertEqual(scancode_expression, resolve.convert_spdx_expression(spdx)) - def test_scanpipe_pipes_resolve_resolve_packages(self): + def test_scanpipe_pipes_resolve_get_packages_from_manifest(self): # ScanCode.io resolvers input_location = self.manifest_location / "Django-4.0.8-py3-none-any.whl.ABOUT" - packages = resolve.resolve_packages(str(input_location)) + packages = resolve.get_packages_from_manifest( + input_location=str(input_location), + package_registry=resolve.sbom_registry, + ) expected = { "filename": "Django-4.0.8-py3-none-any.whl", "download_url": "https://python.org/Django-4.0.8-py3-none-any.whl", @@ -102,13 +106,6 @@ def test_scanpipe_pipes_resolve_resolve_packages(self): } self.assertEqual([expected], packages) - # ScanCode-toolkit resolvers - input_location = self.manifest_location / "package.json" - packages = resolve.resolve_packages(str(input_location)) - expected_location = self.manifest_location / "package.expected.json" - expected = json.loads(expected_location.read_text()) - self.assertEqual(expected, packages) - def test_scanpipe_pipes_resolve_resolve_about_packages(self): input_location = self.manifest_location / "Django-4.0.8-py3-none-any.whl.ABOUT" package = resolve.resolve_about_packages(str(input_location)) @@ -159,3 +156,58 @@ def test_scanpipe_pipes_resolve_spdx_package_to_discovered_package_data(self): "md5": "76cf50f29e47676962645632737365a7", } self.assertEqual(expected, package_data) + + def test_scanpipe_resolve_get_manifest_resources(self): + project1 = Project.objects.create(name="Analysis") + input_location = ( + self.data_location / "manifests" / "python-inspector-0.10.0.zip" + ) + project1.copy_input_from(input_location) + copy_inputs(project1.inputs(), project1.codebase_path) + + extract_archives(project1.codebase_path, recurse=True) + pipes.collect_and_create_codebase_resources(project1) + + resources = resolve.get_manifest_resources(project1) + self.assertTrue(resources.exists()) + requirements_resource = project1.codebaseresources.get( + path=( + "python-inspector-0.10.0.zip-extract/" + "python-inspector-0.10.0/requirements.txt" + ) + ) + self.assertIn(requirements_resource, resources) + + def test_scanpipe_resolve_get_packages_from_sbom(self): + project1 = Project.objects.create(name="Analysis") + input_location = self.data_location / "manifests" / "toml.spdx.json" + + project1.copy_input_from(input_location) + copy_inputs(project1.inputs(), project1.codebase_path) + pipes.collect_and_create_codebase_resources(project1) + resources = resolve.get_manifest_resources(project1) + + packages = resolve.get_packages( + project1, + resolve.sbom_registry, + resources, + ) + self.assertEqual(1, len(packages)) + self.assertEqual("toml", packages[0]["name"]) + + def test_scanpipe_resolve_create_packages_and_dependencies(self): + project1 = Project.objects.create(name="Analysis") + input_location = self.data_location / "manifests" / "toml.spdx.json" + + project1.copy_input_from(input_location) + copy_inputs(project1.inputs(), project1.codebase_path) + pipes.collect_and_create_codebase_resources(project1) + resources = resolve.get_manifest_resources(project1) + packages = resolve.get_packages( + project1, + resolve.sbom_registry, + resources, + ) + resolve.create_packages_and_dependencies(project1, packages) + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(0, project1.discovereddependencies.count()) diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index 4f0b68c0b4..3164890991 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -40,6 +40,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project +from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes import input from scanpipe.pipes import scancode from scanpipe.pipes.input import copy_input @@ -542,3 +543,14 @@ def test_scanpipe_pipes_scancode_get_detection_data(self): results = scancode.get_detection_data(detection_entry) self.assertEqual(expected, results) + + def test_scanpipe_scancode_process_package_data(self): + project1 = Project.objects.create(name="Utility: PurlDB") + package_json_location = self.data_location / "manifests" / "package.json" + copy_input(package_json_location, project1.codebase_path) + collect_and_create_codebase_resources(project1) + scancode.scan_for_application_packages(project1, assemble=False) + scancode.process_package_data(project1) + + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(6, project1.discovereddependencies.count()) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 1045b0ad96..8c915f8720 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -586,8 +586,8 @@ def test_scanpipe_scan_codebase_pipeline_integration(self): expected_file = self.data_location / "is-npm-1.0.0_scan_codebase.json" self.assertPipelineResultEqual(expected_file, result_file) - def test_scanpipe_scan_codebase_packages_does_not_create_packages(self): - pipeline_name = "scan_codebase_packages" + def test_scanpipe_inspect_packages_creates_packages_npm(self): + pipeline_name = "inspect_packages" project1 = Project.objects.create(name="Analysis") filename = "is-npm-1.0.0.tgz" @@ -600,9 +600,27 @@ def test_scanpipe_scan_codebase_packages_does_not_create_packages(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) + self.assertEqual(6, project1.codebaseresources.count()) + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discovereddependencies.count()) + + def test_scanpipe_inspect_packages_creates_packages_pypi(self): + pipeline_name = "inspect_packages" + project1 = Project.objects.create(name="Analysis") + + input_location = ( + self.data_location / "manifests" / "python-inspector-0.10.0.zip" + ) + project1.copy_input_from(input_location) + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) self.assertEqual(6, project1.codebaseresources.count()) self.assertEqual(0, project1.discoveredpackages.count()) - self.assertEqual(0, project1.discovereddependencies.count()) + self.assertEqual(26, project1.discovereddependencies.count()) def test_scanpipe_scan_codebase_can_process_wheel(self): pipeline_name = "scan_codebase" @@ -861,8 +879,8 @@ def test_scanpipe_find_vulnerabilities_pipeline_integration( expected = vulnerability_data[0]["affected_by_vulnerabilities"] self.assertEqual(expected, package1.affected_by_vulnerabilities) - def test_scanpipe_inspect_manifest_pipeline_integration(self): - pipeline_name = "inspect_packages" + def test_scanpipe_resolve_dependencies_pipeline_integration(self): + pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") run = project1.add_pipeline(pipeline_name) @@ -873,11 +891,11 @@ def test_scanpipe_inspect_manifest_pipeline_integration(self): self.assertEqual(1, project1.projectmessages.count()) message = project1.projectmessages.get() self.assertEqual("get_packages_from_manifest", message.model) - expected = "No manifests found for resolving packages" + expected = "No resources found with package data" self.assertIn(expected, message.description) - def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self): - pipeline_name = "inspect_packages" + def test_scanpipe_resolve_dependencies_pipeline_integration_empty_manifest(self): + pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") run = project1.add_pipeline(pipeline_name) @@ -891,8 +909,8 @@ def test_scanpipe_inspect_manifest_pipeline_integration_empty_manifest(self): expected = "No packages could be resolved for" self.assertIn(expected, message.description) - def test_scanpipe_inspect_manifest_pipeline_integration_misc(self): - pipeline_name = "inspect_packages" + def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): + pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") input_location = ( @@ -908,10 +926,10 @@ def test_scanpipe_inspect_manifest_pipeline_integration_misc(self): self.assertEqual(26, project1.discoveredpackages.count()) @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") - def test_scanpipe_inspect_manifest_pipeline_pypi_integration( + def test_scanpipe_resolve_dependencies_pipeline_pypi_integration( self, resolve_dependencies ): - pipeline_name = "inspect_packages" + pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") run = project1.add_pipeline(pipeline_name) @@ -929,8 +947,8 @@ def test_scanpipe_inspect_manifest_pipeline_pypi_integration( if value and field_name not in exclude_fields: self.assertEqual(value, getattr(discoveredpackage, field_name)) - def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self): - pipeline_name = "inspect_packages" + def test_scanpipe_load_sbom_pipeline_aboutfile_integration(self): + pipeline_name = "load_sbom" project1 = Project.objects.create(name="Analysis") input_location = ( @@ -951,8 +969,8 @@ def test_scanpipe_inspect_manifest_pipeline_aboutfile_integration(self): self.assertEqual("4.0.8", discoveredpackage.version) self.assertEqual("bsd-new", discoveredpackage.declared_license_expression) - def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self): - pipeline_name = "inspect_packages" + def test_scanpipe_load_sbom_pipeline_spdx_integration(self): + pipeline_name = "load_sbom" project1 = Project.objects.create(name="Analysis") input_location = self.data_location / "manifests" / "toml.spdx.json" @@ -973,8 +991,8 @@ def test_scanpipe_inspect_manifest_pipeline_spdx_integration(self): self.assertEqual("MIT", discoveredpackage.extracted_license_statement) self.assertEqual("mit", discoveredpackage.declared_license_expression) - def test_scanpipe_inspect_manifest_pipeline_cyclonedx_integration(self): - pipeline_name = "inspect_packages" + def test_scanpipe_load_sbom_pipeline_cyclonedx_integration(self): + pipeline_name = "load_sbom" project1 = Project.objects.create(name="Analysis") input_location = self.data_location / "cyclonedx/nested.cdx.json" @@ -1176,6 +1194,7 @@ def mock_request_post_return(url, data, headers, timeout): pipes.collect_and_create_codebase_resources(project1) scancode.scan_for_application_packages(project1, assemble=False) + scancode.process_package_data(project1) run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() @@ -1183,7 +1202,10 @@ def mock_request_post_return(url, data, headers, timeout): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertIn("Populating PurlDB with 7 detected PURLs", run.log) - self.assertIn("Successfully queued 7 PURLs for indexing in PurlDB", run.log) + self.assertIn("Populating PurlDB with 1 PURLs from DiscoveredPackage", run.log) + self.assertIn( + "Populating PurlDB with 6 unresolved PURLs from DiscoveredDependency", + run.log, + ) self.assertIn("1 PURLs were already present in PurlDB index queue", run.log) self.assertIn("Couldn't index 1 unsupported PURLs", run.log) diff --git a/setup.cfg b/setup.cfg index 7c442883ca..3477ff1129 100644 --- a/setup.cfg +++ b/setup.cfg @@ -128,11 +128,12 @@ scancodeio_pipelines = find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages load_inventory = scanpipe.pipelines.load_inventory:LoadInventory + load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM map_deploy_to_develop = scanpipe.pipelines.deploy_to_develop:DeployToDevelop match_to_purldb = scanpipe.pipelines.match_to_purldb:MatchToPurlDB populate_purldb = scanpipe.pipelines.populate_purldb:PopulatePurlDB + resolve_dependencies = scanpipe.pipelines.resolve_dependencies:ResolveDependencies scan_codebase = scanpipe.pipelines.scan_codebase:ScanCodebase - scan_codebase_packages = scanpipe.pipelines.scan_codebase_packages:ScanCodebasePackages scan_single_package = scanpipe.pipelines.scan_single_package:ScanSinglePackage [isort]