From 037eec6163e973f97f04850711a5a4de8ef74517 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 11 Mar 2024 16:49:29 +0530 Subject: [PATCH 1/4] Update inspect_packages pipeline * Split package/dependencies creation in a seperate step * Only create packages/dependencies from Assemblable PackageData Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipelines/inspect_packages.py | 5 ++++- scanpipe/pipes/scancode.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index 702ae4d8b5..31177fa228 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -49,6 +49,7 @@ def steps(cls): cls.flag_empty_files, cls.flag_ignored_resources, cls.scan_for_application_packages, + cls.create_packages_and_dependencies, ) def scan_for_application_packages(self): @@ -60,5 +61,7 @@ def scan_for_application_packages(self): # only detect package_data in resources and create # Package/Dependency instances directly instead of assembling # the packages and assigning files to them - scancode.scan_for_application_packages(self.project, assemble=False) + scancode.scan_for_application_packages(project=self.project, assemble=False) + + def create_packages_and_dependencies(self): scancode.process_package_data(self.project) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 1ed90c438e..d08ef0a692 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -39,6 +39,7 @@ from extractcode import api as extractcode_api from packagedcode import get_package_handler from packagedcode import models as packagedcode_models +from packagedcode import HANDLER_BY_DATASOURCE_ID from scancode import Scanner from scancode import api as scancode_api from scancode import cli as scancode_cli @@ -460,6 +461,9 @@ def process_package_data(project): logger.info(f" Processing: {resource.path}") for package_mapping in resource.package_data: pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) + if not pd.can_assemble: + continue + logger.info(f" Package data: {pd.purl}") package_data = pd.to_dict() From a75c3f0a0c1a618d42416746c9aebcc461ad6ce7 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 8 Mar 2024 16:12:28 +0530 Subject: [PATCH 2/4] Support fast package-only scan in inspect_packages Support the new only_packages attributes in scancode get_package_data API, to only scan for package data and skip license and copyright detection. Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipelines/inspect_packages.py | 6 +++++- scanpipe/pipes/scancode.py | 23 +++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index 31177fa228..a40528c3a5 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -61,7 +61,11 @@ def scan_for_application_packages(self): # only detect package_data in resources and create # Package/Dependency instances directly instead of assembling # the packages and assigning files to them - scancode.scan_for_application_packages(project=self.project, assemble=False) + scancode.scan_for_application_packages( + project=self.project, + assemble=False, + package_only=True, + ) def create_packages_and_dependencies(self): scancode.process_package_data(self.project) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index d08ef0a692..538d68d4ea 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -39,7 +39,6 @@ from extractcode import api as extractcode_api from packagedcode import get_package_handler from packagedcode import models as packagedcode_models -from packagedcode import HANDLER_BY_DATASOURCE_ID from scancode import Scanner from scancode import api as scancode_api from scancode import cli as scancode_cli @@ -220,14 +219,18 @@ def scan_file(location, with_threading=True, min_license_score=0, **kwargs): return _scan_resource(location, scanners, with_threading=with_threading) -def scan_for_package_data(location, with_threading=True, **kwargs): +def scan_for_package_data(location, with_threading=True, package_only=False, **kwargs): """ Run a package scan on provided `location` using the scancode-toolkit direct API. Return a dict of scan `results` and a list of `errors`. """ + scancode_get_packages = partial( + scancode_api.get_package_data, + package_only=package_only, + ) scanners = [ - Scanner("package_data", scancode_api.get_package_data), + Scanner("package_data", scancode_get_packages), ] return _scan_resource(location, scanners, with_threading=with_threading) @@ -344,7 +347,9 @@ def scan_for_files(project, resource_qs=None, progress_logger=None): ) -def scan_for_application_packages(project, assemble=True, progress_logger=None): +def scan_for_application_packages( + project, assemble=True, package_only=False, progress_logger=None +): """ Run a package scan on resources without a status for a `project`, and add them in their respective `package_data` attribute. @@ -360,6 +365,10 @@ def scan_for_application_packages(project, assemble=True, progress_logger=None): """ resource_qs = project.codebaseresources.no_status() + scan_func_kwargs = { + "package_only": package_only, + } + # Collect detected Package data and save it to the CodebaseResource it was # detected from. scan_resources( @@ -367,6 +376,7 @@ def scan_for_application_packages(project, assemble=True, progress_logger=None): scan_func=scan_for_package_data, save_func=save_scan_package_results, progress_logger=progress_logger, + scan_func_kwargs=scan_func_kwargs, ) # Iterate through CodebaseResources with Package data and handle them using @@ -468,11 +478,12 @@ def process_package_data(project): package_data = pd.to_dict() dependencies = package_data.pop("dependencies") - pipes.update_or_create_package(project, package_data) - for dep in dependencies: pipes.update_or_create_dependency(project, dep) + if pd.purl: + pipes.update_or_create_package(project, package_data) + def get_packages_with_purl_from_resources(project): """ From 1ebf507447c284a501c4e816d5bbcaee48eeb110 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 18:40:55 +0530 Subject: [PATCH 3/4] Add CHANGELOG entry Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cdd732411a..3fd07c0065 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,6 +23,11 @@ v34.1.0 (unreleased) https://github.com/nexB/scancode.io/issues/1121 https://github.com/nexB/scancode.io/issues/1122 +- Use the `package_only` option in scancode `get_package_data` API in + `inspect_packages` pipeline, to skip license and copyright detection in + extracted license and copyright statements found in package metadata. + https://github.com/nexB/scancode-toolkit/pull/3689 + - Rename the ``match_to_purldb`` pipeline to ``match_to_matchcode``, and add MatchCode.io API settings to ScanCode.io settings. From 11b0b353f82360e7ec8b2a4bd67a488499695b02 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 25 Mar 2024 20:24:11 +0530 Subject: [PATCH 4/4] Make resolve_dependencies integration test faster Reference: https://github.com/nexB/scancode.io/issues/1087 Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/tests/data/manifests/requirements.txt | 1 + scanpipe/tests/test_pipelines.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 scanpipe/tests/data/manifests/requirements.txt diff --git a/scanpipe/tests/data/manifests/requirements.txt b/scanpipe/tests/data/manifests/requirements.txt new file mode 100644 index 0000000000..adf24f5901 --- /dev/null +++ b/scanpipe/tests/data/manifests/requirements.txt @@ -0,0 +1 @@ +click==8.1.3 diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 75cdc12af0..5e5f8ade2e 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -922,9 +922,7 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") - input_location = ( - self.data_location / "manifests" / "python-inspector-0.10.0.zip" - ) + input_location = self.data_location / "manifests" / "requirements.txt" project1.copy_input_from(input_location) run = project1.add_pipeline(pipeline_name) @@ -932,7 +930,7 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(26, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discoveredpackages.count()) @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") def test_scanpipe_resolve_dependencies_pipeline_pypi_integration(