diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cdd732411a..3fd07c0065 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,6 +23,11 @@ v34.1.0 (unreleased) https://github.com/nexB/scancode.io/issues/1121 https://github.com/nexB/scancode.io/issues/1122 +- Use the `package_only` option in scancode `get_package_data` API in + `inspect_packages` pipeline, to skip license and copyright detection in + extracted license and copyright statements found in package metadata. + https://github.com/nexB/scancode-toolkit/pull/3689 + - Rename the ``match_to_purldb`` pipeline to ``match_to_matchcode``, and add MatchCode.io API settings to ScanCode.io settings. diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index 702ae4d8b5..a40528c3a5 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -49,6 +49,7 @@ def steps(cls): cls.flag_empty_files, cls.flag_ignored_resources, cls.scan_for_application_packages, + cls.create_packages_and_dependencies, ) def scan_for_application_packages(self): @@ -60,5 +61,11 @@ def scan_for_application_packages(self): # only detect package_data in resources and create # Package/Dependency instances directly instead of assembling # the packages and assigning files to them - scancode.scan_for_application_packages(self.project, assemble=False) + scancode.scan_for_application_packages( + project=self.project, + assemble=False, + package_only=True, + ) + + def create_packages_and_dependencies(self): scancode.process_package_data(self.project) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 1ed90c438e..538d68d4ea 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -219,14 +219,18 @@ def scan_file(location, with_threading=True, min_license_score=0, **kwargs): return _scan_resource(location, scanners, with_threading=with_threading) -def scan_for_package_data(location, with_threading=True, **kwargs): +def scan_for_package_data(location, with_threading=True, package_only=False, **kwargs): """ Run a package scan on provided `location` using the scancode-toolkit direct API. Return a dict of scan `results` and a list of `errors`. """ + scancode_get_packages = partial( + scancode_api.get_package_data, + package_only=package_only, + ) scanners = [ - Scanner("package_data", scancode_api.get_package_data), + Scanner("package_data", scancode_get_packages), ] return _scan_resource(location, scanners, with_threading=with_threading) @@ -343,7 +347,9 @@ def scan_for_files(project, resource_qs=None, progress_logger=None): ) -def scan_for_application_packages(project, assemble=True, progress_logger=None): +def scan_for_application_packages( + project, assemble=True, package_only=False, progress_logger=None +): """ Run a package scan on resources without a status for a `project`, and add them in their respective `package_data` attribute. @@ -359,6 +365,10 @@ def scan_for_application_packages(project, assemble=True, progress_logger=None): """ resource_qs = project.codebaseresources.no_status() + scan_func_kwargs = { + "package_only": package_only, + } + # Collect detected Package data and save it to the CodebaseResource it was # detected from. scan_resources( @@ -366,6 +376,7 @@ def scan_for_application_packages(project, assemble=True, progress_logger=None): scan_func=scan_for_package_data, save_func=save_scan_package_results, progress_logger=progress_logger, + scan_func_kwargs=scan_func_kwargs, ) # Iterate through CodebaseResources with Package data and handle them using @@ -460,15 +471,19 @@ def process_package_data(project): logger.info(f" Processing: {resource.path}") for package_mapping in resource.package_data: pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) + if not pd.can_assemble: + continue + logger.info(f" Package data: {pd.purl}") package_data = pd.to_dict() dependencies = package_data.pop("dependencies") - pipes.update_or_create_package(project, package_data) - for dep in dependencies: pipes.update_or_create_dependency(project, dep) + if pd.purl: + pipes.update_or_create_package(project, package_data) + def get_packages_with_purl_from_resources(project): """ diff --git a/scanpipe/tests/data/manifests/requirements.txt b/scanpipe/tests/data/manifests/requirements.txt new file mode 100644 index 0000000000..adf24f5901 --- /dev/null +++ b/scanpipe/tests/data/manifests/requirements.txt @@ -0,0 +1 @@ +click==8.1.3 diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 75cdc12af0..5e5f8ade2e 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -922,9 +922,7 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") - input_location = ( - self.data_location / "manifests" / "python-inspector-0.10.0.zip" - ) + input_location = self.data_location / "manifests" / "requirements.txt" project1.copy_input_from(input_location) run = project1.add_pipeline(pipeline_name) @@ -932,7 +930,7 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) - self.assertEqual(26, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discoveredpackages.count()) @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") def test_scanpipe_resolve_dependencies_pipeline_pypi_integration(