diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1653b71013..4b3d60a067 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,14 @@ v34.1.0 (unreleased) Universal Ctags. https://github.com/nexB/scancode.io/pull/1116 +- Capture errors during the `inspect_elf_binaries` pipeline execution. + Errors on resource inspection are stored as project error message instead of global + pipeline failure. + The problematic resource path is stored in the message details and displayed in the + message list UI as a link to the resource details view. + https://github.com/nexB/scancode.io/issues/1121 + https://github.com/nexB/scancode.io/issues/1122 + v34.0.0 (2024-03-04) -------------------- diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 8f0143a10f..2ce946bfb0 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -45,7 +45,7 @@ Analyse Docker Windows Image .. _pipeline_collect_symbols: Collect Codebase Symbols (addon) ---------------------------------- +-------------------------------- .. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols() :members: :member-order: bysource diff --git a/docs/scanpipe-pipes.rst b/docs/scanpipe-pipes.rst index d11f4068c1..f8f04f2597 100644 --- a/docs/scanpipe-pipes.rst +++ b/docs/scanpipe-pipes.rst @@ -33,28 +33,53 @@ Docker .. automodule:: scanpipe.pipes.docker :members: +ELF +--- +.. automodule:: scanpipe.pipes.elf + :members: + Fetch ----- .. automodule:: scanpipe.pipes.fetch :members: :exclude-members: Download +Flag +---- +.. automodule:: scanpipe.pipes.flag + :members: + Input ----- .. automodule:: scanpipe.pipes.input :members: +JS +-- +.. automodule:: scanpipe.pipes.js + :members: + JVM --- .. automodule:: scanpipe.pipes.jvm :members: +MatchCode +--------- +.. automodule:: scanpipe.pipes.matchcode + :members: + Output ------ .. automodule:: scanpipe.pipes.output :members: :exclude-members: JSONResultsGenerator +PathMap +------- +.. automodule:: scanpipe.pipes.pathmap + :members: + PurlDB ------ .. automodule:: scanpipe.pipes.purldb @@ -80,9 +105,9 @@ SPDX .. automodule:: scanpipe.pipes.spdx :members: -Flag ----- -.. automodule:: scanpipe.pipes.flag +Symbols +------- +.. automodule:: scanpipe.pipes.symbols :members: VulnerableCode diff --git a/scanpipe/models.py b/scanpipe/models.py index d55f08d91e..1f25fa09be 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1097,12 +1097,20 @@ def get_next_run(self): return self.runs.not_started().earliest("created_date") def add_message( - self, severity, description="", model="", details=None, exception=None + self, + severity, + description="", + model="", + details=None, + exception=None, + resource=None, ): """ Create a ProjectMessage record for this Project. The ``model`` attribute can be provided as a string or as a Model class. + A ``resource`` can be provided to keep track of the codebase resource that was + analyzed when the error occurred. """ if inspect.isclass(model): model = model.__name__ @@ -1114,29 +1122,61 @@ def add_message( if exception and not description: description = str(exception) + details = details or {} + if resource: + # Do not change this field name as it has special behavior in templates. + details["resource_path"] = resource.path + return ProjectMessage.objects.create( project=self, severity=severity, description=description, model=model, - details=details or {}, + details=details, traceback=traceback, ) - def add_info(self, description="", model="", details=None, exception=None): + def add_info( + self, + description="", + model="", + details=None, + exception=None, + resource=None, + ): """Create an INFO ProjectMessage record for this project.""" severity = ProjectMessage.Severity.INFO - return self.add_message(severity, description, model, details, exception) + return self.add_message( + severity, description, model, details, exception, resource + ) - def add_warning(self, description="", model="", details=None, exception=None): + def add_warning( + self, + description="", + model="", + details=None, + exception=None, + resource=None, + ): """Create a WARNING ProjectMessage record for this project.""" severity = ProjectMessage.Severity.WARNING - return self.add_message(severity, description, model, details, exception) + return self.add_message( + severity, description, model, details, exception, resource + ) - def add_error(self, description="", model="", details=None, exception=None): + def add_error( + self, + description="", + model="", + details=None, + exception=None, + resource=None, + ): """Create an ERROR ProjectMessage record using for this project.""" severity = ProjectMessage.Severity.ERROR - return self.add_message(severity, description, model, details, exception) + return self.add_message( + severity, description, model, details, exception, resource + ) def get_absolute_url(self): """Return this project's details URL.""" @@ -1409,10 +1449,15 @@ def add_error(self, exception): Create a ProjectMessage record using the provided ``exception`` Exception instance. """ + resource = None + if isinstance(self, CodebaseResource): + resource = self + return self.project.add_error( model=self.__class__, details=model_to_dict(self), exception=exception, + resource=resource, ) def add_errors(self, exceptions): @@ -2538,12 +2583,9 @@ def create_and_add_package(self, package_data): except Exception as exception: self.project.add_warning( model=DiscoveredPackage, - details={ - "codebase_resource_path": self.path, - "codebase_resource_pk": self.pk, - **package_data, - }, + details=package_data, exception=exception, + resource=self, ) else: self.add_package(package) diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 849c2ae1e7..1900876432 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -230,25 +230,35 @@ def download_missing_inputs(self): if errors: raise InputFileError(errors) - def add_error(self, exception): + def add_error(self, exception, resource=None): """Create a ``ProjectMessage`` ERROR record on the current `project`.""" - self.project.add_error(model=self.pipeline_name, exception=exception) + self.project.add_error( + model=self.pipeline_name, + exception=exception, + resource=resource, + ) @contextmanager - def save_errors(self, *exceptions): + def save_errors(self, *exceptions, **kwargs): """ Context manager to save specified exceptions as ``ProjectMessage`` in the database. - Example in a Pipeline step: + - Example in a Pipeline step:: with self.save_errors(rootfs.DistroNotFound): rootfs.scan_rootfs_for_system_packages(self.project, rfs) + + - Example when iterating over resources:: + + for resource in self.project.codebaseresources.all(): + with self.save_errors(Exception, resource=resource): + analyse(resource) """ try: yield except exceptions as error: - self.add_error(exception=error) + self.add_error(exception=error, **kwargs) class Pipeline(BasePipeline): diff --git a/scanpipe/pipelines/inspect_elf_binaries.py b/scanpipe/pipelines/inspect_elf_binaries.py index 1d109a6127..f596c01dc0 100644 --- a/scanpipe/pipelines/inspect_elf_binaries.py +++ b/scanpipe/pipelines/inspect_elf_binaries.py @@ -20,11 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from pathlib import Path - -from elf_inspector.dwarf import get_dwarf_paths - from scanpipe.pipelines import Pipeline +from scanpipe.pipes.elf import collect_dwarf_source_path_references class InspectELFBinaries(Pipeline): @@ -38,10 +35,7 @@ def steps(cls): return (cls.collect_dwarf_source_path_references,) def collect_dwarf_source_path_references(self): - """ - Update ``extra_data`` of ELF files with - dwarf data extracted from ELF files. - """ - for elf in self.project.codebaseresources.elfs(): - dwarf_paths = get_dwarf_paths(Path(self.project.codebase_path / elf.path)) - elf.update_extra_data(dwarf_paths) + """Collect DWARF paths from ELF files and set values on the extra_data field.""" + for elf_resource in self.project.codebaseresources.elfs(): + with self.save_errors(Exception, resource=elf_resource): + collect_dwarf_source_path_references(elf_resource) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 74021c46ef..04a3feb214 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -826,15 +826,13 @@ def create_indexes(cls, project, from_about_files, logger=None): package_data = resolve.resolve_about_package( input_location=str(about_file_resource.location_path) ) - error_message_details = { - "path": about_file_resource.path, - "package_data": package_data, - } + error_message_details = {"package_data": package_data} if not package_data: project.add_error( description="Cannot create package from ABOUT file", model="map_about_files", details=error_message_details, + resource=about_file_resource, ) continue @@ -846,6 +844,7 @@ def create_indexes(cls, project, from_about_files, logger=None): description="ABOUT file does not have about_resource", model="map_about_files", details=error_message_details, + resource=about_file_resource, ) continue else: diff --git a/scanpipe/pipes/elf.py b/scanpipe/pipes/elf.py new file mode 100644 index 0000000000..b820b60cc3 --- /dev/null +++ b/scanpipe/pipes/elf.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from elf_inspector.dwarf import get_dwarf_paths + + +def collect_dwarf_source_path_references(resource): + """Collect and store the DWARF debug paths of the provided ELF ``resource``.""" + dwarf_paths = get_dwarf_paths(resource.location_path) + resource.update_extra_data(dwarf_paths) + return dwarf_paths diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index b13e0966a8..c6caead544 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -57,16 +57,13 @@ def get_packages(project, package_registry, manifest_resources, model=None): return for resource in manifest_resources: - if packages := get_packages_from_manifest( - input_location=resource.location, - package_registry=package_registry, - ): + if packages := get_packages_from_manifest(resource.location, package_registry): resolved_packages.extend(packages) else: project.add_error( description="No packages could be resolved for", model=model, - details={"path": resource.path}, + resource=resource, ) return resolved_packages diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 35bf91c173..1ed90c438e 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -391,11 +391,10 @@ def add_resource_to_package(package_uid, resource, project): try: package = project.discoveredpackages.get(package_uid=package_uid) except ObjectDoesNotExist as error: - details = { - "package_uid": str(package_uid), - "resource": str(resource), - } - project.add_error(error, model="assemble_package", details=details) + details = {"package_uid": str(package_uid)} + project.add_error( + error, model="assemble_package", details=details, resource=resource + ) return resource.discovered_packages.add(package) diff --git a/scanpipe/templates/scanpipe/message_list.html b/scanpipe/templates/scanpipe/message_list.html index d84f554aff..b07d072034 100644 --- a/scanpipe/templates/scanpipe/message_list.html +++ b/scanpipe/templates/scanpipe/message_list.html @@ -38,11 +38,11 @@
- {% if message.details.codebase_resource_pk and message.details.codebase_resource_path %} + {% if message.details.resource_path %}
- Codebase resource: - - {{ message.details.codebase_resource_path }} + Resource: + + {{ message.details.resource_path }}
{% endif %} diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index d3b6112ce1..85cc2dc42c 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -273,7 +273,7 @@ def test_scanpipe_add_resource_to_package(self): self.assertEqual(1, project1.projectmessages.count()) error = project1.projectmessages.get() self.assertEqual("assemble_package", error.model) - expected = {"resource": "filename.ext", "package_uid": "not_available"} + expected = {"resource_path": "filename.ext", "package_uid": "not_available"} self.assertEqual(expected, error.details) scancode.add_resource_to_package(package1.package_uid, resource1, project1) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 051a2ce0de..7c5714a7d3 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2304,7 +2304,7 @@ def test_scanpipe_project_model_update_extra_data(self): def test_scanpipe_codebase_resource_model_add_error(self): project1 = Project.objects.create(name="Analysis") - codebase_resource = CodebaseResource.objects.create(project=project1) + codebase_resource = CodebaseResource.objects.create(project=project1, path="a") error = codebase_resource.add_error(Exception("Error message")) self.assertEqual(error, ProjectMessage.objects.get()) @@ -2312,6 +2312,7 @@ def test_scanpipe_codebase_resource_model_add_error(self): self.assertTrue(error.details) self.assertEqual("Error message", error.description) self.assertEqual("", error.traceback) + self.assertEqual(codebase_resource.path, error.details["resource_path"]) def test_scanpipe_codebase_resource_model_add_errors(self): project1 = Project.objects.create(name="Analysis") @@ -2475,6 +2476,5 @@ def test_scanpipe_codebase_resource_create_and_add_package_warnings(self): expected_message = "value too long for type character varying(100)" self.assertEqual(expected_message, message.description) self.assertEqual(bad_data["version"], message.details["version"]) - self.assertTrue(message.details["codebase_resource_pk"]) - self.assertEqual(resource.path, message.details["codebase_resource_path"]) + self.assertEqual(resource.path, message.details["resource_path"]) self.assertIn("in save", message.traceback) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 07fe221176..75cdc12af0 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -36,6 +36,7 @@ from scancode.cli_test_utils import purl_with_fake_uuid from scanpipe import pipes +from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.pipelines import InputFileError @@ -203,6 +204,12 @@ def test_scanpipe_pipeline_class_save_errors_context_manager(self): self.assertEqual("Error message", message.description) self.assertIn('raise Exception("Error message")', message.traceback) + resource1 = CodebaseResource.objects.create(project=project1, path="filename") + with pipeline.save_errors(Exception, resource=resource1): + raise Exception("Error message") + message = project1.projectmessages.latest("created_date") + self.assertEqual({"resource_path": str(resource1.path)}, message.details) + def test_scanpipe_pipelines_is_pipeline(self): self.assertFalse(is_pipeline(None)) self.assertFalse(is_pipeline(Pipeline))