From c1599f2086e42459ce176664dff308e7293c1050 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Jun 2022 18:56:48 +0530 Subject: [PATCH] Do not fail without packages in cyclonedx #2987 Avoids crashing when generating a cyclonedx sbom from scancode-toolkit when there aren't any package options specified. Also show a warning message in the CLI and add a warning in the BOM metadata. Reference: nexB/scancode-toolkit#2987 Signed-off-by: Ayan Sinha Mahapatra --- src/formattedcode/output_cyclonedx.py | 73 +++++++++++++++++-- .../cyclonedx/expected-without-packages.json | 7 ++ tests/formattedcode/test_output_cyclonedx.py | 13 +++- 3 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 tests/formattedcode/data/cyclonedx/expected-without-packages.json diff --git a/src/formattedcode/output_cyclonedx.py b/src/formattedcode/output_cyclonedx.py index 69e2d2da6a3..9b162112e44 100644 --- a/src/formattedcode/output_cyclonedx.py +++ b/src/formattedcode/output_cyclonedx.py @@ -8,12 +8,15 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import os import json +import logging import uuid from collections import defaultdict from datetime import datetime from enum import Enum from typing import List +import warnings import attr from lxml import etree @@ -26,6 +29,25 @@ from plugincode.output import output_impl +TRACE = os.environ.get('SCANCODE_DEBUG_OUTPUTS', False) + + +def logger_debug(*args): + pass + + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + + class ToDictMixin: def to_dict(self): @@ -274,7 +296,7 @@ def from_package(cls, package): purl = package.get('purl') - return CycloneDxComponent( + return cls( bom_ref=purl, purl=purl, name=name, @@ -301,7 +323,7 @@ def from_packages(cls, packages): """ components_by_purl = defaultdict(list) for package in packages: - comp = CycloneDxComponent.from_package(package) + comp = cls.from_package(package) if not comp: continue components_by_purl[comp.purl].append(comp) @@ -497,7 +519,7 @@ def from_package(cls, package, components_by_purl): warnings_by_dependent[purl].append(msg) for ref, dependsOn in dependencies_by_dependent.items(): - yield CycloneDxDependency( + yield cls( ref=ref, dependsOn=dependsOn, warnings=warnings_by_dependent.get(purl, []) @@ -556,6 +578,11 @@ def from_headers(cls, headers): headers = [h for h in headers if h.get('tool_name') == 'scancode-toolkit'] scancode_header = headers[0] if headers else {} + if TRACE: + logger_debug('CycloneDxMetadata: headers') + from pprint import pformat + logger_debug(pformat(headers)) + try: tool_header = { 'vendor': 'AboutCode.org', @@ -568,11 +595,17 @@ def from_headers(cls, headers): props = dict( notice=scancode_header.get('notice'), errors=scancode_header.get('errors', []), + warnings=scancode_header.get('warnings', []), message=scancode_header.get('message'), ) props.update(scancode_header.get('extra_data', {})) properties = [CycloneDxProperty(k, v) for k, v in props.items()] + if TRACE: + logger_debug('CycloneDxMetadata: properties') + from pprint import pformat + logger_debug(pformat(properties)) + return CycloneDxMetadata( tools=[tool_header], properties=properties, @@ -596,6 +629,10 @@ def to_xml_element(self): return xmetadata +class CycloneDxPluginNoPackagesWarning(DeprecationWarning): + pass + + @attr.s class CycloneDxBom: """ @@ -629,10 +666,32 @@ def from_codebase(cls, codebase): """ Return a CycloneDxBom built from a ScanCode ``codebase``. """ - metadata = CycloneDxMetadata.from_headers(codebase.get_headers()) - packages = codebase.attributes.packages - components = list(CycloneDxComponent.from_packages(packages)) - dependencies = list(CycloneDxDependency.from_packages(packages, components)) + components = [] + dependencies = [] + + packages_not_found_message = ( + "The --cyclonedx-xml option will not output any component/dependency data " + "as there are no package data in the present scan. To get package data " + "please rerun the scan with --package or --system-package CLI options enabled." + ) + codebase.get_or_create_current_header() + + if hasattr(codebase.attributes, 'packages'): + packages = codebase.attributes.packages + components = list(CycloneDxComponent.from_packages(packages)) + dependencies = list(CycloneDxDependency.from_packages(packages, components)) + else: + warnings.simplefilter('always', CycloneDxPluginNoPackagesWarning) + warnings.warn( + packages_not_found_message, + CycloneDxPluginNoPackagesWarning, + stacklevel=2, + ) + headers = codebase.get_or_create_current_header() + headers.warnings.append(packages_not_found_message) + + codebase_headers = codebase.get_headers() + metadata = CycloneDxMetadata.from_headers(codebase_headers) return CycloneDxBom( metadata=metadata, diff --git a/tests/formattedcode/data/cyclonedx/expected-without-packages.json b/tests/formattedcode/data/cyclonedx/expected-without-packages.json new file mode 100644 index 00000000000..f36135bbfb9 --- /dev/null +++ b/tests/formattedcode/data/cyclonedx/expected-without-packages.json @@ -0,0 +1,7 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.3", + "version": 1, + "components": [], + "dependencies": [] +} \ No newline at end of file diff --git a/tests/formattedcode/test_output_cyclonedx.py b/tests/formattedcode/test_output_cyclonedx.py index cefac2538ca..49f5f7eb567 100644 --- a/tests/formattedcode/test_output_cyclonedx.py +++ b/tests/formattedcode/test_output_cyclonedx.py @@ -197,7 +197,8 @@ def test_CycloneDxMetadata_from_headers(): 'notice': 'some notice', 'message': 'some message', 'errors': ['some error'], - 'extra_data': {'WARNING': 'some warning', 'spdx_version': '3.1.2'} + 'warnings': ['some warning'], + 'extra_data': {'spdx_version': '3.1.2'} }] m = CycloneDxMetadata.from_headers(headers).to_dict() m.pop('timestamp') @@ -205,8 +206,8 @@ def test_CycloneDxMetadata_from_headers(): 'properties': [ {'name': 'notice', 'value': 'some notice'}, {'name': 'errors', 'value': ['some error']}, + {'name': 'warnings', 'value': ['some warning']}, {'name': 'message', 'value': 'some message'}, - {'name': 'WARNING', 'value': 'some warning'}, {'name': 'spdx_version', 'value': '3.1.2'}, ], 'tools': [ @@ -216,6 +217,14 @@ def test_CycloneDxMetadata_from_headers(): assert m == expected +def test_cyclonedx_plugin_does_not_fail_without_packages(): + test_dir = test_env.get_test_loc('cyclonedx/simple') + result_file = test_env.get_temp_file('cyclonedx.json') + run_scan_click([test_dir, '--cyclonedx', result_file]) + expected_file = test_env.get_test_loc('cyclonedx/expected-without-packages.json') + check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES) + + def test_cyclonedx_plugin_json(): test_dir = test_env.get_test_loc('cyclonedx/simple') result_file = test_env.get_temp_file('cyclonedx.json')