Merge pull request #3005 from nexB/2987-cyclonedx

Do not fail without packages in cyclonedx #2987

Author: JonoYang

src/formattedcode/output_cyclonedx.py

@@ -8,12 +8,15 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import os
 import json
+import logging
 import uuid
 from collections import defaultdict
 from datetime import datetime
 from enum import Enum
 from typing import List
+import warnings
 
 import attr
 from lxml import etree
@@ -26,6 +29,25 @@
 from plugincode.output import output_impl
 
 
+TRACE = os.environ.get('SCANCODE_DEBUG_OUTPUTS', False)
+
+
+def logger_debug(*args):
+    pass
+
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
+
+
 class ToDictMixin:
 
     def to_dict(self):
@@ -274,7 +296,7 @@ def from_package(cls, package):
 
         purl = package.get('purl')
 
-        return CycloneDxComponent(
+        return cls(
             bom_ref=purl,
             purl=purl,
             name=name,
@@ -301,7 +323,7 @@ def from_packages(cls, packages):
         """
         components_by_purl = defaultdict(list)
         for package in packages:
-            comp = CycloneDxComponent.from_package(package)
+            comp = cls.from_package(package)
             if not comp:
                 continue
             components_by_purl[comp.purl].append(comp)
@@ -497,7 +519,7 @@ def from_package(cls, package, components_by_purl):
                     warnings_by_dependent[purl].append(msg)
 
         for ref, dependsOn in dependencies_by_dependent.items():
-            yield CycloneDxDependency(
+            yield cls(
                 ref=ref,
                 dependsOn=dependsOn,
                 warnings=warnings_by_dependent.get(purl, [])
@@ -556,6 +578,11 @@ def from_headers(cls, headers):
         headers = [h for h in headers if h.get('tool_name') == 'scancode-toolkit']
         scancode_header = headers[0] if headers else {}
 
+        if TRACE:
+            logger_debug('CycloneDxMetadata: headers')
+            from pprint import pformat
+            logger_debug(pformat(headers))
+
         try:
             tool_header = {
                 'vendor': 'AboutCode.org',
@@ -568,11 +595,17 @@ def from_headers(cls, headers):
         props = dict(
             notice=scancode_header.get('notice'),
             errors=scancode_header.get('errors', []),
+            warnings=scancode_header.get('warnings', []),
             message=scancode_header.get('message'),
         )
         props.update(scancode_header.get('extra_data', {}))
         properties = [CycloneDxProperty(k, v) for k, v in props.items()]
 
+        if TRACE:
+            logger_debug('CycloneDxMetadata: properties')
+            from pprint import pformat
+            logger_debug(pformat(properties))
+
         return CycloneDxMetadata(
             tools=[tool_header],
             properties=properties,
@@ -596,6 +629,10 @@ def to_xml_element(self):
         return xmetadata
 
 
+class CycloneDxPluginNoPackagesWarning(DeprecationWarning):
+    pass
+
+
 @attr.s
 class CycloneDxBom:
     """
@@ -629,10 +666,32 @@ def from_codebase(cls, codebase):
         """
         Return a CycloneDxBom built from a ScanCode ``codebase``.
         """
-        metadata = CycloneDxMetadata.from_headers(codebase.get_headers())
-        packages = codebase.attributes.packages
-        components = list(CycloneDxComponent.from_packages(packages))
-        dependencies = list(CycloneDxDependency.from_packages(packages, components))
+        components = []
+        dependencies = []
+
+        packages_not_found_message = (
+            "The --cyclonedx-xml option will not output any component/dependency data "
+            "as there are no package data in the present scan. To get package data "
+            "please rerun the scan with --package or --system-package CLI options enabled."
+        )
+        codebase.get_or_create_current_header()
+
+        if hasattr(codebase.attributes, 'packages'):
+            packages = codebase.attributes.packages
+            components = list(CycloneDxComponent.from_packages(packages))
+            dependencies = list(CycloneDxDependency.from_packages(packages, components))
+        else:
+            warnings.simplefilter('always', CycloneDxPluginNoPackagesWarning)
+            warnings.warn(
+                packages_not_found_message,
+                CycloneDxPluginNoPackagesWarning,
+                stacklevel=2,
+            )
+            headers = codebase.get_or_create_current_header()
+            headers.warnings.append(packages_not_found_message)
+
+        codebase_headers = codebase.get_headers()
+        metadata = CycloneDxMetadata.from_headers(codebase_headers)
 
         return CycloneDxBom(
             metadata=metadata,

tests/formattedcode/data/cyclonedx/expected-without-packages.json

@@ -0,0 +1,7 @@
+{
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.3",
+  "version": 1,
+  "components": [],
+  "dependencies": []
+}

tests/formattedcode/test_output_cyclonedx.py

@@ -197,16 +197,17 @@ def test_CycloneDxMetadata_from_headers():
         'notice': 'some notice',
         'message': 'some message',
         'errors': ['some error'],
-        'extra_data': {'WARNING': 'some warning', 'spdx_version': '3.1.2'}
+        'warnings': ['some warning'],
+        'extra_data': {'spdx_version': '3.1.2'}
     }]
     m = CycloneDxMetadata.from_headers(headers).to_dict()
     m.pop('timestamp')
     expected = {
         'properties': [
             {'name': 'notice', 'value': 'some notice'},
             {'name': 'errors', 'value': ['some error']},
+            {'name': 'warnings', 'value': ['some warning']},
             {'name': 'message', 'value': 'some message'},
-            {'name': 'WARNING', 'value': 'some warning'},
             {'name': 'spdx_version', 'value': '3.1.2'},
         ],
         'tools': [
@@ -216,6 +217,14 @@ def test_CycloneDxMetadata_from_headers():
     assert m == expected
 
 
+def test_cyclonedx_plugin_does_not_fail_without_packages():
+    test_dir = test_env.get_test_loc('cyclonedx/simple')
+    result_file = test_env.get_temp_file('cyclonedx.json')
+    run_scan_click([test_dir, '--cyclonedx', result_file])
+    expected_file = test_env.get_test_loc('cyclonedx/expected-without-packages.json')
+    check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
+
+
 def test_cyclonedx_plugin_json():
     test_dir = test_env.get_test_loc('cyclonedx/simple')
     result_file = test_env.get_temp_file('cyclonedx.json')