From 052c441cc162848d313439c0860afc43370a0bb5 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Tue, 28 Mar 2023 13:07:29 -0100 Subject: [PATCH 1/3] Move the spdx module into the pipes Signed-off-by: Thomas Druez --- scanpipe/models.py | 7 ++++++- scanpipe/pipes/output.py | 2 +- scanpipe/pipes/resolve.py | 2 +- scanpipe/{spdx => pipes/schemas}/spdx-schema-2.3.json | 0 scanpipe/{spdx/__init__.py => pipes/spdx.py} | 8 ++++---- scanpipe/{spdx => tests/pipes}/test_spdx.py | 7 ++----- 6 files changed, 14 insertions(+), 12 deletions(-) rename scanpipe/{spdx => pipes/schemas}/spdx-schema-2.3.json (100%) rename scanpipe/{spdx/__init__.py => pipes/spdx.py} (98%) rename scanpipe/{spdx => tests/pipes}/test_spdx.py (98%) diff --git a/scanpipe/models.py b/scanpipe/models.py index b48ba614b1..2e7191b7e5 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -78,7 +78,6 @@ from rq.job import JobStatus from scancodeio import __version__ as scancodeio_version -from scanpipe import spdx from scanpipe import tasks logger = logging.getLogger(__name__) @@ -1873,6 +1872,8 @@ def get_spdx_types(self): def as_spdx(self): """Return this CodebaseResource as an SPDX Package entry.""" + from scanpipe.pipes import spdx + spdx_license_keys = [license["spdx_license_key"] for license in self.licenses] copyrights = [copyright["copyright"] for copyright in self.copyrights] holders = [holder["holder"] for holder in self.holders] @@ -2200,6 +2201,8 @@ def get_license_expression_spdx_id(self): def as_spdx(self): """Return this DiscoveredPackage as an SPDX Package entry.""" + from scanpipe.pipes import spdx + checksums = [ spdx.Checksum(algorithm=algorithm, value=checksum_value) for algorithm in ["sha1", "md5"] @@ -2496,6 +2499,8 @@ def spdx_id(self): def as_spdx(self): """Return this Package as an SPDX Package entry.""" + from scanpipe.pipes import spdx + external_refs = [] if package_url := self.package_url: diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 8ff7455663..209721ccac 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -40,7 +40,7 @@ from scancodeio import SCAN_NOTICE from scancodeio import __version__ as scancodeio_version -from scanpipe import spdx +from scanpipe.pipes import spdx scanpipe_app = apps.get_app_config("scanpipe") diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 130ad476a2..4b387d6113 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -34,8 +34,8 @@ from python_inspector.resolve_cli import resolver_api from scanpipe import cyclonedx -from scanpipe import spdx from scanpipe.models import DiscoveredPackage +from scanpipe.pipes import spdx """ Resolve packages from manifest, lockfile, and SBOM. diff --git a/scanpipe/spdx/spdx-schema-2.3.json b/scanpipe/pipes/schemas/spdx-schema-2.3.json similarity index 100% rename from scanpipe/spdx/spdx-schema-2.3.json rename to scanpipe/pipes/schemas/spdx-schema-2.3.json diff --git a/scanpipe/spdx/__init__.py b/scanpipe/pipes/spdx.py similarity index 98% rename from scanpipe/spdx/__init__.py rename to scanpipe/pipes/spdx.py index f02346761c..82decbb8e3 100644 --- a/scanpipe/spdx/__init__.py +++ b/scanpipe/pipes/spdx.py @@ -30,9 +30,9 @@ SPDX_SPEC_VERSION = "2.3" SPDX_LICENSE_LIST_VERSION = "3.18" -SPDX_JSON_SCHEMA_LOCATION = "spdx-schema-2.3.json" -SPDX_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / SPDX_JSON_SCHEMA_LOCATION -SPDX_JSON_SCHEMA_URL = ( +SPDX_SCHEMA_NAME = "spdx-schema-2.3.json" +SPDX_SCHEMA_PATH = pathlib.Path(__file__).parent / "schemas" / SPDX_SCHEMA_NAME +SPDX_SCHEMA_URL = ( "https://raw.githubusercontent.com/spdx/spdx-spec/v2.3/schemas/spdx-schema.json" ) @@ -612,7 +612,7 @@ def validate(self, schema): return validate_document(document=self.as_dict(), schema=schema) -def validate_document(document, schema=SPDX_JSON_SCHEMA_PATH): +def validate_document(document, schema=SPDX_SCHEMA_PATH): """ SPDX document validation. Requires the `jsonschema` library. diff --git a/scanpipe/spdx/test_spdx.py b/scanpipe/tests/pipes/test_spdx.py similarity index 98% rename from scanpipe/spdx/test_spdx.py rename to scanpipe/tests/pipes/test_spdx.py index b8312c092a..08fc4e8077 100644 --- a/scanpipe/spdx/test_spdx.py +++ b/scanpipe/tests/pipes/test_spdx.py @@ -20,17 +20,14 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -import pathlib from unittest import TestCase -from scanpipe import spdx +from scanpipe.pipes import spdx class SPDXTestCase(TestCase): def setUp(self): - spdx_module_location = pathlib.Path(spdx.__file__).parent - spdx_schema_location = spdx_module_location / spdx.SPDX_JSON_SCHEMA_LOCATION - self.schema = spdx_schema_location.read_text() + self.schema = spdx.SPDX_SCHEMA_PATH.read_text() self.creation_info_data = { "person_name": "John Doe", From 04e2630108f42719bcf7c43c302d106148665db1 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Tue, 28 Mar 2023 13:17:21 -0100 Subject: [PATCH 2/3] Move the cyclonedx module into the pipes Signed-off-by: Thomas Druez --- .../__init__.py => pipes/cyclonedx.py} | 24 +++++++++---------- scanpipe/pipes/resolve.py | 2 +- .../schemas}/bom-1.4.schema.json | 0 .../schemas}/jsf-0.82.schema.json | 0 .../schemas}/spdx.schema.json | 0 scanpipe/tests/{ => pipes}/test_cyclonedx.py | 6 ++--- scanpipe/tests/pipes/test_spdx.py | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) rename scanpipe/{cyclonedx/__init__.py => pipes/cyclonedx.py} (89%) rename scanpipe/{cyclonedx => pipes/schemas}/bom-1.4.schema.json (100%) rename scanpipe/{cyclonedx => pipes/schemas}/jsf-0.82.schema.json (100%) rename scanpipe/{cyclonedx => pipes/schemas}/spdx.schema.json (100%) rename scanpipe/tests/{ => pipes}/test_cyclonedx.py (97%) diff --git a/scanpipe/cyclonedx/__init__.py b/scanpipe/pipes/cyclonedx.py similarity index 89% rename from scanpipe/cyclonedx/__init__.py rename to scanpipe/pipes/cyclonedx.py index f238bf31f8..4a35ee301f 100644 --- a/scanpipe/cyclonedx/__init__.py +++ b/scanpipe/pipes/cyclonedx.py @@ -31,21 +31,21 @@ CyclonedxSoftwareBillOfMaterialsStandard as Bom_1_4, ) +SCHEMAS_PATH = pathlib.Path(__file__).parent / "schemas" + CYCLONEDX_SPEC_VERSION = "1.4" -CYCLONEDX_JSON_SCHEMA_LOCATION = "bom-1.4.schema.json" -CYCLONEDX_JSON_SCHEMA_PATH = ( - pathlib.Path(__file__).parent / CYCLONEDX_JSON_SCHEMA_LOCATION -) -CYCLONEDX_JSON_SCHEMA_URL = ( +CYCLONEDX_SCHEMA_NAME = "bom-1.4.schema.json" +CYCLONEDX_SCHEMA_PATH = SCHEMAS_PATH / CYCLONEDX_SCHEMA_NAME +CYCLONEDX_SCHEMA_URL = ( "https://raw.githubusercontent.com/" "CycloneDX/specification/master/schema/bom-1.4.schema.json" ) -SPDX_JSON_SCHEMA_LOCATION = "spdx.schema.json" -SPDX_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / SPDX_JSON_SCHEMA_LOCATION +SPDX_SCHEMA_NAME = "spdx.schema.json" +SPDX_SCHEMA_PATH = SCHEMAS_PATH / SPDX_SCHEMA_NAME -JSF_JSON_SCHEMA_LOCATION = "jsf-0.82.schema.json" -JSF_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / JSF_JSON_SCHEMA_LOCATION +JSF_SCHEMA_NAME = "jsf-0.82.schema.json" +JSF_SCHEMA_PATH = SCHEMAS_PATH / JSF_SCHEMA_NAME def get_bom(cyclonedx_document): @@ -152,7 +152,7 @@ def get_properties_data(component): return properties_data -def validate_document(document, schema=CYCLONEDX_JSON_SCHEMA_PATH): +def validate_document(document, schema=CYCLONEDX_SCHEMA_PATH): """Check the validity of this CycloneDX document.""" if isinstance(document, str): document = json.loads(document) @@ -163,8 +163,8 @@ def validate_document(document, schema=CYCLONEDX_JSON_SCHEMA_PATH): if isinstance(schema, str): schema = json.loads(schema) - spdx_schema = SPDX_JSON_SCHEMA_PATH.read_text() - jsf_schema = JSF_JSON_SCHEMA_PATH.read_text() + spdx_schema = SPDX_SCHEMA_PATH.read_text() + jsf_schema = JSF_SCHEMA_PATH.read_text() store = { "http://cyclonedx.org/schema/spdx.schema.json": json.loads(spdx_schema), diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 4b387d6113..7d0a557022 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -33,8 +33,8 @@ from packageurl import PackageURL from python_inspector.resolve_cli import resolver_api -from scanpipe import cyclonedx from scanpipe.models import DiscoveredPackage +from scanpipe.pipes import cyclonedx from scanpipe.pipes import spdx """ diff --git a/scanpipe/cyclonedx/bom-1.4.schema.json b/scanpipe/pipes/schemas/bom-1.4.schema.json similarity index 100% rename from scanpipe/cyclonedx/bom-1.4.schema.json rename to scanpipe/pipes/schemas/bom-1.4.schema.json diff --git a/scanpipe/cyclonedx/jsf-0.82.schema.json b/scanpipe/pipes/schemas/jsf-0.82.schema.json similarity index 100% rename from scanpipe/cyclonedx/jsf-0.82.schema.json rename to scanpipe/pipes/schemas/jsf-0.82.schema.json diff --git a/scanpipe/cyclonedx/spdx.schema.json b/scanpipe/pipes/schemas/spdx.schema.json similarity index 100% rename from scanpipe/cyclonedx/spdx.schema.json rename to scanpipe/pipes/schemas/spdx.schema.json diff --git a/scanpipe/tests/test_cyclonedx.py b/scanpipe/tests/pipes/test_cyclonedx.py similarity index 97% rename from scanpipe/tests/test_cyclonedx.py rename to scanpipe/tests/pipes/test_cyclonedx.py index f97c0b873b..3004185f39 100644 --- a/scanpipe/tests/test_cyclonedx.py +++ b/scanpipe/tests/pipes/test_cyclonedx.py @@ -33,11 +33,11 @@ from hoppr_cyclonedx_models.cyclonedx_1_4 import LicenseChoice from hoppr_cyclonedx_models.cyclonedx_1_4 import LicenseChoice1 -from scanpipe import cyclonedx +from scanpipe.pipes import cyclonedx -class CycloneDXUnitTest(TestCase): - bom_file = Path(__file__).parent / "data/cyclonedx/nested.bom.json" +class ScanPipeCycloneDXPipesTest(TestCase): + bom_file = Path(__file__).parent.parent / "data/cyclonedx/nested.bom.json" bom_json = bom_file.read_text() bom_parsed = json.loads(bom_json) bom = Bom_1_4(**bom_parsed) diff --git a/scanpipe/tests/pipes/test_spdx.py b/scanpipe/tests/pipes/test_spdx.py index 08fc4e8077..6610479e69 100644 --- a/scanpipe/tests/pipes/test_spdx.py +++ b/scanpipe/tests/pipes/test_spdx.py @@ -25,7 +25,7 @@ from scanpipe.pipes import spdx -class SPDXTestCase(TestCase): +class ScanPipeSPDXPipesTest(TestCase): def setUp(self): self.schema = spdx.SPDX_SCHEMA_PATH.read_text() From db5eb3f420994e6d60b2f866164e680e0dbf9e68 Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Tue, 28 Mar 2023 13:18:36 -0100 Subject: [PATCH 3/3] Add changelog entry Signed-off-by: Thomas Druez --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d78ae6ffb0..6d78d4c4b6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,9 @@ v33.0.0 (unreleased) once. https://github.com/nexB/scancode.io/issues/646 +- Move the ``cyclonedx`` and ``spdx`` root modules into the ``pipes`` module. + https://github.com/nexB/scancode.io/issues/657 + - Remove the admin app and views. https://github.com/nexB/scancode.io/issues/645