diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d78ae6ffb0..6d78d4c4b6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,9 @@ v33.0.0 (unreleased) once. https://github.com/nexB/scancode.io/issues/646 +- Move the ``cyclonedx`` and ``spdx`` root modules into the ``pipes`` module. + https://github.com/nexB/scancode.io/issues/657 + - Remove the admin app and views. https://github.com/nexB/scancode.io/issues/645 diff --git a/scanpipe/models.py b/scanpipe/models.py index b48ba614b1..2e7191b7e5 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -78,7 +78,6 @@ from rq.job import JobStatus from scancodeio import __version__ as scancodeio_version -from scanpipe import spdx from scanpipe import tasks logger = logging.getLogger(__name__) @@ -1873,6 +1872,8 @@ def get_spdx_types(self): def as_spdx(self): """Return this CodebaseResource as an SPDX Package entry.""" + from scanpipe.pipes import spdx + spdx_license_keys = [license["spdx_license_key"] for license in self.licenses] copyrights = [copyright["copyright"] for copyright in self.copyrights] holders = [holder["holder"] for holder in self.holders] @@ -2200,6 +2201,8 @@ def get_license_expression_spdx_id(self): def as_spdx(self): """Return this DiscoveredPackage as an SPDX Package entry.""" + from scanpipe.pipes import spdx + checksums = [ spdx.Checksum(algorithm=algorithm, value=checksum_value) for algorithm in ["sha1", "md5"] @@ -2496,6 +2499,8 @@ def spdx_id(self): def as_spdx(self): """Return this Package as an SPDX Package entry.""" + from scanpipe.pipes import spdx + external_refs = [] if package_url := self.package_url: diff --git a/scanpipe/cyclonedx/__init__.py b/scanpipe/pipes/cyclonedx.py similarity index 89% rename from scanpipe/cyclonedx/__init__.py rename to scanpipe/pipes/cyclonedx.py index f238bf31f8..4a35ee301f 100644 --- a/scanpipe/cyclonedx/__init__.py +++ b/scanpipe/pipes/cyclonedx.py @@ -31,21 +31,21 @@ CyclonedxSoftwareBillOfMaterialsStandard as Bom_1_4, ) +SCHEMAS_PATH = pathlib.Path(__file__).parent / "schemas" + CYCLONEDX_SPEC_VERSION = "1.4" -CYCLONEDX_JSON_SCHEMA_LOCATION = "bom-1.4.schema.json" -CYCLONEDX_JSON_SCHEMA_PATH = ( - pathlib.Path(__file__).parent / CYCLONEDX_JSON_SCHEMA_LOCATION -) -CYCLONEDX_JSON_SCHEMA_URL = ( +CYCLONEDX_SCHEMA_NAME = "bom-1.4.schema.json" +CYCLONEDX_SCHEMA_PATH = SCHEMAS_PATH / CYCLONEDX_SCHEMA_NAME +CYCLONEDX_SCHEMA_URL = ( "https://raw.githubusercontent.com/" "CycloneDX/specification/master/schema/bom-1.4.schema.json" ) -SPDX_JSON_SCHEMA_LOCATION = "spdx.schema.json" -SPDX_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / SPDX_JSON_SCHEMA_LOCATION +SPDX_SCHEMA_NAME = "spdx.schema.json" +SPDX_SCHEMA_PATH = SCHEMAS_PATH / SPDX_SCHEMA_NAME -JSF_JSON_SCHEMA_LOCATION = "jsf-0.82.schema.json" -JSF_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / JSF_JSON_SCHEMA_LOCATION +JSF_SCHEMA_NAME = "jsf-0.82.schema.json" +JSF_SCHEMA_PATH = SCHEMAS_PATH / JSF_SCHEMA_NAME def get_bom(cyclonedx_document): @@ -152,7 +152,7 @@ def get_properties_data(component): return properties_data -def validate_document(document, schema=CYCLONEDX_JSON_SCHEMA_PATH): +def validate_document(document, schema=CYCLONEDX_SCHEMA_PATH): """Check the validity of this CycloneDX document.""" if isinstance(document, str): document = json.loads(document) @@ -163,8 +163,8 @@ def validate_document(document, schema=CYCLONEDX_JSON_SCHEMA_PATH): if isinstance(schema, str): schema = json.loads(schema) - spdx_schema = SPDX_JSON_SCHEMA_PATH.read_text() - jsf_schema = JSF_JSON_SCHEMA_PATH.read_text() + spdx_schema = SPDX_SCHEMA_PATH.read_text() + jsf_schema = JSF_SCHEMA_PATH.read_text() store = { "http://cyclonedx.org/schema/spdx.schema.json": json.loads(spdx_schema), diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 8ff7455663..209721ccac 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -40,7 +40,7 @@ from scancodeio import SCAN_NOTICE from scancodeio import __version__ as scancodeio_version -from scanpipe import spdx +from scanpipe.pipes import spdx scanpipe_app = apps.get_app_config("scanpipe") diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 130ad476a2..7d0a557022 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -33,9 +33,9 @@ from packageurl import PackageURL from python_inspector.resolve_cli import resolver_api -from scanpipe import cyclonedx -from scanpipe import spdx from scanpipe.models import DiscoveredPackage +from scanpipe.pipes import cyclonedx +from scanpipe.pipes import spdx """ Resolve packages from manifest, lockfile, and SBOM. diff --git a/scanpipe/cyclonedx/bom-1.4.schema.json b/scanpipe/pipes/schemas/bom-1.4.schema.json similarity index 100% rename from scanpipe/cyclonedx/bom-1.4.schema.json rename to scanpipe/pipes/schemas/bom-1.4.schema.json diff --git a/scanpipe/cyclonedx/jsf-0.82.schema.json b/scanpipe/pipes/schemas/jsf-0.82.schema.json similarity index 100% rename from scanpipe/cyclonedx/jsf-0.82.schema.json rename to scanpipe/pipes/schemas/jsf-0.82.schema.json diff --git a/scanpipe/spdx/spdx-schema-2.3.json b/scanpipe/pipes/schemas/spdx-schema-2.3.json similarity index 100% rename from scanpipe/spdx/spdx-schema-2.3.json rename to scanpipe/pipes/schemas/spdx-schema-2.3.json diff --git a/scanpipe/cyclonedx/spdx.schema.json b/scanpipe/pipes/schemas/spdx.schema.json similarity index 100% rename from scanpipe/cyclonedx/spdx.schema.json rename to scanpipe/pipes/schemas/spdx.schema.json diff --git a/scanpipe/spdx/__init__.py b/scanpipe/pipes/spdx.py similarity index 98% rename from scanpipe/spdx/__init__.py rename to scanpipe/pipes/spdx.py index f02346761c..82decbb8e3 100644 --- a/scanpipe/spdx/__init__.py +++ b/scanpipe/pipes/spdx.py @@ -30,9 +30,9 @@ SPDX_SPEC_VERSION = "2.3" SPDX_LICENSE_LIST_VERSION = "3.18" -SPDX_JSON_SCHEMA_LOCATION = "spdx-schema-2.3.json" -SPDX_JSON_SCHEMA_PATH = pathlib.Path(__file__).parent / SPDX_JSON_SCHEMA_LOCATION -SPDX_JSON_SCHEMA_URL = ( +SPDX_SCHEMA_NAME = "spdx-schema-2.3.json" +SPDX_SCHEMA_PATH = pathlib.Path(__file__).parent / "schemas" / SPDX_SCHEMA_NAME +SPDX_SCHEMA_URL = ( "https://raw.githubusercontent.com/spdx/spdx-spec/v2.3/schemas/spdx-schema.json" ) @@ -612,7 +612,7 @@ def validate(self, schema): return validate_document(document=self.as_dict(), schema=schema) -def validate_document(document, schema=SPDX_JSON_SCHEMA_PATH): +def validate_document(document, schema=SPDX_SCHEMA_PATH): """ SPDX document validation. Requires the `jsonschema` library. diff --git a/scanpipe/tests/test_cyclonedx.py b/scanpipe/tests/pipes/test_cyclonedx.py similarity index 97% rename from scanpipe/tests/test_cyclonedx.py rename to scanpipe/tests/pipes/test_cyclonedx.py index f97c0b873b..3004185f39 100644 --- a/scanpipe/tests/test_cyclonedx.py +++ b/scanpipe/tests/pipes/test_cyclonedx.py @@ -33,11 +33,11 @@ from hoppr_cyclonedx_models.cyclonedx_1_4 import LicenseChoice from hoppr_cyclonedx_models.cyclonedx_1_4 import LicenseChoice1 -from scanpipe import cyclonedx +from scanpipe.pipes import cyclonedx -class CycloneDXUnitTest(TestCase): - bom_file = Path(__file__).parent / "data/cyclonedx/nested.bom.json" +class ScanPipeCycloneDXPipesTest(TestCase): + bom_file = Path(__file__).parent.parent / "data/cyclonedx/nested.bom.json" bom_json = bom_file.read_text() bom_parsed = json.loads(bom_json) bom = Bom_1_4(**bom_parsed) diff --git a/scanpipe/spdx/test_spdx.py b/scanpipe/tests/pipes/test_spdx.py similarity index 98% rename from scanpipe/spdx/test_spdx.py rename to scanpipe/tests/pipes/test_spdx.py index b8312c092a..6610479e69 100644 --- a/scanpipe/spdx/test_spdx.py +++ b/scanpipe/tests/pipes/test_spdx.py @@ -20,17 +20,14 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -import pathlib from unittest import TestCase -from scanpipe import spdx +from scanpipe.pipes import spdx -class SPDXTestCase(TestCase): +class ScanPipeSPDXPipesTest(TestCase): def setUp(self): - spdx_module_location = pathlib.Path(spdx.__file__).parent - spdx_schema_location = spdx_module_location / spdx.SPDX_JSON_SCHEMA_LOCATION - self.schema = spdx_schema_location.read_text() + self.schema = spdx.SPDX_SCHEMA_PATH.read_text() self.creation_info_data = { "person_name": "John Doe",