diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b3d60a067..cdd732411a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,6 +23,9 @@ v34.1.0 (unreleased) https://github.com/nexB/scancode.io/issues/1121 https://github.com/nexB/scancode.io/issues/1122 +- Rename the ``match_to_purldb`` pipeline to ``match_to_matchcode``, and add + MatchCode.io API settings to ScanCode.io settings. + v34.0.0 (2024-03-04) -------------------- diff --git a/docs/application-settings.rst b/docs/application-settings.rst index 373c5d628c..0e3d08f188 100644 --- a/docs/application-settings.rst +++ b/docs/application-settings.rst @@ -314,6 +314,26 @@ you can provide the API key using ``VULNERABLECODE_API_KEY``:: VULNERABLECODE_API_KEY=insert_your_api_key_here +.. _scancodeio_settings_matchcodeio: + +MATCHCODE.IO +^^^^^^^^^^^^ + +There is currently no public instance of MatchCode.io. + +Alternatively, you can deploy your own instance of MatchCode.io by +following the instructions provided in the documentation at +https://purldb.readthedocs.io/. + +To configure your local environment, set the ``MATCHCODEIO_URL`` in your ``.env`` file:: + + MATCHCODEIO_URL=https://
/ + +If authentication is enabled on your MatchCode.io instance, you can provide the +API key using ``MATCHCODEIO_API_KEY``:: + + MATCHCODEIO_API_KEY=insert_your_api_key_here + .. _scancodeio_settings_fetch_authentication: Fetch Authentication diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 2ce946bfb0..ac47cdadef 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -122,17 +122,17 @@ Map Deploy To Develop :members: :member-order: bysource -.. _pipeline_match_to_purldb: +.. _pipeline_match_to_matchcode: -Match to PurlDB (addon) ------------------------ +Match to MatchCode (addon) +-------------------------- .. warning:: - This pipeline requires access to a PurlDB service. - Refer to :ref:`scancodeio_settings_purldb` to configure access to PurlDB in your - ScanCode.io instance. + This pipeline requires access to a MatchCode.io service. + Refer to :ref:`scancodeio_settings_matchcodeio` to configure access to + MatchCode.io in your ScanCode.io instance. -.. autoclass:: scanpipe.pipelines.match_to_purldb.MatchToPurlDB() +.. autoclass:: scanpipe.pipelines.match_to_matchcode.MatchToMatchCode() :members: :member-order: bysource diff --git a/docs/faq.rst b/docs/faq.rst index 045c905ad8..659a4c9d23 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -70,10 +70,10 @@ existing data, allowing for more comprehensive analysis and insights. Before executing this pipeline, make sure to set up :ref:`PurlDB `. -- To **match your project codebase resources to PurlDB for Package matches**, - utilize the :ref:`match_to_purldb ` pipeline. - It's essential to set up :ref:`PurlDB ` before executing - this pipeline. +- To **match your project codebase resources to MatchCode.io for Package matches**, + utilize the :ref:`match_to_matchcode ` pipeline. + It's essential to set up :ref:`MatchCode.io ` before + executing this pipeline. What is the difference between scan_codebase and scan_single_package pipelines? ------------------------------------------------------------------------------- diff --git a/scancodeio/settings.py b/scancodeio/settings.py index d3ee176d3c..112de2deff 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -403,3 +403,10 @@ PURLDB_USER = env.str("PURLDB_USER", default="") PURLDB_PASSWORD = env.str("PURLDB_PASSWORD", default="") PURLDB_API_KEY = env.str("PURLDB_API_KEY", default="") + +# MatchCode.io integration + +MATCHCODEIO_URL = env.str("MATCHCODEIO_URL", default="") +MATCHCODEIO_USER = env.str("MATCHCODEIO_USER", default="") +MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="") +MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="") diff --git a/scanpipe/migrations/0054_rename_pipeline.py b/scanpipe/migrations/0054_rename_pipeline.py new file mode 100644 index 0000000000..f15e2a4613 --- /dev/null +++ b/scanpipe/migrations/0054_rename_pipeline.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.3 on 2024-03-20 22:52 + +from django.db import migrations + + +pipeline_old_names_mapping = { + "match_to_purldb": "match_to_matchcode", +} + + +def rename_pipelines_data(apps, schema_editor): + Run = apps.get_model("scanpipe", "Run") + for old_name, new_name in pipeline_old_names_mapping.items(): + Run.objects.filter(pipeline_name=old_name).update(pipeline_name=new_name) + + +def reverse_rename_pipelines_data(apps, schema_editor): + Run = apps.get_model("scanpipe", "Run") + for old_name, new_name in pipeline_old_names_mapping.items(): + Run.objects.filter(pipeline_name=new_name).update(pipeline_name=old_name) + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0053_restructure_pipelines_data"), + ] + + operations = [ + migrations.RunPython( + rename_pipelines_data, + reverse_code=reverse_rename_pipelines_data, + ), + ] diff --git a/scanpipe/pipelines/match_to_purldb.py b/scanpipe/pipelines/match_to_matchcode.py similarity index 53% rename from scanpipe/pipelines/match_to_purldb.py rename to scanpipe/pipelines/match_to_matchcode.py index 7f9904a7ad..10c85b7584 100644 --- a/scanpipe/pipelines/match_to_purldb.py +++ b/scanpipe/pipelines/match_to_matchcode.py @@ -21,19 +21,26 @@ # Visit https://github.com/nexB/scancode.io for support and download. from scanpipe.pipelines import Pipeline -from scanpipe.pipes import purldb +from scanpipe.pipes import matchcode -class MatchToPurlDB(Pipeline): +class MatchToMatchCode(Pipeline): """ - Match the codebase resources of a project against PurlDB to identify packages. + Match the codebase resources of a project against MatchCode.io to identify packages. This process involves: - 1. generating a JSON scan of the project codebase - 2. transmitting it to MatchCode on PurlDB and awaiting match results - 3. creating discovered packages from the package data obtained - 4. associating the codebase resources with those discovered packages + 1. Generating a JSON scan of the project codebase + 2. Transmitting it to MatchCode.io and awaiting match results + 3. Creating discovered packages from the package data obtained + 4. Associating the codebase resources with those discovered packages + + Currently, MatchCode.io can only match for archives, directories, and files + from Maven and npm Packages. + + This pipeline requires a MatchCode.io instance to be configured and available. + There is currently no public instance of MatchCode.io. Reach out to nexB, Inc. + for other arrangements. """ download_inputs = False @@ -42,29 +49,34 @@ class MatchToPurlDB(Pipeline): @classmethod def steps(cls): return ( - cls.check_purldb_service_availability, + cls.check_matchcode_service_availability, cls.send_project_json_to_matchcode, cls.poll_matching_results, cls.create_packages_from_match_results, ) - def check_purldb_service_availability(self): - """Check if the PurlDB service if configured and available.""" - if not purldb.is_configured(): - raise Exception("PurlDB is not configured.") + def check_matchcode_service_availability(self): + """Check if the MatchCode.io service if configured and available.""" + if not matchcode.is_configured(): + msg = ( + "MatchCode.io is not configured. Set the MatchCode.io " + "related settings to a MatchCode.io instance or reach out " + "to the maintainers for other arrangements." + ) + raise Exception(msg) - if not purldb.is_available(): - raise Exception("PurlDB is not available.") + if not matchcode.is_available(): + raise Exception("MatchCode.io is not available.") def send_project_json_to_matchcode(self): - """Create a JSON scan of the project Codebase and send it to MatchCode.""" - self.run_url = purldb.send_project_json_to_matchcode(self.project) + """Create a JSON scan of the project Codebase and send it to MatchCode.io.""" + self.run_url = matchcode.send_project_json_to_matchcode(self.project) def poll_matching_results(self): """Wait until the match results are ready by polling the match run status.""" - purldb.poll_until_success(self.run_url) + matchcode.poll_until_success(self.run_url) def create_packages_from_match_results(self): """Create DiscoveredPackages from match results.""" - match_results = purldb.get_match_results(self.run_url) - purldb.create_packages_from_match_results(self.project, match_results) + match_results = matchcode.get_match_results(self.run_url) + matchcode.create_packages_from_match_results(self.project, match_results) diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index a56501d919..1e26a08602 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -21,12 +21,100 @@ # Visit https://github.com/nexB/scancode.io for support and download. import logging +import time +from collections import defaultdict +from django.conf import settings + +import requests from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints +from scanpipe.models import AbstractTaskFieldsModel from scanpipe.pipes import codebase +from scanpipe.pipes import flag +from scanpipe.pipes.output import to_json + + +class MatchCodeIOException(Exception): + pass + +label = "MatchCode" logger = logging.getLogger(__name__) +session = requests.Session() + +# Only MATCHCODEIO_URL can be provided through setting +MATCHCODEIO_API_URL = None +MATCHCODEIO_URL = settings.MATCHCODEIO_URL +if MATCHCODEIO_URL: + MATCHCODEIO_API_URL = f'{MATCHCODEIO_URL.rstrip("/")}/api/' + +# Basic Authentication +MATCHCODEIO_USER = settings.MATCHCODEIO_USER +MATCHCODEIO_PASSWORD = settings.MATCHCODEIO_PASSWORD +basic_auth_enabled = MATCHCODEIO_USER and MATCHCODEIO_PASSWORD +if basic_auth_enabled: + session.auth = (MATCHCODEIO_USER, MATCHCODEIO_PASSWORD) + +# Authentication with single API key +MATCHCODEIO_API_KEY = settings.MATCHCODEIO_API_KEY +if MATCHCODEIO_API_KEY: + session.headers.update({"Authorization": f"Token {MATCHCODEIO_API_KEY}"}) + +DEFAULT_TIMEOUT = 60 + + +def is_configured(): + """Return True if the required MatchCode.io settings have been set.""" + if MATCHCODEIO_API_URL: + return True + return False + + +def is_available(): + """Return True if the configured MatchCode.io server is available.""" + if not is_configured(): + return False + + try: + response = session.head(MATCHCODEIO_API_URL) + response.raise_for_status() + except requests.exceptions.RequestException as request_exception: + logger.debug(f"{label} is_available() error: {request_exception}") + return False + + return response.status_code == requests.codes.ok + + +def request_get(url, payload=None, timeout=DEFAULT_TIMEOUT): + """Wrap the HTTP request calls on the API.""" + if not url: + return + + params = {} + if "format=json" not in url: + params.update({"format": "json"}) + if payload: + params.update(payload) + + logger.debug(f"{label}: url={url} params={params}") + try: + response = session.get(url, params=params, timeout=timeout) + response.raise_for_status() + return response.json() + except (requests.RequestException, ValueError, TypeError) as exception: + logger.debug(f"{label} [Exception] {exception}") + + +def request_post(url, data=None, headers=None, files=None, timeout=DEFAULT_TIMEOUT): + try: + response = session.post( + url, data=data, timeout=timeout, headers=headers, files=files + ) + response.raise_for_status() + return response.json() + except (requests.RequestException, ValueError, TypeError) as exception: + logger.debug(f"{label} [Exception] {exception}") def save_directory_fingerprints(project, virtual_codebase, to_codebase_only=False): @@ -100,3 +188,111 @@ def fingerprint_codebase_directories(project, to_codebase_only=False): save_directory_fingerprints( project, virtual_codebase, to_codebase_only=to_codebase_only ) + + +def send_project_json_to_matchcode( + project, timeout=DEFAULT_TIMEOUT, api_url=MATCHCODEIO_API_URL +): + """ + Given a `project`, create a JSON scan of the `project` CodebaseResources and + send it to MatchCode.io for matching. Return a tuple containing strings of the url + to the particular match run and the url to the match results. + """ + scan_output_location = to_json(project) + with open(scan_output_location, "rb") as f: + files = {"upload_file": f} + response = request_post( + url=f"{api_url}matching/", + timeout=timeout, + files=files, + ) + run_url = response["runs"][0]["url"] + return run_url + + +def poll_until_success(run_url, sleep=10): + """ + Given a URL to a scancode.io run instance, `run_url`, return True when the + run instance has completed successfully. + + Raise a MatchCodeIOException when the run instance has failed, stopped, or gone + stale. + """ + run_status = AbstractTaskFieldsModel.Status + while True: + response = request_get(run_url) + if response: + status = response["status"] + if status == run_status.SUCCESS: + return True + + if status in [ + run_status.NOT_STARTED, + run_status.QUEUED, + run_status.RUNNING, + ]: + continue + + if status in [ + run_status.FAILURE, + run_status.STOPPED, + run_status.STALE, + ]: + log = response["log"] + msg = f"Matching run has stopped:\n\n{log}" + raise MatchCodeIOException(msg) + + time.sleep(sleep) + + +def get_match_results(run_url): + """ + Given the `run_url` for a pipeline running the matchcode matching pipeline, + return the match results for that run. + """ + response = request_get(run_url) + project_url = response["project"] + # `project_url` can have params, such as "?format=json" + if "?" in project_url: + project_url, _ = project_url.split("?") + project_url = project_url.rstrip("/") + results_url = project_url + "/results/" + return request_get(results_url) + + +def map_match_results(match_results): + """ + Given `match_results`, which is a mapping of ScanCode.io codebase results, + return a defaultdict(list) where the keys are the package_uid of matched + packages and the value is a list containing the paths of Resources + associated with the package_uid. + """ + resource_results = match_results.get("files", []) + resource_paths_by_package_uids = defaultdict(list) + for resource in resource_results: + for_packages = resource.get("for_packages", []) + for package_uid in for_packages: + resource_paths_by_package_uids[package_uid].append(resource["path"]) + return resource_paths_by_package_uids + + +def create_packages_from_match_results(project, match_results): + """ + Given `match_results`, which is a mapping of ScanCode.io codebase results, + use the Package data from it to create DiscoveredPackages for `project` and + associate the proper Resources of `project` to the DiscoveredPackages. + """ + from scanpipe.pipes.d2d import create_package_from_purldb_data + + resource_paths_by_package_uids = map_match_results(match_results) + matched_packages = match_results.get("packages", []) + for matched_package in matched_packages: + package_uid = matched_package["package_uid"] + resource_paths = resource_paths_by_package_uids[package_uid] + resources = project.codebaseresources.filter(path__in=resource_paths) + create_package_from_purldb_data( + project, + resources=resources, + package_data=matched_package, + status=flag.MATCHED_TO_PURLDB_PACKAGE, + ) diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 1e2335fad6..1b46c7a0e9 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -22,8 +22,6 @@ import json import logging -import time -from collections import defaultdict from django.conf import settings @@ -32,15 +30,7 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import InvalidVersionRange -from scanpipe.models import AbstractTaskFieldsModel from scanpipe.pipes import LoopProgress -from scanpipe.pipes import flag -from scanpipe.pipes.output import to_json - - -class PurlDBException(Exception): - pass - label = "PurlDB" logger = logging.getLogger(__name__) @@ -332,111 +322,3 @@ def populate_purldb_with_discovered_dependencies(project, logger=logger.info): chunk_size=10, logger=logger, ) - - -def send_project_json_to_matchcode( - project, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL -): - """ - Given a `project`, create a JSON scan of the `project` CodebaseResources and - send it to PurlDB for matching. Return a tuple containing strings of the url - to the particular match run and the url to the match results. - """ - scan_output_location = to_json(project) - with open(scan_output_location, "rb") as f: - files = {"upload_file": f} - response = request_post( - url=f"{api_url}matching/", - timeout=timeout, - files=files, - ) - run_url = response["runs"][0]["url"] - return run_url - - -def poll_until_success(run_url, sleep=10): - """ - Given a URL to a scancode.io run instance, `run_url`, return True when the - run instance has completed successfully. - - Raise a PurlDBException when the run instance has faield, stopped, or gone - stale. - """ - run_status = AbstractTaskFieldsModel.Status - while True: - response = request_get(run_url) - if response: - status = response["status"] - if status == run_status.SUCCESS: - return True - - if status in [ - run_status.NOT_STARTED, - run_status.QUEUED, - run_status.RUNNING, - ]: - continue - - if status in [ - run_status.FAILURE, - run_status.STOPPED, - run_status.STALE, - ]: - log = response["log"] - msg = f"Matching run has stopped:\n\n{log}" - raise PurlDBException(msg) - - time.sleep(sleep) - - -def get_match_results(run_url): - """ - Given the `run_url` for a pipeline running the matchcode matching pipeline, - return the match results for that run. - """ - response = request_get(run_url) - project_url = response["project"] - # `project_url` can have params, such as "?format=json" - if "?" in project_url: - project_url, _ = project_url.split("?") - project_url = project_url.rstrip("/") - results_url = project_url + "/results/" - return request_get(results_url) - - -def map_match_results(match_results): - """ - Given `match_results`, which is a mapping of ScanCode.io codebase results, - return a defaultdict(list) where the keys are the package_uid of matched - packages and the value is a list containing the paths of Resources - associated with the package_uid. - """ - resource_results = match_results.get("files", []) - resource_paths_by_package_uids = defaultdict(list) - for resource in resource_results: - for_packages = resource.get("for_packages", []) - for package_uid in for_packages: - resource_paths_by_package_uids[package_uid].append(resource["path"]) - return resource_paths_by_package_uids - - -def create_packages_from_match_results(project, match_results): - """ - Given `match_results`, which is a mapping of ScanCode.io codebase results, - use the Package data from it to create DiscoveredPackages for `project` and - associate the proper Resources of `project` to the DiscoveredPackages. - """ - from scanpipe.pipes.d2d import create_package_from_purldb_data - - resource_paths_by_package_uids = map_match_results(match_results) - matched_packages = match_results.get("packages", []) - for matched_package in matched_packages: - package_uid = matched_package["package_uid"] - resource_paths = resource_paths_by_package_uids[package_uid] - resources = project.codebaseresources.filter(path__in=resource_paths) - create_package_from_purldb_data( - project, - resources=resources, - package_data=matched_package, - status=flag.MATCHED_TO_PURLDB_PACKAGE, - ) diff --git a/scanpipe/tests/data/purldb/match_to_purldb/codebase.json b/scanpipe/tests/data/matchcode/match_to_matchcode/codebase.json similarity index 100% rename from scanpipe/tests/data/purldb/match_to_purldb/codebase.json rename to scanpipe/tests/data/matchcode/match_to_matchcode/codebase.json diff --git a/scanpipe/tests/data/purldb/match_to_purldb/request_get_check_response.json b/scanpipe/tests/data/matchcode/match_to_matchcode/request_get_check_response.json similarity index 100% rename from scanpipe/tests/data/purldb/match_to_purldb/request_get_check_response.json rename to scanpipe/tests/data/matchcode/match_to_matchcode/request_get_check_response.json diff --git a/scanpipe/tests/data/purldb/match_to_purldb/request_get_results_response.json b/scanpipe/tests/data/matchcode/match_to_matchcode/request_get_results_response.json similarity index 100% rename from scanpipe/tests/data/purldb/match_to_purldb/request_get_results_response.json rename to scanpipe/tests/data/matchcode/match_to_matchcode/request_get_results_response.json diff --git a/scanpipe/tests/data/purldb/match_to_purldb/request_post_response.json b/scanpipe/tests/data/matchcode/match_to_matchcode/request_post_response.json similarity index 100% rename from scanpipe/tests/data/purldb/match_to_purldb/request_post_response.json rename to scanpipe/tests/data/matchcode/match_to_matchcode/request_post_response.json diff --git a/scanpipe/tests/pipes/test_matchcode.py b/scanpipe/tests/pipes/test_matchcode.py index 4192caabd3..75cc40e71a 100644 --- a/scanpipe/tests/pipes/test_matchcode.py +++ b/scanpipe/tests/pipes/test_matchcode.py @@ -20,18 +20,26 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +import json +from collections import defaultdict from pathlib import Path +from unittest import mock from django.core.management import call_command from django.test import TestCase +from scanpipe.models import AbstractTaskFieldsModel from scanpipe.models import Project from scanpipe.pipes import matchcode +from scanpipe.tests import make_resource_file class MatchCodePipesTest(TestCase): data_location = Path(__file__).parent.parent / "data" + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + def test_scanpipe_pipes_matchcode_fingerprint_codebase_directories(self): fixtures = self.data_location / "asgiref-3.3.0_fixtures.json" call_command("loaddata", fixtures, **{"verbosity": 0}) @@ -46,3 +54,220 @@ def test_scanpipe_pipes_matchcode_fingerprint_codebase_directories(self): "directory_structure": "0000000e0e30a50b5eb8c495f880c087325e6062", } self.assertEqual(expected_directory_fingerprints, directory.extra_data) + + @mock.patch("scanpipe.pipes.matchcode.request_post") + @mock.patch("scanpipe.pipes.matchcode.is_available") + def test_scanpipe_pipes_matchcode_send_project_json_to_matchcode( + self, mock_is_available, mock_request_post + ): + mock_is_available.return_value = True + + def mock_request_post_return(url, files, timeout): + request_post_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_post_response.json" + ) + with open(request_post_response_loc, "r") as f: + return json.load(f) + + mock_request_post.side_effect = mock_request_post_return + + run_url = matchcode.send_project_json_to_matchcode(self.project1) + expected_run_url = ( + "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + ) + self.assertEqual(expected_run_url, run_url) + + @mock.patch("scanpipe.pipes.matchcode.request_get") + @mock.patch("scanpipe.pipes.matchcode.is_available") + def test_scanpipe_pipes_matchcode_poll_until_success( + self, mock_is_available, mock_request_get + ): + run_status = AbstractTaskFieldsModel.Status + + mock_is_available.return_value = True + + # Success + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.SUCCESS, + }, + ] + return_value = matchcode.poll_until_success(run_url) + self.assertEqual(True, return_value) + + # Failure + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.FAILURE, + "log": "failure message", + }, + ] + with self.assertRaises(Exception) as context: + matchcode.poll_until_success(run_url) + self.assertTrue("failure message" in str(context.exception)) + + # Stopped + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.STOPPED, + "log": "stop message", + }, + ] + with self.assertRaises(Exception) as context: + matchcode.poll_until_success(run_url) + self.assertTrue("stop message" in str(context.exception)) + + # Stale + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.STALE, + "log": "stale message", + }, + ] + with self.assertRaises(Exception) as context: + matchcode.poll_until_success(run_url) + self.assertTrue("stale message" in str(context.exception)) + + def test_scanpipe_pipes_matchcode_map_match_results(self): + request_post_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_get_results_response.json" + ) + with open(request_post_response_loc, "r") as f: + match_results = json.load(f) + + resource_paths_by_package_uids = matchcode.map_match_results(match_results) + expected = defaultdict(list) + expected_package_uid = ( + "pkg:maven/org.elasticsearch/elasticsearch-x-content@7.17.9" + "?classifier=sources&uuid=a8814800-8120-4f50-ba4f-08c443ccda8e" + ) + expected[expected_package_uid].append( + "elasticsearch-x-content-7.17.9-sources.jar" + ) + self.assertEqual(expected, resource_paths_by_package_uids) + + def test_scanpipe_pipes_matchcode_create_packages_from_match_results(self): + r1 = make_resource_file( + self.project1, + path="elasticsearch-x-content-7.17.9-sources.jar", + sha1="30d21add57abe04beece3f28a079671dbc9043e4", + ) + r2 = make_resource_file( + self.project1, + path="something-else.json", + sha1="deadbeef", + ) + + request_get_results_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_get_results_response.json" + ) + with open(request_get_results_response_loc, "r") as f: + match_results = json.load(f) + + self.assertEqual(0, self.project1.discoveredpackages.all().count()) + self.assertFalse(0, len(r1.for_packages)) + self.assertFalse(0, len(r2.for_packages)) + + matchcode.create_packages_from_match_results(self.project1, match_results) + + self.assertEqual(1, self.project1.discoveredpackages.all().count()) + package = self.project1.discoveredpackages.first() + self.assertEqual([package.package_uid], r1.for_packages) + # This resource should not have a Package match + self.assertFalse(0, len(r2.for_packages)) + + @mock.patch("scanpipe.pipes.matchcode.request_get") + @mock.patch("scanpipe.pipes.matchcode.is_available") + def test_scanpipe_pipes_matchcode_get_match_results( + self, mock_is_available, mock_request_get + ): + mock_is_available.return_value = True + + request_get_check_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_get_check_response.json" + ) + with open(request_get_check_response_loc, "r") as f: + mock_request_get_check_return = json.load(f) + + request_get_results_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_get_results_response.json" + ) + with open(request_get_results_response_loc, "r") as f: + mock_request_get_results_return = json.load(f) + mock_request_get.side_effect = [ + mock_request_get_check_return, + mock_request_get_results_return, + ] + + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + match_results = matchcode.get_match_results(run_url) + + self.assertEqual(mock_request_get_results_return, match_results) diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 441f929c96..8a418e963d 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -21,14 +21,11 @@ # Visit https://github.com/nexB/scancode.io for support and download. import io -import json -from collections import defaultdict from pathlib import Path from unittest import mock from django.test import TestCase -from scanpipe.models import AbstractTaskFieldsModel from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage @@ -36,7 +33,6 @@ from scanpipe.pipes import purldb from scanpipe.tests import dependency_data2 from scanpipe.tests import dependency_data3 -from scanpipe.tests import make_resource_file from scanpipe.tests import package_data1 @@ -106,220 +102,3 @@ def mock_request_post_return(url, data, headers, timeout): self.assertIn( "1 PURLs were already present in PurlDB index queue", expected_log ) - - @mock.patch("scanpipe.pipes.purldb.request_post") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_send_project_json_to_matchcode( - self, mock_is_available, mock_request_post - ): - mock_is_available.return_value = True - - def mock_request_post_return(url, files, timeout): - request_post_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_post_response.json" - ) - with open(request_post_response_loc, "r") as f: - return json.load(f) - - mock_request_post.side_effect = mock_request_post_return - - run_url = purldb.send_project_json_to_matchcode(self.project1) - expected_run_url = ( - "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - ) - self.assertEqual(expected_run_url, run_url) - - @mock.patch("scanpipe.pipes.purldb.request_get") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_poll_until_success( - self, mock_is_available, mock_request_get - ): - run_status = AbstractTaskFieldsModel.Status - - mock_is_available.return_value = True - - # Success - run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.SUCCESS, - }, - ] - return_value = purldb.poll_until_success(run_url) - self.assertEqual(True, return_value) - - # Failure - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.FAILURE, - "log": "failure message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_until_success(run_url) - self.assertTrue("failure message" in str(context.exception)) - - # Stopped - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.STOPPED, - "log": "stop message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_until_success(run_url) - self.assertTrue("stop message" in str(context.exception)) - - # Stale - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.STALE, - "log": "stale message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_until_success(run_url) - self.assertTrue("stale message" in str(context.exception)) - - def test_scanpipe_pipes_purldb_map_match_results(self): - request_post_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_post_response_loc, "r") as f: - match_results = json.load(f) - - resource_paths_by_package_uids = purldb.map_match_results(match_results) - expected = defaultdict(list) - expected_package_uid = ( - "pkg:maven/org.elasticsearch/elasticsearch-x-content@7.17.9" - "?classifier=sources&uuid=a8814800-8120-4f50-ba4f-08c443ccda8e" - ) - expected[expected_package_uid].append( - "elasticsearch-x-content-7.17.9-sources.jar" - ) - self.assertEqual(expected, resource_paths_by_package_uids) - - def test_scanpipe_pipes_purldb_create_packages_from_match_results(self): - r1 = make_resource_file( - self.project1, - path="elasticsearch-x-content-7.17.9-sources.jar", - sha1="30d21add57abe04beece3f28a079671dbc9043e4", - ) - r2 = make_resource_file( - self.project1, - path="something-else.json", - sha1="deadbeef", - ) - - request_get_results_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_get_results_response_loc, "r") as f: - match_results = json.load(f) - - self.assertEqual(0, self.project1.discoveredpackages.all().count()) - self.assertFalse(0, len(r1.for_packages)) - self.assertFalse(0, len(r2.for_packages)) - - purldb.create_packages_from_match_results(self.project1, match_results) - - self.assertEqual(1, self.project1.discoveredpackages.all().count()) - package = self.project1.discoveredpackages.first() - self.assertEqual([package.package_uid], r1.for_packages) - # This resource should not have a Package match - self.assertFalse(0, len(r2.for_packages)) - - @mock.patch("scanpipe.pipes.purldb.request_get") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_get_match_results( - self, mock_is_available, mock_request_get - ): - mock_is_available.return_value = True - - request_get_check_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_check_response.json" - ) - with open(request_get_check_response_loc, "r") as f: - mock_request_get_check_return = json.load(f) - - request_get_results_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_get_results_response_loc, "r") as f: - mock_request_get_results_return = json.load(f) - mock_request_get.side_effect = [ - mock_request_get_check_return, - mock_request_get_results_return, - ] - - run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - match_results = purldb.get_match_results(run_url) - - self.assertEqual(mock_request_get_results_return, match_results) diff --git a/setup.cfg b/setup.cfg index 91a360f732..5a2ed8a12f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -136,7 +136,7 @@ scancodeio_pipelines = load_inventory = scanpipe.pipelines.load_inventory:LoadInventory load_sbom = scanpipe.pipelines.load_sbom:LoadSBOM map_deploy_to_develop = scanpipe.pipelines.deploy_to_develop:DeployToDevelop - match_to_purldb = scanpipe.pipelines.match_to_purldb:MatchToPurlDB + match_to_matchcode = scanpipe.pipelines.match_to_matchcode:MatchToMatchCode populate_purldb = scanpipe.pipelines.populate_purldb:PopulatePurlDB resolve_dependencies = scanpipe.pipelines.resolve_dependencies:ResolveDependencies scan_codebase = scanpipe.pipelines.scan_codebase:ScanCodebase