From a25c41c72cb9799f7c4c3585b202b32231e94631 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 7 Feb 2024 17:59:47 -0800 Subject: [PATCH 01/30] Create management command for purldb package scanning Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 121 +++++++++++++++++ scanpipe/pipes/purldb.py | 122 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 scanpipe/management/commands/package-scan-worker.py diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py new file mode 100644 index 0000000000..f50325e66a --- /dev/null +++ b/scanpipe/management/commands/package-scan-worker.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import time +from scanpipe.pipes import purldb +from django.core.exceptions import ValidationError +from django.core.management import CommandError +from django.core.management import call_command +from django.core.management.base import BaseCommand + +from scanpipe.management.commands import AddInputCommandMixin +from scanpipe.management.commands import extract_group_from_pipelines +from scanpipe.management.commands import validate_copy_from +from scanpipe.management.commands import validate_pipelines +from scanpipe.models import Project +from scanpipe.pipes import output + + + +class Command(AddInputCommandMixin, BaseCommand): + help = "Create a ScanPipe project." + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + "--sleep", + help="Number in seconds how long the loop should sleep for before polling.", + ) + + def handle(self, *args, **options): + sleep = options["sleep"] + + while True: + # 1. get download url from purldb + response = purldb.get_next_job() + if response: + download_url, package_uuid = response + else: + time.sleep(sleep) + continue + + # 2. create and run project + # TODO: create name based off of purl + uuid + name = package_uuid + project = Project(name=name) + try: + project.full_clean(exclude=["slug"]) + except ValidationError as e: + raise CommandError("\n".join(e.messages)) + + # Run validation before creating the project in the database + pipelines = ['scan_and_fingerprint_package'] + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + project.save() + self.project = project + msg = f"Project {name} created with work directory {project.work_directory}" + self.stdout.write(msg, self.style.SUCCESS) + + for pipeline_name, selected_groups in pipelines_data.items(): + self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) + + input_urls = [download_url] + self.handle_input_urls(input_urls) + + call_command( + "execute", + project=project, + stderr=self.stderr, + stdout=self.stdout, + **{"async": True}, + ) + + # 3. poll project results + # TODO: consider refactoring `purldb.poll_until_success` to work here + run = project.runs.first() + status = run.Status + while True: + run_status = run.status + if run_status == status.SUCCESS: + break + + if run_status in [ + status.NOT_STARTED, + status.QUEUED, + status.RUNNING, + ]: + time.sleep(sleep) + continue + + if run_status in [ + status.FAILURE, + status.STOPPED, + status.STALE, + ]: + self.stderr.write(run.log) + continue + + # 4. get project results and send to purldb + scan_output_location = to_json(project) + purldb.send_results_to_purldb(package_uuid, scan_output_location) + diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index eb5ac6c822..d4edd9e60c 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -344,3 +344,125 @@ def find_packages(payload): response = request_get(package_api_url, payload=payload) if response and response.get("count") > 0: return response.get("results") + + +def poll_until_success(run_url, sleep=10): + """ + Given a URL to a scancode.io run instance, `run_url`, return True when the + run instance has completed successfully. + + Raise a PurlDBException when the run instance has failed, stopped, or gone + stale. + """ + run_status = AbstractTaskFieldsModel.Status + while True: + response = request_get(run_url) + if response: + status = response["status"] + if status == run_status.SUCCESS: + return True + + if status in [ + run_status.NOT_STARTED, + run_status.QUEUED, + run_status.RUNNING, + ]: + continue + + if status in [ + run_status.FAILURE, + run_status.STOPPED, + run_status.STALE, + ]: + log = response["log"] + msg = f"Matching run has stopped:\n\n{log}" + raise PurlDBException(msg) + + time.sleep(sleep) + + +def get_match_results(run_url): + """ + Given the `run_url` for a pipeline running the matchcode matching pipeline, + return the match results for that run. + """ + response = request_get(run_url) + project_url = response["project"] + # `project_url` can have params, such as "?format=json" + if "?" in project_url: + project_url, _ = project_url.split("?") + project_url = project_url.rstrip("/") + results_url = project_url + "/results/" + return request_get(results_url) + + +def map_match_results(match_results): + """ + Given `match_results`, which is a mapping of ScanCode.io codebase results, + return a defaultdict(list) where the keys are the package_uid of matched + packages and the value is a list containing the paths of Resources + associated with the package_uid. + """ + resource_results = match_results.get("files", []) + resource_paths_by_package_uids = defaultdict(list) + for resource in resource_results: + for_packages = resource.get("for_packages", []) + for package_uid in for_packages: + resource_paths_by_package_uids[package_uid].append(resource["path"]) + return resource_paths_by_package_uids + + +def create_packages_from_match_results(project, match_results): + """ + Given `match_results`, which is a mapping of ScanCode.io codebase results, + use the Package data from it to create DiscoveredPackages for `project` and + associate the proper Resources of `project` to the DiscoveredPackages. + """ + from scanpipe.pipes.d2d import create_package_from_purldb_data + + resource_paths_by_package_uids = map_match_results(match_results) + matched_packages = match_results.get("packages", []) + for matched_package in matched_packages: + package_uid = matched_package["package_uid"] + resource_paths = resource_paths_by_package_uids[package_uid] + resources = project.codebaseresources.filter(path__in=resource_paths) + create_package_from_purldb_data( + project, + resources=resources, + package_data=matched_package, + status=flag.MATCHED_TO_PURLDB_PACKAGE, + ) + + +def get_next_job( + timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL +): + """ + Return the download URL and Package UUID of the next Package to be scanned from PurlDB + + Return None if no job is available + """ + response = request_get( + url=f"{api_url}scan_queue/get_next_download_url", + timeout=timeout, + ) + if response: + download_url = response['download_url'] + package_uuid = response['package_uuid'] + return download_url, package_uuid + + +def send_results_to_purldb( + package_uuid, scan_output_location, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, +): + with open(scan_output_location, "rb") as f: + data={ + "package_uuid": package_uuid, + "scan_file": f, + } + response = request_post( + url=f"{api_url}scan_queue/submit_scan_results", + timeout=timeout, + data=data, + ) + return response From 91cb76ca50bf2dc1e6b6120d1cd58ffd9d439962 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 9 Feb 2024 19:48:42 -0800 Subject: [PATCH 02/30] Fix bugs in package-scan-worker Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 11 ++++++++--- scanpipe/pipes/purldb.py | 7 +++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index f50325e66a..bb760c6d3c 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -43,6 +43,7 @@ def add_arguments(self, parser): super().add_arguments(parser) parser.add_argument( "--sleep", + type=int, help="Number in seconds how long the loop should sleep for before polling.", ) @@ -55,6 +56,12 @@ def handle(self, *args, **options): if response: download_url, package_uuid = response else: + self.stdout.write("bad response") + time.sleep(sleep) + continue + + if not download_url or not package_uuid: + self.stdout.write("no new job") time.sleep(sleep) continue @@ -87,7 +94,6 @@ def handle(self, *args, **options): project=project, stderr=self.stderr, stdout=self.stdout, - **{"async": True}, ) # 3. poll project results @@ -116,6 +122,5 @@ def handle(self, *args, **options): continue # 4. get project results and send to purldb - scan_output_location = to_json(project) + scan_output_location = output.to_json(project) purldb.send_results_to_purldb(package_uuid, scan_output_location) - diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index d4edd9e60c..eed32a3c45 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -443,7 +443,7 @@ def get_next_job( Return None if no job is available """ response = request_get( - url=f"{api_url}scan_queue/get_next_download_url", + url=f"{api_url}scan_queue/get_next_download_url/", timeout=timeout, ) if response: @@ -458,11 +458,14 @@ def send_results_to_purldb( with open(scan_output_location, "rb") as f: data={ "package_uuid": package_uuid, + } + files={ "scan_file": f, } response = request_post( - url=f"{api_url}scan_queue/submit_scan_results", + url=f"{api_url}scan_queue/submit_scan_results/", timeout=timeout, data=data, + files=files, ) return response From c31e7e79d05418a97e5bf1dee0bd671011ec1840 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 14 Feb 2024 16:49:38 -0800 Subject: [PATCH 03/30] Send updates to purldb regarding scan status Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 42 ++++++++++----- scanpipe/pipes/purldb.py | 51 ++++++++++++++----- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index bb760c6d3c..4f5118f136 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -21,7 +21,7 @@ # Visit https://github.com/nexB/scancode.io for support and download. import time -from scanpipe.pipes import purldb + from django.core.exceptions import ValidationError from django.core.management import CommandError from django.core.management import call_command @@ -29,11 +29,10 @@ from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import extract_group_from_pipelines -from scanpipe.management.commands import validate_copy_from from scanpipe.management.commands import validate_pipelines from scanpipe.models import Project from scanpipe.pipes import output - +from scanpipe.pipes import purldb class Command(AddInputCommandMixin, BaseCommand): @@ -54,20 +53,20 @@ def handle(self, *args, **options): # 1. get download url from purldb response = purldb.get_next_job() if response: - download_url, package_uuid = response + download_url, scannable_uri_uuid = response else: self.stdout.write("bad response") time.sleep(sleep) continue - if not download_url or not package_uuid: + if not download_url or not scannable_uri_uuid: self.stdout.write("no new job") time.sleep(sleep) continue # 2. create and run project # TODO: create name based off of purl + uuid - name = package_uuid + name = scannable_uri_uuid project = Project(name=name) try: project.full_clean(exclude=["slug"]) @@ -75,7 +74,7 @@ def handle(self, *args, **options): raise CommandError("\n".join(e.messages)) # Run validation before creating the project in the database - pipelines = ['scan_and_fingerprint_package'] + pipelines = ["scan_and_fingerprint_package"] pipelines_data = extract_group_from_pipelines(pipelines) pipelines_data = validate_pipelines(pipelines_data) project.save() @@ -84,7 +83,9 @@ def handle(self, *args, **options): self.stdout.write(msg, self.style.SUCCESS) for pipeline_name, selected_groups in pipelines_data.items(): - self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) + self.project.add_pipeline( + pipeline_name, selected_groups=selected_groups + ) input_urls = [download_url] self.handle_input_urls(input_urls) @@ -100,6 +101,8 @@ def handle(self, *args, **options): # TODO: consider refactoring `purldb.poll_until_success` to work here run = project.runs.first() status = run.Status + error_log = "" + scan_started = False while True: run_status = run.status if run_status == status.SUCCESS: @@ -110,6 +113,12 @@ def handle(self, *args, **options): status.QUEUED, status.RUNNING, ]: + if run_status == status.RUNNING and not scan_started: + scan_started = True + purldb.update_status( + scannable_uri_uuid, + status="in progress", + ) time.sleep(sleep) continue @@ -118,9 +127,18 @@ def handle(self, *args, **options): status.STOPPED, status.STALE, ]: + error_log = run.log self.stderr.write(run.log) - continue + break - # 4. get project results and send to purldb - scan_output_location = output.to_json(project) - purldb.send_results_to_purldb(package_uuid, scan_output_location) + if error_log: + # send error response to purldb + purldb.update_status( + scannable_uri_uuid, + status="failed", + scan_log=error_log, + ) + else: + # 4. get project results and send to purldb + scan_output_location = output.to_json(project) + purldb.send_results_to_purldb(scannable_uri_uuid, scan_output_location) diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index eed32a3c45..d268cfa412 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -434,38 +434,65 @@ def create_packages_from_match_results(project, match_results): ) -def get_next_job( - timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL -): +def get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): """ - Return the download URL and Package UUID of the next Package to be scanned from PurlDB + Return the download URL and Package UUID of the next Package to be scanned + from PurlDB - Return None if no job is available + Return None if the request was not successful """ response = request_get( url=f"{api_url}scan_queue/get_next_download_url/", timeout=timeout, ) if response: - download_url = response['download_url'] - package_uuid = response['package_uuid'] + download_url = response["download_url"] + package_uuid = response["package_uuid"] return download_url, package_uuid def send_results_to_purldb( - package_uuid, scan_output_location, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, + scannable_uri_uuid, + scan_output_location, + timeout=DEFAULT_TIMEOUT, + api_url=PURLDB_API_URL, ): + """ + Send project results to purldb for the package handeled by the ScannableURI + with uuid of `scannable_uri_uuid` + """ with open(scan_output_location, "rb") as f: - data={ - "package_uuid": package_uuid, + data = { + "scannable_uri_uuid": scannable_uri_uuid, + "scan_status": "scanned", } - files={ + files = { "scan_file": f, } response = request_post( - url=f"{api_url}scan_queue/submit_scan_results/", + url=f"{api_url}scan_queue/update_status/", timeout=timeout, data=data, files=files, ) return response + + +def update_status( + scannable_uri_uuid, + status, + scan_log="", + timeout=DEFAULT_TIMEOUT, + api_url=PURLDB_API_URL, +): + data = { + "scannable_uri_uuid": scannable_uri_uuid, + "scan_status": status, + "scan_log": scan_log, + } + response = request_post( + url=f"{api_url}scan_queue/update_status/", + timeout=timeout, + data=data, + ) + return response From 6bad6bc6aa2c5fbef0f5084f5527e0c16dc5901f Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 14 Feb 2024 17:50:06 -0800 Subject: [PATCH 04/30] Return scan project url when updating scan status on purldb Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 2 ++ scanpipe/pipes/purldb.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 4f5118f136..d3c5b19a88 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -115,9 +115,11 @@ def handle(self, *args, **options): ]: if run_status == status.RUNNING and not scan_started: scan_started = True + scan_project_url = project.get_absolute_url() purldb.update_status( scannable_uri_uuid, status="in progress", + scan_project_url=scan_project_url, ) time.sleep(sleep) continue diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index d268cfa412..3038591050 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -482,6 +482,7 @@ def update_status( scannable_uri_uuid, status, scan_log="", + scan_project_url="", timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): @@ -489,6 +490,7 @@ def update_status( "scannable_uri_uuid": scannable_uri_uuid, "scan_status": status, "scan_log": scan_log, + "scan_project_url": scan_project_url, } response = request_post( url=f"{api_url}scan_queue/update_status/", From bad9d620ebeab7a17a64008ebbd90f5b95c1cc39 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 23 Feb 2024 12:05:24 -0800 Subject: [PATCH 05/30] Get scananble_uri_uuid instead of package_uuid * Add sleep to main work loop Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 2 ++ scanpipe/pipes/purldb.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index d3c5b19a88..3b6475e2f9 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -133,6 +133,8 @@ def handle(self, *args, **options): self.stderr.write(run.log) break + time.sleep(sleep) + if error_log: # send error response to purldb purldb.update_status( diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 3038591050..6833ae15bf 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -447,8 +447,8 @@ def get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): ) if response: download_url = response["download_url"] - package_uuid = response["package_uuid"] - return download_url, package_uuid + scannable_uri_uuid = response["scannable_uri_uuid"] + return download_url, scannable_uri_uuid def send_results_to_purldb( From 57526394dbf1dd80d26bc61066e7c34cef1810a1 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 26 Feb 2024 18:19:05 -0800 Subject: [PATCH 06/30] Move project creation logic into its own function * Handle exceptions in package-scan-worker and send to purldb as errors Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 41 ++++ .../management/commands/create-project.py | 49 ++--- .../commands/package-scan-worker.py | 182 +++++++++--------- 3 files changed, 148 insertions(+), 124 deletions(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 6c3c1051e9..5a980c9a98 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -25,6 +25,7 @@ from django.apps import apps from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ValidationError from django.core.management.base import BaseCommand from django.core.management.base import CommandError from django.template.defaultfilters import pluralize @@ -288,3 +289,43 @@ def validate_pipelines(pipelines_data): ) return pipelines_data + + +def create_project( + command, name, pipelines=[], input_files=[], input_urls=[], copy_from="", notes="" +): + project = Project(name=name) + if notes: + project.notes = notes + + try: + project.full_clean(exclude=["slug"]) + except ValidationError as e: + raise CommandError("\n".join(e.messages)) + + # Run validation before creating the project in the database + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + + input_files_data = command.extract_tag_from_input_files(input_files) + command.validate_input_files(input_files=input_files_data.keys()) + validate_copy_from(copy_from) + + project.save() + command.project = project + msg = f"Project {name} created with work directory {project.work_directory}" + command.stdout.write(msg, command.style.SUCCESS) + + for pipeline_name, selected_groups in pipelines_data.items(): + command.project.add_pipeline(pipeline_name, selected_groups=selected_groups) + + if input_files: + command.handle_input_files(input_files_data) + + if input_urls: + command.handle_input_urls(input_urls) + + if copy_from: + command.handle_copy_codebase(copy_from) + + return project diff --git a/scanpipe/management/commands/create-project.py b/scanpipe/management/commands/create-project.py index e1816aa9e6..5bfbee7eaf 100644 --- a/scanpipe/management/commands/create-project.py +++ b/scanpipe/management/commands/create-project.py @@ -20,16 +20,12 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from django.core.exceptions import ValidationError from django.core.management import CommandError from django.core.management import call_command from django.core.management.base import BaseCommand from scanpipe.management.commands import AddInputCommandMixin -from scanpipe.management.commands import extract_group_from_pipelines -from scanpipe.management.commands import validate_copy_from -from scanpipe.management.commands import validate_pipelines -from scanpipe.models import Project +from scanpipe.management.commands import create_project class Command(AddInputCommandMixin, BaseCommand): @@ -75,43 +71,20 @@ def handle(self, *args, **options): input_urls = options["input_urls"] copy_from = options["copy_codebase"] execute = options["execute"] - - project = Project(name=name) - if notes := options["notes"]: - project.notes = notes - - try: - project.full_clean(exclude=["slug"]) - except ValidationError as e: - raise CommandError("\n".join(e.messages)) - - # Run validation before creating the project in the database - pipelines_data = extract_group_from_pipelines(pipelines) - pipelines_data = validate_pipelines(pipelines_data) - - input_files_data = self.extract_tag_from_input_files(input_files) - self.validate_input_files(input_files=input_files_data.keys()) - validate_copy_from(copy_from) + notes = options["notes"] if execute and not pipelines: raise CommandError("The --execute option requires one or more pipelines.") - project.save() - self.project = project - msg = f"Project {name} created with work directory {project.work_directory}" - self.stdout.write(msg, self.style.SUCCESS) - - for pipeline_name, selected_groups in pipelines_data.items(): - self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) - - if input_files: - self.handle_input_files(input_files_data) - - if input_urls: - self.handle_input_urls(input_urls) - - if copy_from: - self.handle_copy_codebase(copy_from) + project = create_project( + command=self, + name=name, + pipelines=pipelines, + input_files=input_files, + input_urls=input_urls, + copy_from=copy_from, + notes=notes, + ) if execute: call_command( diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 3b6475e2f9..c923f65d8e 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -21,16 +21,13 @@ # Visit https://github.com/nexB/scancode.io for support and download. import time +from traceback import format_tb -from django.core.exceptions import ValidationError -from django.core.management import CommandError from django.core.management import call_command from django.core.management.base import BaseCommand from scanpipe.management.commands import AddInputCommandMixin -from scanpipe.management.commands import extract_group_from_pipelines -from scanpipe.management.commands import validate_pipelines -from scanpipe.models import Project +from scanpipe.management.commands import create_project from scanpipe.pipes import output from scanpipe.pipes import purldb @@ -56,93 +53,106 @@ def handle(self, *args, **options): download_url, scannable_uri_uuid = response else: self.stdout.write("bad response") - time.sleep(sleep) - continue if not download_url or not scannable_uri_uuid: self.stdout.write("no new job") - time.sleep(sleep) - continue - - # 2. create and run project - # TODO: create name based off of purl + uuid - name = scannable_uri_uuid - project = Project(name=name) - try: - project.full_clean(exclude=["slug"]) - except ValidationError as e: - raise CommandError("\n".join(e.messages)) - - # Run validation before creating the project in the database - pipelines = ["scan_and_fingerprint_package"] - pipelines_data = extract_group_from_pipelines(pipelines) - pipelines_data = validate_pipelines(pipelines_data) - project.save() - self.project = project - msg = f"Project {name} created with work directory {project.work_directory}" - self.stdout.write(msg, self.style.SUCCESS) - - for pipeline_name, selected_groups in pipelines_data.items(): - self.project.add_pipeline( - pipeline_name, selected_groups=selected_groups - ) - - input_urls = [download_url] - self.handle_input_urls(input_urls) - - call_command( - "execute", - project=project, - stderr=self.stderr, - stdout=self.stdout, - ) - - # 3. poll project results - # TODO: consider refactoring `purldb.poll_until_success` to work here - run = project.runs.first() - status = run.Status - error_log = "" - scan_started = False - while True: - run_status = run.status - if run_status == status.SUCCESS: - break - - if run_status in [ - status.NOT_STARTED, - status.QUEUED, - status.RUNNING, - ]: - if run_status == status.RUNNING and not scan_started: - scan_started = True - scan_project_url = project.get_absolute_url() + else: + try: + # 2. create and run project + # TODO: create name based off of purl + uuid + name = scannable_uri_uuid + pipelines = ["scan_and_fingerprint_package"] + input_urls = [download_url] + project = create_project( + self, + name=name, + pipelines=pipelines, + input_urls=input_urls, + ) + + call_command( + "execute", + project=project, + stderr=self.stderr, + stdout=self.stdout, + ) + + # 3. poll project results + error_log = poll_run_status( + command=self, + project=project, + scannable_uri_uuid=scannable_uri_uuid, + sleep=sleep, + ) + + if error_log: + # send error response to purldb purldb.update_status( scannable_uri_uuid, - status="in progress", - scan_project_url=scan_project_url, + status="failed", + scan_log=error_log, + ) + else: + # 4. get project results and send to purldb + scan_output_location = output.to_json(project) + purldb.send_results_to_purldb( + scannable_uri_uuid, scan_output_location ) - time.sleep(sleep) - continue - - if run_status in [ - status.FAILURE, - status.STOPPED, - status.STALE, - ]: - error_log = run.log - self.stderr.write(run.log) - break - - time.sleep(sleep) - - if error_log: - # send error response to purldb + + except Exception as e: + traceback = "" + if hasattr(e, "__traceback__"): + traceback = "".join(format_tb(e.__traceback__)) + purldb.update_status( + scannable_uri_uuid, + status="failed", + scan_log=traceback, + ) + + time.sleep(sleep) + + +def poll_run_status(command, project, scannable_uri_uuid, sleep): + """ + Poll the status of the first run of `project`. Return the log of the run if + the run has stopped, failed, or gone stale, otherwise return an empty + string. + """ + # TODO: consider refactoring `purldb.poll_until_success` to work here + run = project.runs.first() + status = run.Status + error_log = "" + scan_started = False + while True: + run_status = run.status + + if run_status in [ + status.SUCCESS, + status.FAILURE, + status.STOPPED, + status.STALE, + ]: + if run_status in [ + status.FAILURE, + status.STOPPED, + status.STALE, + ]: + error_log = run.log + command.stderr.write(error_log) + return error_log + + if run_status in [ + status.NOT_STARTED, + status.QUEUED, + status.RUNNING, + ]: + if run_status == status.RUNNING and not scan_started: + scan_started = True + scan_project_url = project.get_absolute_url() purldb.update_status( scannable_uri_uuid, - status="failed", - scan_log=error_log, + status="in progress", + scan_project_url=scan_project_url, ) - else: - # 4. get project results and send to purldb - scan_output_location = output.to_json(project) - purldb.send_results_to_purldb(scannable_uri_uuid, scan_output_location) + + time.sleep(sleep) From 6d65cb51e7aea0f92d8b50fa84c03beeb574d89d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 26 Feb 2024 18:40:15 -0800 Subject: [PATCH 07/30] Create project name from download url and uuid Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index c923f65d8e..8b60654027 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -25,6 +25,7 @@ from django.core.management import call_command from django.core.management.base import BaseCommand +from django.utils.text import slugify from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import create_project @@ -59,8 +60,7 @@ def handle(self, *args, **options): else: try: # 2. create and run project - # TODO: create name based off of purl + uuid - name = scannable_uri_uuid + name = create_project_name(download_url, scannable_uri_uuid) pipelines = ["scan_and_fingerprint_package"] input_urls = [download_url] project = create_project( @@ -112,6 +112,13 @@ def handle(self, *args, **options): time.sleep(sleep) +def create_project_name(download_url, scannable_uri_uuid): + """Create a project name from `download_url` and `scannable_uri_uuid`""" + if len(download_url) > 50: + download_url = download_url[0:50] + return f"{slugify(download_url)}-{scannable_uri_uuid[0:8]}" + + def poll_run_status(command, project, scannable_uri_uuid, sleep): """ Poll the status of the first run of `project`. Return the log of the run if From 65ab28937473cdf75d8f4c216ed5e4a882fac17d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Feb 2024 15:18:36 -0800 Subject: [PATCH 08/30] Refactor poll_until_success Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 51 ++++---------- scanpipe/pipes/purldb.py | 69 +++++++++++++------ 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 8b60654027..9b5d838da7 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -125,41 +125,18 @@ def poll_run_status(command, project, scannable_uri_uuid, sleep): the run has stopped, failed, or gone stale, otherwise return an empty string. """ - # TODO: consider refactoring `purldb.poll_until_success` to work here run = project.runs.first() - status = run.Status - error_log = "" - scan_started = False - while True: - run_status = run.status - - if run_status in [ - status.SUCCESS, - status.FAILURE, - status.STOPPED, - status.STALE, - ]: - if run_status in [ - status.FAILURE, - status.STOPPED, - status.STALE, - ]: - error_log = run.log - command.stderr.write(error_log) - return error_log - - if run_status in [ - status.NOT_STARTED, - status.QUEUED, - status.RUNNING, - ]: - if run_status == status.RUNNING and not scan_started: - scan_started = True - scan_project_url = project.get_absolute_url() - purldb.update_status( - scannable_uri_uuid, - status="in progress", - scan_project_url=scan_project_url, - ) - - time.sleep(sleep) + if purldb.poll_until_success( + check=get_run_status, + run=run + ): + return "" + else: + error_log = run.log + command.stderr.write(error_log) + return error_log + + +def get_run_status(run, **kwargs): + run.refresh_from_db() + return run.status diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 6833ae15bf..4644fa6cb8 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -346,7 +346,7 @@ def find_packages(payload): return response.get("results") -def poll_until_success(run_url, sleep=10): +def poll_run_url_until_success(run_url, sleep=10): """ Given a URL to a scancode.io run instance, `run_url`, return True when the run instance has completed successfully. @@ -354,33 +354,60 @@ def poll_until_success(run_url, sleep=10): Raise a PurlDBException when the run instance has failed, stopped, or gone stale. """ - run_status = AbstractTaskFieldsModel.Status - while True: + if poll_until_success( + check=get_run_status, + sleep=sleep, + run_url=run_url + ): + return True + else: response = request_get(run_url) if response: - status = response["status"] - if status == run_status.SUCCESS: - return True - - if status in [ - run_status.NOT_STARTED, - run_status.QUEUED, - run_status.RUNNING, - ]: - continue + log = response["log"] + msg = f"Matching run has stopped:\n\n{log}" + raise PurlDBException(msg) - if status in [ - run_status.FAILURE, - run_status.STOPPED, - run_status.STALE, - ]: - log = response["log"] - msg = f"Matching run has stopped:\n\n{log}" - raise PurlDBException(msg) + +def poll_until_success(check, sleep=10, **kwargs): + """ + Given a function `check`, which returns the status of a run, return True + when the run instance has completed successfully. + + Return False when the run instance has failed, stopped, or gone stale. + + The arguments for `check` need to be provided as keyword argument into this + function. + """ + run_status = AbstractTaskFieldsModel.Status + while True: + status = check(**kwargs) + if status == run_status.SUCCESS: + return True + + if status in [ + run_status.NOT_STARTED, + run_status.QUEUED, + run_status.RUNNING, + ]: + continue + + if status in [ + run_status.FAILURE, + run_status.STOPPED, + run_status.STALE, + ]: + return False time.sleep(sleep) +def get_run_status(run_url, **kwargs): + response = request_get(run_url) + if response: + status = response["status"] + return status + + def get_match_results(run_url): """ Given the `run_url` for a pipeline running the matchcode matching pipeline, From 4b704063ebb64533230133119cb65890cec3558c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Feb 2024 19:16:43 -0800 Subject: [PATCH 09/30] Add tests for new purldb functions Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 4 +- scanpipe/pipes/purldb.py | 45 ++- scanpipe/tests/pipes/test_purldb.py | 275 ++++++++++++++++++ scanpipe/tests/test_commands.py | 8 + 4 files changed, 329 insertions(+), 3 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 9b5d838da7..dd92ea9602 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -119,7 +119,7 @@ def create_project_name(download_url, scannable_uri_uuid): return f"{slugify(download_url)}-{scannable_uri_uuid[0:8]}" -def poll_run_status(command, project, scannable_uri_uuid, sleep): +def poll_run_status(command, project, sleep=10): """ Poll the status of the first run of `project`. Return the log of the run if the run has stopped, failed, or gone stale, otherwise return an empty @@ -128,6 +128,7 @@ def poll_run_status(command, project, scannable_uri_uuid, sleep): run = project.runs.first() if purldb.poll_until_success( check=get_run_status, + sleep=sleep, run=run ): return "" @@ -138,5 +139,6 @@ def poll_run_status(command, project, scannable_uri_uuid, sleep): def get_run_status(run, **kwargs): + """Refresh the values of `run` and return its status""" run.refresh_from_db() return run.status diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 4644fa6cb8..611aa8b311 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -30,7 +30,16 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import InvalidVersionRange +from django.utils.text import slugify +from scanpipe.models import AbstractTaskFieldsModel from scanpipe.pipes import LoopProgress +from scanpipe.pipes import flag +from scanpipe.pipes.output import to_json + + +class PurlDBException(Exception): + pass + label = "PurlDB" logger = logging.getLogger(__name__) @@ -355,7 +364,7 @@ def poll_run_url_until_success(run_url, sleep=10): stale. """ if poll_until_success( - check=get_run_status, + check=get_run_url_status, sleep=sleep, run_url=run_url ): @@ -401,7 +410,7 @@ def poll_until_success(check, sleep=10, **kwargs): time.sleep(sleep) -def get_run_status(run_url, **kwargs): +def get_run_url_status(run_url, **kwargs): response = request_get(run_url) if response: status = response["status"] @@ -525,3 +534,35 @@ def update_status( data=data, ) return response + + +def create_project_name(download_url, scannable_uri_uuid): + """Create a project name from `download_url` and `scannable_uri_uuid`""" + if len(download_url) > 50: + download_url = download_url[0:50] + return f"{slugify(download_url)}-{scannable_uri_uuid[0:8]}" + + +def poll_run_status(command, project, sleep=10): + """ + Poll the status of the first run of `project`. Return the log of the run if + the run has stopped, failed, or gone stale, otherwise return an empty + string. + """ + run = project.runs.first() + if poll_until_success( + check=get_run_status, + sleep=sleep, + run=run + ): + return "" + else: + error_log = run.log + command.stderr.write(error_log) + return error_log + + +def get_run_status(run, **kwargs): + """Refresh the values of `run` and return its status""" + run.refresh_from_db() + return run.status diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 8a418e963d..f020f34cfb 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -102,3 +102,278 @@ def mock_request_post_return(url, data, headers, timeout): self.assertIn( "1 PURLs were already present in PurlDB index queue", expected_log ) + + @mock.patch("scanpipe.pipes.purldb.request_post") + @mock.patch("scanpipe.pipes.purldb.is_available") + def test_scanpipe_pipes_purldb_send_project_json_to_matchcode( + self, mock_is_available, mock_request_post + ): + mock_is_available.return_value = True + + def mock_request_post_return(url, files, timeout): + request_post_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_post_response.json" + ) + with open(request_post_response_loc, "r") as f: + return json.load(f) + + mock_request_post.side_effect = mock_request_post_return + + run_url = purldb.send_project_json_to_matchcode(self.project1) + expected_run_url = ( + "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + ) + self.assertEqual(expected_run_url, run_url) + + @mock.patch("scanpipe.pipes.purldb.request_get") + @mock.patch("scanpipe.pipes.purldb.is_available") + def test_scanpipe_pipes_purldb_poll_run_url_until_success( + self, mock_is_available, mock_request_get + ): + run_status = AbstractTaskFieldsModel.Status + + mock_is_available.return_value = True + + # Success + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.SUCCESS, + }, + ] + return_value = purldb.poll_run_url_until_success(run_url) + self.assertEqual(True, return_value) + + # Failure + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.FAILURE, + "log": "failure message", + }, + { + "url": run_url, + "status": run_status.FAILURE, + "log": "failure message", + }, + ] + with self.assertRaises(Exception) as context: + purldb.poll_run_url_until_success(run_url) + self.assertTrue("failure message" in str(context.exception)) + + # Stopped + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.STOPPED, + "log": "stop message", + }, + { + "url": run_url, + "status": run_status.STOPPED, + "log": "stop message", + }, + ] + with self.assertRaises(Exception) as context: + purldb.poll_run_url_until_success(run_url) + self.assertTrue("stop message" in str(context.exception)) + + # Stale + mock_request_get.side_effect = [ + { + "url": run_url, + "status": run_status.NOT_STARTED, + }, + { + "url": run_url, + "status": run_status.QUEUED, + }, + { + "url": run_url, + "status": run_status.RUNNING, + }, + { + "url": run_url, + "status": run_status.STALE, + "log": "stale message", + }, + { + "url": run_url, + "status": run_status.STALE, + "log": "stale message", + }, + ] + with self.assertRaises(Exception) as context: + purldb.poll_run_url_until_success(run_url) + self.assertTrue("stale message" in str(context.exception)) + + def test_scanpipe_pipes_purldb_map_match_results(self): + request_post_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_get_results_response.json" + ) + with open(request_post_response_loc, "r") as f: + match_results = json.load(f) + + resource_paths_by_package_uids = purldb.map_match_results(match_results) + expected = defaultdict(list) + expected_package_uid = ( + "pkg:maven/org.elasticsearch/elasticsearch-x-content@7.17.9" + "?classifier=sources&uuid=a8814800-8120-4f50-ba4f-08c443ccda8e" + ) + expected[expected_package_uid].append( + "elasticsearch-x-content-7.17.9-sources.jar" + ) + self.assertEqual(expected, resource_paths_by_package_uids) + + def test_scanpipe_pipes_purldb_create_packages_from_match_results(self): + r1 = make_resource_file( + self.project1, + path="elasticsearch-x-content-7.17.9-sources.jar", + sha1="30d21add57abe04beece3f28a079671dbc9043e4", + ) + r2 = make_resource_file( + self.project1, + path="something-else.json", + sha1="deadbeef", + ) + + request_get_results_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_get_results_response.json" + ) + with open(request_get_results_response_loc, "r") as f: + match_results = json.load(f) + + self.assertEqual(0, self.project1.discoveredpackages.all().count()) + self.assertFalse(0, len(r1.for_packages)) + self.assertFalse(0, len(r2.for_packages)) + + purldb.create_packages_from_match_results(self.project1, match_results) + + self.assertEqual(1, self.project1.discoveredpackages.all().count()) + package = self.project1.discoveredpackages.first() + self.assertEqual([package.package_uid], r1.for_packages) + # This resource should not have a Package match + self.assertFalse(0, len(r2.for_packages)) + + @mock.patch("scanpipe.pipes.purldb.request_get") + @mock.patch("scanpipe.pipes.purldb.is_available") + def test_scanpipe_pipes_purldb_get_match_results( + self, mock_is_available, mock_request_get + ): + mock_is_available.return_value = True + + request_get_check_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_get_check_response.json" + ) + with open(request_get_check_response_loc, "r") as f: + mock_request_get_check_return = json.load(f) + + request_get_results_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_get_results_response.json" + ) + with open(request_get_results_response_loc, "r") as f: + mock_request_get_results_return = json.load(f) + mock_request_get.side_effect = [ + mock_request_get_check_return, + mock_request_get_results_return, + ] + + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + match_results = purldb.get_match_results(run_url) + + self.assertEqual(mock_request_get_results_return, match_results) + + @mock.patch("scanpipe.pipes.purldb.request_get") + @mock.patch("scanpipe.pipes.purldb.is_available") + def test_scanpipe_pipes_purldb_get_run_url_status( + self, mock_is_available, mock_request_get + ): + mock_is_available.return_value = True + + request_get_check_response_loc = ( + self.data_location + / "purldb" + / "match_to_purldb" + / "request_get_check_response.json" + ) + with open(request_get_check_response_loc, "r") as f: + mock_request_get_check_return = json.load(f) + + mock_request_get.side_effect = [ + mock_request_get_check_return, + ] + + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + status = purldb.get_run_url_status(run_url) + + self.assertEqual("success", status) + + @mock.patch("scanpipe.pipes.purldb.request_get") + @mock.patch("scanpipe.pipes.purldb.is_available") + def test_scanpipe_pipes_purldb_get_next_job( + self, mock_is_available, mock_request_get + ): + mock_is_available.return_value = True + expected_download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" + expected_scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" + mock_request_get.side_effect = [ + { + "download_url": expected_download_url, + "scannable_uri_uuid": expected_scannable_uri_uuid, + }, + ] + download_url, scannable_uri_uuid = purldb.get_next_job() + self.assertEqual(expected_download_url, download_url) + self.assertEqual(expected_scannable_uri_uuid, scannable_uri_uuid) diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index d820ade5f7..e22c03b0c8 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -34,6 +34,7 @@ from django.test import override_settings from django.utils import timezone +from scanpipe.pipes import purldb from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project @@ -595,3 +596,10 @@ def test_scanpipe_management_command_create_user(self): ) with self.assertRaisesMessage(CommandError, expected): call_command("create-user", "--no-input", username) + +class PackageScanWorkerManagementCommandTest(TestCase): + def test_package_scan_worker_management_command_create_project_name(self): + download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" + scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" + project_name = purldb.create_project_name(download_url, scannable_uri_uuid) + self.assertEqual("httpsregistrynpmjsorgasdf-asdf-101tgz-52b2930d", project_name) From 7a7fd04d16488b051361e015790d106cce438671 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 1 Mar 2024 17:35:03 -0800 Subject: [PATCH 10/30] Collect possible exceptions when getting next job Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 61 ++++++------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index dd92ea9602..d060bfa497 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -25,7 +25,6 @@ from django.core.management import call_command from django.core.management.base import BaseCommand -from django.utils.text import slugify from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import create_project @@ -48,19 +47,23 @@ def handle(self, *args, **options): sleep = options["sleep"] while True: - # 1. get download url from purldb - response = purldb.get_next_job() - if response: - download_url, scannable_uri_uuid = response - else: - self.stdout.write("bad response") + try: + # 1. get download url from purldb + response = purldb.get_next_job() + if response: + download_url, scannable_uri_uuid = response + else: + self.stderr.write("bad response") + except Exception as e: + traceback = get_traceback_from_exception(e) + self.stderr.write(f"exception occured when calling `purldb.get_next_job()`:\n\n{traceback}") if not download_url or not scannable_uri_uuid: self.stdout.write("no new job") else: try: # 2. create and run project - name = create_project_name(download_url, scannable_uri_uuid) + name = purldb.create_project_name(download_url, scannable_uri_uuid) pipelines = ["scan_and_fingerprint_package"] input_urls = [download_url] project = create_project( @@ -78,7 +81,7 @@ def handle(self, *args, **options): ) # 3. poll project results - error_log = poll_run_status( + error_log = purldb.poll_run_status( command=self, project=project, scannable_uri_uuid=scannable_uri_uuid, @@ -100,45 +103,19 @@ def handle(self, *args, **options): ) except Exception as e: - traceback = "" - if hasattr(e, "__traceback__"): - traceback = "".join(format_tb(e.__traceback__)) + traceback = get_traceback_from_exception(e) purldb.update_status( scannable_uri_uuid, status="failed", scan_log=traceback, ) + self.stderr.write(f"exception occured during scan project:\n\n{traceback}") time.sleep(sleep) -def create_project_name(download_url, scannable_uri_uuid): - """Create a project name from `download_url` and `scannable_uri_uuid`""" - if len(download_url) > 50: - download_url = download_url[0:50] - return f"{slugify(download_url)}-{scannable_uri_uuid[0:8]}" - - -def poll_run_status(command, project, sleep=10): - """ - Poll the status of the first run of `project`. Return the log of the run if - the run has stopped, failed, or gone stale, otherwise return an empty - string. - """ - run = project.runs.first() - if purldb.poll_until_success( - check=get_run_status, - sleep=sleep, - run=run - ): - return "" - else: - error_log = run.log - command.stderr.write(error_log) - return error_log - - -def get_run_status(run, **kwargs): - """Refresh the values of `run` and return its status""" - run.refresh_from_db() - return run.status +def get_traceback_from_exception(exception): + traceback = "" + if hasattr(exception, "__traceback__"): + traceback = "".join(format_tb(exception.__traceback__)) + return traceback From 05a684bfc43c66f0c50b9c72fa804a2b74bc268c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 5 Mar 2024 19:14:28 -0800 Subject: [PATCH 11/30] Get pipelines to run from purldb Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 10 +++---- scanpipe/pipes/purldb.py | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index d060bfa497..822bd1512b 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -51,7 +51,7 @@ def handle(self, *args, **options): # 1. get download url from purldb response = purldb.get_next_job() if response: - download_url, scannable_uri_uuid = response + scannable_uri_uuid, download_url, pipelines = response else: self.stderr.write("bad response") except Exception as e: @@ -64,7 +64,6 @@ def handle(self, *args, **options): try: # 2. create and run project name = purldb.create_project_name(download_url, scannable_uri_uuid) - pipelines = ["scan_and_fingerprint_package"] input_urls = [download_url] project = create_project( self, @@ -84,7 +83,6 @@ def handle(self, *args, **options): error_log = purldb.poll_run_status( command=self, project=project, - scannable_uri_uuid=scannable_uri_uuid, sleep=sleep, ) @@ -103,13 +101,13 @@ def handle(self, *args, **options): ) except Exception as e: - traceback = get_traceback_from_exception(e) + error_log = f"exception occured during scan project:\n\n{str(e)}" purldb.update_status( scannable_uri_uuid, status="failed", - scan_log=traceback, + scan_log=error_log, ) - self.stderr.write(f"exception occured during scan project:\n\n{traceback}") + self.stderr.write(error_log) time.sleep(sleep) diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 611aa8b311..097c389241 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -482,9 +482,10 @@ def get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): timeout=timeout, ) if response: - download_url = response["download_url"] scannable_uri_uuid = response["scannable_uri_uuid"] - return download_url, scannable_uri_uuid + download_url = response["download_url"] + pipelines = response["pipelines"] + return scannable_uri_uuid, download_url, pipelines def send_results_to_purldb( @@ -545,21 +546,21 @@ def create_project_name(download_url, scannable_uri_uuid): def poll_run_status(command, project, sleep=10): """ - Poll the status of the first run of `project`. Return the log of the run if + Poll the status of all runs of `project`. Return the log of the run if the run has stopped, failed, or gone stale, otherwise return an empty string. """ - run = project.runs.first() - if poll_until_success( - check=get_run_status, - sleep=sleep, - run=run - ): - return "" - else: - error_log = run.log - command.stderr.write(error_log) - return error_log + runs = project.runs.all() + for run in runs: + if not poll_until_success( + check=get_run_status, + sleep=sleep, + run=run + ): + error_log = run.log + command.stderr.write(error_log) + return error_log + return "" def get_run_status(run, **kwargs): From 28883864043896b4ef0bba3247bc375f7f287680 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 6 Mar 2024 15:24:10 -0800 Subject: [PATCH 12/30] Do not use sys.exit() in execute.py Signed-off-by: Jono Yang --- scanpipe/management/commands/execute.py | 35 ++++++++++++------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/scanpipe/management/commands/execute.py b/scanpipe/management/commands/execute.py index d44621277f..9715aac338 100644 --- a/scanpipe/management/commands/execute.py +++ b/scanpipe/management/commands/execute.py @@ -60,24 +60,23 @@ def handle(self, *args, **options): run.start() msg = f"{run.pipeline_name} added to the tasks queue for execution." self.stdout.write(msg, self.style.SUCCESS) - sys.exit(0) - - self.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") + else: + self.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") - try: - tasks.execute_pipeline_task(run.pk) - except KeyboardInterrupt: - run.set_task_stopped() - raise CommandError("Pipeline execution stopped.") - except Exception as e: - run.set_task_ended(exitcode=1, output=str(e)) - raise CommandError(e) + try: + tasks.execute_pipeline_task(run.pk) + except KeyboardInterrupt: + run.set_task_stopped() + raise CommandError("Pipeline execution stopped.") + except Exception as e: + run.set_task_ended(exitcode=1, output=str(e)) + raise CommandError(e) - run.refresh_from_db() + run.refresh_from_db() - if run.task_succeeded: - msg = f"{run.pipeline_name} successfully executed on project {self.project}" - self.stdout.write(msg, self.style.SUCCESS) - else: - msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" - raise CommandError(msg) + if run.task_succeeded: + msg = f"{run.pipeline_name} successfully executed on project {self.project}" + self.stdout.write(msg, self.style.SUCCESS) + else: + msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" + raise CommandError(msg) From 4bebc21dbc368228e4dcbac6d34b5d8e6598b75a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 6 Mar 2024 19:18:25 -0800 Subject: [PATCH 13/30] Add async arg to package-scan-worker.py Signed-off-by: Jono Yang --- .../management/commands/package-scan-worker.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 822bd1512b..517fbca4de 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -43,6 +43,16 @@ def add_arguments(self, parser): help="Number in seconds how long the loop should sleep for before polling.", ) + parser.add_argument( + "--async", + action="store_true", + dest="async", + help=( + "Add the pipeline runs to the tasks queue for execution by a worker " + "instead of running in the current thread." + ), + ) + def handle(self, *args, **options): sleep = options["sleep"] @@ -72,14 +82,18 @@ def handle(self, *args, **options): input_urls=input_urls, ) + # TODO: test this from the docker context + # TODO: refactor execute to run project without having to call the execute command through cli call_command( "execute", project=project, stderr=self.stderr, stdout=self.stdout, + **{"async": options["async"]}, ) # 3. poll project results + # TODO: see if we can block waiting for a signal when the project is done running error_log = purldb.poll_run_status( command=self, project=project, From 1fffca41236ed4150b7f0db647e9c0b0341a8f6c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 11 Mar 2024 19:59:12 -0700 Subject: [PATCH 14/30] Refactor commonly used command functions Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 124 ++++++++++++++---- .../management/commands/create-project.py | 31 +---- scanpipe/management/commands/execute.py | 56 +------- .../commands/package-scan-worker.py | 85 +++++------- scanpipe/pipes/purldb.py | 26 ++-- scanpipe/tests/pipes/test_purldb.py | 93 ++++++++++++- scanpipe/tests/test_commands.py | 3 +- 7 files changed, 238 insertions(+), 180 deletions(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 5a980c9a98..31ab7bce9d 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -24,12 +24,14 @@ from pathlib import Path from django.apps import apps +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.core.management.base import BaseCommand from django.core.management.base import CommandError from django.template.defaultfilters import pluralize +from scanpipe import tasks from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project @@ -291,41 +293,105 @@ def validate_pipelines(pipelines_data): return pipelines_data -def create_project( - command, name, pipelines=[], input_files=[], input_urls=[], copy_from="", notes="" -): - project = Project(name=name) - if notes: - project.notes = notes +class ExecuteProjectCommandMixin: + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + "--async", + action="store_true", + dest="async", + help=( + "Add the pipeline run to the tasks queue for execution by a worker " + "instead of running in the current thread." + ), + ) + + def execute_project(self, run_async=False): + run = self.project.get_next_run() + + if not run: + raise CommandError(f"No pipelines to run on project {self.project}") - try: - project.full_clean(exclude=["slug"]) - except ValidationError as e: - raise CommandError("\n".join(e.messages)) + if run_async: + if not settings.SCANCODEIO_ASYNC: + msg = "SCANCODEIO_ASYNC=False is not compatible with --async option." + raise CommandError(msg) - # Run validation before creating the project in the database - pipelines_data = extract_group_from_pipelines(pipelines) - pipelines_data = validate_pipelines(pipelines_data) + run.start() + msg = f"{run.pipeline_name} added to the tasks queue for execution." + self.stdout.write(msg, self.style.SUCCESS) + else: + self.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") + + try: + tasks.execute_pipeline_task(run.pk) + except KeyboardInterrupt: + run.set_task_stopped() + raise CommandError("Pipeline execution stopped.") + except Exception as e: + run.set_task_ended(exitcode=1, output=str(e)) + raise CommandError(e) + + run.refresh_from_db() + + if run.task_succeeded: + msg = ( + f"{run.pipeline_name} successfully executed on " + f"project {self.project}" + ) + self.stdout.write(msg, self.style.SUCCESS) + else: + msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" + raise CommandError(msg) + + +class CreateProjectCommandMixin(ExecuteProjectCommandMixin): + def create_project( + self, + name, + pipelines=[], + input_files=[], + input_urls=[], + copy_from="", + notes="", + execute=False, + run_async=False, + ): + project = Project(name=name) + if notes: + project.notes = notes + + try: + project.full_clean(exclude=["slug"]) + except ValidationError as e: + raise CommandError("\n".join(e.messages)) - input_files_data = command.extract_tag_from_input_files(input_files) - command.validate_input_files(input_files=input_files_data.keys()) - validate_copy_from(copy_from) + # Run validation before creating the project in the database + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) - project.save() - command.project = project - msg = f"Project {name} created with work directory {project.work_directory}" - command.stdout.write(msg, command.style.SUCCESS) + input_files_data = self.extract_tag_from_input_files(input_files) + self.validate_input_files(input_files=input_files_data.keys()) + validate_copy_from(copy_from) - for pipeline_name, selected_groups in pipelines_data.items(): - command.project.add_pipeline(pipeline_name, selected_groups=selected_groups) + project.save() + self.project = project + msg = f"Project {name} created with work directory {project.work_directory}" + self.stdout.write(msg, self.style.SUCCESS) - if input_files: - command.handle_input_files(input_files_data) + for pipeline_name, selected_groups in pipelines_data.items(): + self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) - if input_urls: - command.handle_input_urls(input_urls) + if input_files: + self.handle_input_files(input_files_data) - if copy_from: - command.handle_copy_codebase(copy_from) + if input_urls: + self.handle_input_urls(input_urls) + + if copy_from: + self.handle_copy_codebase(copy_from) + + if execute: + self.execute_project(run_async=run_async) - return project + return project diff --git a/scanpipe/management/commands/create-project.py b/scanpipe/management/commands/create-project.py index 5bfbee7eaf..cc35c578d3 100644 --- a/scanpipe/management/commands/create-project.py +++ b/scanpipe/management/commands/create-project.py @@ -21,14 +21,13 @@ # Visit https://github.com/nexB/scancode.io for support and download. from django.core.management import CommandError -from django.core.management import call_command from django.core.management.base import BaseCommand from scanpipe.management.commands import AddInputCommandMixin -from scanpipe.management.commands import create_project +from scanpipe.management.commands import CreateProjectCommandMixin -class Command(AddInputCommandMixin, BaseCommand): +class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand): help = "Create a ScanPipe project." def add_arguments(self, parser): @@ -50,15 +49,6 @@ def add_arguments(self, parser): action="store_true", help="Execute the pipelines right after the project creation.", ) - parser.add_argument( - "--async", - action="store_true", - help=( - "Add the pipeline run to the tasks queue for execution by a worker " - "instead of running in the current thread. " - "Applies only when --execute is provided." - ), - ) parser.add_argument( "--notes", help="Optional notes about the project.", @@ -70,27 +60,20 @@ def handle(self, *args, **options): input_files = options["input_files"] input_urls = options["input_urls"] copy_from = options["copy_codebase"] - execute = options["execute"] notes = options["notes"] + execute = options["execute"] + run_async = options["async"] if execute and not pipelines: raise CommandError("The --execute option requires one or more pipelines.") - project = create_project( - command=self, + self.create_project( name=name, pipelines=pipelines, input_files=input_files, input_urls=input_urls, copy_from=copy_from, notes=notes, + execute=execute, + run_async=run_async, ) - - if execute: - call_command( - "execute", - project=project, - stderr=self.stderr, - stdout=self.stdout, - **{"async": options["async"]}, - ) diff --git a/scanpipe/management/commands/execute.py b/scanpipe/management/commands/execute.py index 9715aac338..ce2f28b589 100644 --- a/scanpipe/management/commands/execute.py +++ b/scanpipe/management/commands/execute.py @@ -20,63 +20,13 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -import sys - -from django.conf import settings -from django.core.management import CommandError - -from scanpipe import tasks +from scanpipe.management.commands import ExecuteProjectCommandMixin from scanpipe.management.commands import ProjectCommand -class Command(ProjectCommand): +class Command(ExecuteProjectCommandMixin, ProjectCommand): help = "Run pipelines on a project." - def add_arguments(self, parser): - super().add_arguments(parser) - parser.add_argument( - "--async", - action="store_true", - dest="async", - help=( - "Add the pipeline run to the tasks queue for execution by a worker " - "instead of running in the current thread." - ), - ) - def handle(self, *args, **options): super().handle(*args, **options) - - run = self.project.get_next_run() - - if not run: - raise CommandError(f"No pipelines to run on project {self.project}") - - if options["async"]: - if not settings.SCANCODEIO_ASYNC: - msg = "SCANCODEIO_ASYNC=False is not compatible with --async option." - raise CommandError(msg) - - run.start() - msg = f"{run.pipeline_name} added to the tasks queue for execution." - self.stdout.write(msg, self.style.SUCCESS) - else: - self.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") - - try: - tasks.execute_pipeline_task(run.pk) - except KeyboardInterrupt: - run.set_task_stopped() - raise CommandError("Pipeline execution stopped.") - except Exception as e: - run.set_task_ended(exitcode=1, output=str(e)) - raise CommandError(e) - - run.refresh_from_db() - - if run.task_succeeded: - msg = f"{run.pipeline_name} successfully executed on project {self.project}" - self.stdout.write(msg, self.style.SUCCESS) - else: - msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" - raise CommandError(msg) + self.execute_project(run_async=options["async"]) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 517fbca4de..c71fbd3c6f 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -21,18 +21,15 @@ # Visit https://github.com/nexB/scancode.io for support and download. import time -from traceback import format_tb -from django.core.management import call_command from django.core.management.base import BaseCommand -from scanpipe.management.commands import AddInputCommandMixin -from scanpipe.management.commands import create_project +from scanpipe.management.commands import CreateProjectCommandMixin from scanpipe.pipes import output from scanpipe.pipes import purldb -class Command(AddInputCommandMixin, BaseCommand): +class Command(CreateProjectCommandMixin, BaseCommand): help = "Create a ScanPipe project." def add_arguments(self, parser): @@ -43,79 +40,57 @@ def add_arguments(self, parser): help="Number in seconds how long the loop should sleep for before polling.", ) - parser.add_argument( - "--async", - action="store_true", - dest="async", - help=( - "Add the pipeline runs to the tasks queue for execution by a worker " - "instead of running in the current thread." - ), - ) - def handle(self, *args, **options): sleep = options["sleep"] + run_async = options["async"] while True: - try: - # 1. get download url from purldb - response = purldb.get_next_job() - if response: - scannable_uri_uuid, download_url, pipelines = response - else: - self.stderr.write("bad response") - except Exception as e: - traceback = get_traceback_from_exception(e) - self.stderr.write(f"exception occured when calling `purldb.get_next_job()`:\n\n{traceback}") + time.sleep(sleep) + + # 1. Get download url from purldb + scannable_uri_uuid, download_url, pipelines, error_msg = get_next_job() + if error_msg: + self.stderr.write(error_msg) + continue if not download_url or not scannable_uri_uuid: - self.stdout.write("no new job") + self.stdout.write("No new job from PurlDB.") else: try: - # 2. create and run project + # 2. Create and run project name = purldb.create_project_name(download_url, scannable_uri_uuid) input_urls = [download_url] - project = create_project( - self, + project = self.create_project( name=name, pipelines=pipelines, input_urls=input_urls, + execute=run_async, ) - # TODO: test this from the docker context - # TODO: refactor execute to run project without having to call the execute command through cli - call_command( - "execute", - project=project, - stderr=self.stderr, - stdout=self.stdout, - **{"async": options["async"]}, - ) - - # 3. poll project results - # TODO: see if we can block waiting for a signal when the project is done running + # 3. Poll project results + # TODO: see if we can block waiting for a signal when the + # project is done running error_log = purldb.poll_run_status( - command=self, project=project, sleep=sleep, ) if error_log: - # send error response to purldb + # Send error response to PurlDB purldb.update_status( scannable_uri_uuid, status="failed", scan_log=error_log, ) else: - # 4. get project results and send to purldb + # 4. Get project results and send to PurlDB scan_output_location = output.to_json(project) purldb.send_results_to_purldb( scannable_uri_uuid, scan_output_location ) except Exception as e: - error_log = f"exception occured during scan project:\n\n{str(e)}" + error_log = f"Exception occured during scan project:\n\n{str(e)}" purldb.update_status( scannable_uri_uuid, status="failed", @@ -123,11 +98,17 @@ def handle(self, *args, **options): ) self.stderr.write(error_log) - time.sleep(sleep) - -def get_traceback_from_exception(exception): - traceback = "" - if hasattr(exception, "__traceback__"): - traceback = "".join(format_tb(exception.__traceback__)) - return traceback +def get_next_job(): + # 1. Get download url from purldb + scannable_uri_uuid, download_url, pipelines = None + msg = "" + try: + response = purldb.get_next_job() + if response: + scannable_uri_uuid, download_url, pipelines = response + else: + msg = "Bad response from PurlDB, unable to get next job." + except Exception as e: + msg = f"Exception occured when calling `purldb.get_next_job()`:\n\n{str(e)}" + return scannable_uri_uuid, download_url, pipelines, msg diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 097c389241..40be6accea 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -24,6 +24,7 @@ import logging from django.conf import settings +from django.utils.text import slugify import requests from packageurl import PackageURL @@ -355,7 +356,7 @@ def find_packages(payload): return response.get("results") -def poll_run_url_until_success(run_url, sleep=10): +def poll_run_url_status(run_url, sleep=10): """ Given a URL to a scancode.io run instance, `run_url`, return True when the run instance has completed successfully. @@ -363,11 +364,7 @@ def poll_run_url_until_success(run_url, sleep=10): Raise a PurlDBException when the run instance has failed, stopped, or gone stale. """ - if poll_until_success( - check=get_run_url_status, - sleep=sleep, - run_url=run_url - ): + if poll_until_success(check=get_run_url_status, sleep=sleep, run_url=run_url): return True else: response = request_get(run_url) @@ -411,6 +408,10 @@ def poll_until_success(check, sleep=10, **kwargs): def get_run_url_status(run_url, **kwargs): + """ + Given a `run_url`, which is a URL to a ScanCode.io Project run, return its + status, otherwise return None. + """ response = request_get(run_url) if response: status = response["status"] @@ -523,6 +524,7 @@ def update_status( timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): + """Update the status of a ScannableURI on a PurlDB scan queue""" data = { "scannable_uri_uuid": scannable_uri_uuid, "scan_status": status, @@ -544,7 +546,7 @@ def create_project_name(download_url, scannable_uri_uuid): return f"{slugify(download_url)}-{scannable_uri_uuid[0:8]}" -def poll_run_status(command, project, sleep=10): +def poll_run_status(project, sleep=10): """ Poll the status of all runs of `project`. Return the log of the run if the run has stopped, failed, or gone stale, otherwise return an empty @@ -552,14 +554,8 @@ def poll_run_status(command, project, sleep=10): """ runs = project.runs.all() for run in runs: - if not poll_until_success( - check=get_run_status, - sleep=sleep, - run=run - ): - error_log = run.log - command.stderr.write(error_log) - return error_log + if not poll_until_success(check=get_run_status, sleep=sleep, run=run): + return run.log return "" diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index f020f34cfb..7808263fb4 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -25,11 +25,13 @@ from unittest import mock from django.test import TestCase +from django.utils import timezone from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project +from scanpipe.models import Run from scanpipe.pipes import purldb from scanpipe.tests import dependency_data2 from scanpipe.tests import dependency_data3 @@ -38,9 +40,18 @@ class ScanPipePurlDBTest(TestCase): data_location = Path(__file__).parent.parent / "data" + fixtures = [data_location / "asgiref-3.3.0_fixtures.json"] def setUp(self): self.project1 = Project.objects.create(name="Analysis") + self.project_asgiref = Project.objects.get(name="asgiref") + + def create_run(self, pipeline="pipeline", **kwargs): + return Run.objects.create( + project=self.project1, + pipeline_name=pipeline, + **kwargs, + ) def test_scanpipe_pipes_purldb_get_unique_resolved_purls(self): DiscoveredPackage.create_from_data(self.project1, package_data1) @@ -130,7 +141,7 @@ def mock_request_post_return(url, files, timeout): @mock.patch("scanpipe.pipes.purldb.request_get") @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_poll_run_url_until_success( + def test_scanpipe_pipes_purldb_poll_run_url_status( self, mock_is_available, mock_request_get ): run_status = AbstractTaskFieldsModel.Status @@ -157,7 +168,7 @@ def test_scanpipe_pipes_purldb_poll_run_url_until_success( "status": run_status.SUCCESS, }, ] - return_value = purldb.poll_run_url_until_success(run_url) + return_value = purldb.poll_run_url_status(run_url) self.assertEqual(True, return_value) # Failure @@ -186,7 +197,7 @@ def test_scanpipe_pipes_purldb_poll_run_url_until_success( }, ] with self.assertRaises(Exception) as context: - purldb.poll_run_url_until_success(run_url) + purldb.poll_run_url_status(run_url) self.assertTrue("failure message" in str(context.exception)) # Stopped @@ -215,7 +226,7 @@ def test_scanpipe_pipes_purldb_poll_run_url_until_success( }, ] with self.assertRaises(Exception) as context: - purldb.poll_run_url_until_success(run_url) + purldb.poll_run_url_status(run_url) self.assertTrue("stop message" in str(context.exception)) # Stale @@ -244,7 +255,7 @@ def test_scanpipe_pipes_purldb_poll_run_url_until_success( }, ] with self.assertRaises(Exception) as context: - purldb.poll_run_url_until_success(run_url) + purldb.poll_run_url_status(run_url) self.assertTrue("stale message" in str(context.exception)) def test_scanpipe_pipes_purldb_map_match_results(self): @@ -368,12 +379,82 @@ def test_scanpipe_pipes_purldb_get_next_job( mock_is_available.return_value = True expected_download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" expected_scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" + expected_pipelines = ["scan_and_fingerprint_package"] mock_request_get.side_effect = [ { "download_url": expected_download_url, "scannable_uri_uuid": expected_scannable_uri_uuid, + "pipelines": expected_pipelines, }, ] - download_url, scannable_uri_uuid = purldb.get_next_job() + scannable_uri_uuid, download_url, pipelines = purldb.get_next_job() self.assertEqual(expected_download_url, download_url) self.assertEqual(expected_scannable_uri_uuid, scannable_uri_uuid) + self.assertEqual(expected_pipelines, pipelines) + + def test_scanpipe_pipes_purldb_poll_run_status(self): + now = timezone.now() + + # Test poll_run_status on individual pipelines + self.assertEqual(0, self.project1.runs.count()) + self.create_run( + pipeline="succeed", + task_start_date=now, + task_end_date=now, + task_exitcode=0, + ) + error_message = purldb.poll_run_status(project=self.project1) + self.assertEqual("", error_message) + self.project1.runs.all().delete() + + self.create_run( + pipeline="failed", + task_start_date=now, + task_end_date=now, + task_exitcode=1, + log="failed", + ) + error_message = purldb.poll_run_status(project=self.project1) + self.assertEqual("failed", error_message) + self.project1.runs.all().delete() + + self.create_run( + pipeline="stopped", + task_start_date=now, + task_end_date=now, + task_exitcode=99, + log="stopped", + ) + error_message = purldb.poll_run_status(project=self.project1) + self.assertEqual("stopped", error_message) + self.project1.runs.all().delete() + + self.create_run( + pipeline="stale", + task_start_date=now, + task_end_date=now, + task_exitcode=88, + log="stale", + ) + error_message = purldb.poll_run_status(project=self.project1) + self.assertEqual("stale", error_message) + self.project1.runs.all().delete() + + # Test pipelines success, then failure + self.assertEqual(0, self.project1.runs.count()) + self.create_run( + pipeline="succeed", + task_start_date=now, + task_end_date=now, + task_exitcode=0, + ) + self.create_run( + pipeline="failed", + task_start_date=now, + task_end_date=now, + task_exitcode=1, + log="failed", + ) + error_message = purldb.poll_run_status(project=self.project1) + self.assertEqual("failed", error_message) + self.project1.runs.all().delete() diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index e22c03b0c8..a77647f702 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -34,11 +34,11 @@ from django.test import override_settings from django.utils import timezone -from scanpipe.pipes import purldb from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import Run +from scanpipe.pipes import purldb scanpipe_app = apps.get_app_config("scanpipe") @@ -597,6 +597,7 @@ def test_scanpipe_management_command_create_user(self): with self.assertRaisesMessage(CommandError, expected): call_command("create-user", "--no-input", username) + class PackageScanWorkerManagementCommandTest(TestCase): def test_package_scan_worker_management_command_create_project_name(self): download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" From c494f02a7e4b1759567d177e0b2426327bfb9767 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 12 Mar 2024 20:04:45 -0700 Subject: [PATCH 15/30] Create tests for Command mixins Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 3 + scanpipe/tests/test_commands.py | 92 +++++++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 31ab7bce9d..1fb974f9a7 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -357,6 +357,9 @@ def create_project( execute=False, run_async=False, ): + if execute and not pipelines: + raise CommandError("The execute argument requires one or more pipelines.") + project = Project(name=name) if notes: project.notes = notes diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index a77647f702..960f98bded 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -25,8 +25,10 @@ from io import StringIO from pathlib import Path from unittest import mock - +import sys +from contextlib import redirect_stderr, redirect_stdout from django.apps import apps +from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from django.core.management import CommandError from django.core.management import call_command @@ -39,6 +41,7 @@ from scanpipe.models import Project from scanpipe.models import Run from scanpipe.pipes import purldb +from scanpipe.management import commands scanpipe_app = apps.get_app_config("scanpipe") @@ -598,6 +601,93 @@ def test_scanpipe_management_command_create_user(self): call_command("create-user", "--no-input", username) +class ScanPipeManagementCommandMixinTest(TestCase): + class CreateProjectCommand(commands.CreateProjectCommandMixin, commands.AddInputCommandMixin, BaseCommand): + pass + + create_project_command = CreateProjectCommand() + pipeline_name = "analyze_docker_image" + pipeline_class = scanpipe_app.pipelines.get(pipeline_name) + + def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_base(self): + expected = "This field cannot be blank." + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project(name="") + + project = self.create_project_command.create_project(name="my_project") + self.assertTrue("my_project", project.name) + + expected = "Project with this Name already exists." + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project(name="my_project") + + def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_notes(self): + notes = "Some notes about my project" + project = self.create_project_command.create_project(name="my_project", notes=notes) + self.assertEqual(notes, project.notes) + + def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_pipelines(self): + expected = "non-existing is not a valid pipeline" + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project(name="my_project", pipelines=["non-existing"]) + + pipelines = [ + self.pipeline_name, + "analyze_root_filesystem_or_vm_image:group1,group2", + "scan_package", + ] + project = self.create_project_command.create_project(name="my_project", pipelines=pipelines) + expected = [ + self.pipeline_name, + "analyze_root_filesystem_or_vm_image", + "scan_single_package", + ] + self.assertEqual(expected, [run.pipeline_name for run in project.runs.all()]) + run = project.runs.get(pipeline_name="analyze_root_filesystem_or_vm_image") + self.assertEqual(["group1", "group2"], run.selected_groups) + + def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_inputs(self): + expected = "non-existing not found or not a file" + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project(name="my_project", input_files=["non-existing"]) + + parent_path = Path(__file__).parent + input_files = [ + str(parent_path / "test_commands.py"), + str(parent_path / "test_models.py:tag"), + ] + project = self.create_project_command.create_project(name="my_project", input_files=input_files) + expected = sorted(["test_commands.py", "test_models.py"]) + self.assertEqual(expected, sorted(project.input_files)) + tagged_source = project.inputsources.get(filename="test_models.py") + self.assertEqual("tag", tagged_source.tag) + + def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_execute(self): + expected = "The execute argument requires one or more pipelines." + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project(name="my_project", execute=True) + + pipeline = "load_inventory" + with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): + project = self.create_project_command.create_project( + name="my_project", + pipelines=[pipeline], + execute=True + ) + run = project.runs.first() + self.assertTrue(run.task_succeeded) + + expected = "SCANCODEIO_ASYNC=False is not compatible with --async option." + with override_settings(SCANCODEIO_ASYNC=False): + with self.assertRaisesMessage(CommandError, expected): + self.create_project_command.create_project( + name="other_project", + pipelines=[pipeline], + execute=True, + run_async=True + ) + + class PackageScanWorkerManagementCommandTest(TestCase): def test_package_scan_worker_management_command_create_project_name(self): download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" From d500b5ef74c3118dea88221243cb63407debdf4c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 12 Mar 2024 20:23:22 -0700 Subject: [PATCH 16/30] Move get_next_job() to pipes/purldb.py Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 17 +---------- scanpipe/pipes/purldb.py | 20 +++++++++++-- scanpipe/tests/pipes/test_purldb.py | 30 ++++++++++++++++++- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index c71fbd3c6f..817cc54a67 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -48,7 +48,7 @@ def handle(self, *args, **options): time.sleep(sleep) # 1. Get download url from purldb - scannable_uri_uuid, download_url, pipelines, error_msg = get_next_job() + scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() if error_msg: self.stderr.write(error_msg) continue @@ -97,18 +97,3 @@ def handle(self, *args, **options): scan_log=error_log, ) self.stderr.write(error_log) - - -def get_next_job(): - # 1. Get download url from purldb - scannable_uri_uuid, download_url, pipelines = None - msg = "" - try: - response = purldb.get_next_job() - if response: - scannable_uri_uuid, download_url, pipelines = response - else: - msg = "Bad response from PurlDB, unable to get next job." - except Exception as e: - msg = f"Exception occured when calling `purldb.get_next_job()`:\n\n{str(e)}" - return scannable_uri_uuid, download_url, pipelines, msg diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 40be6accea..b4c5bf8b4f 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -471,10 +471,10 @@ def create_packages_from_match_results(project, match_results): ) -def get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): +def _get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): """ - Return the download URL and Package UUID of the next Package to be scanned - from PurlDB + Return the ScannableURI UUID, download URL, and pipelines for the next + Package to be scanned from PurlDB Return None if the request was not successful """ @@ -489,6 +489,20 @@ def get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): return scannable_uri_uuid, download_url, pipelines +def get_next_job(): + scannable_uri_uuid = download_url = pipelines = None + msg = "" + try: + response = _get_next_job() + if response: + scannable_uri_uuid, download_url, pipelines = response + else: + msg = "Bad response from PurlDB, unable to get next job." + except Exception as e: + msg = f"Exception occured when calling `purldb.get_next_job()`:\n\n{str(e)}" + return scannable_uri_uuid, download_url, pipelines, msg + + def send_results_to_purldb( scannable_uri_uuid, scan_output_location, diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 7808263fb4..bc0a1532f3 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -386,11 +386,39 @@ def test_scanpipe_pipes_purldb_get_next_job( "scannable_uri_uuid": expected_scannable_uri_uuid, "pipelines": expected_pipelines, }, + { + "download_url": "", + "scannable_uri_uuid": "", + "pipelines": [] + }, + None, + Exception() ] - scannable_uri_uuid, download_url, pipelines = purldb.get_next_job() + results = purldb.get_next_job() + self.assertTrue(results) + scannable_uri_uuid, download_url, pipelines, error_msg = results self.assertEqual(expected_download_url, download_url) self.assertEqual(expected_scannable_uri_uuid, scannable_uri_uuid) self.assertEqual(expected_pipelines, pipelines) + self.assertEqual("", error_msg) + + scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() + self.assertFalse(scannable_uri_uuid) + self.assertFalse(download_url) + self.assertFalse(pipelines) + self.assertEqual("", error_msg) + + scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() + self.assertFalse(scannable_uri_uuid) + self.assertFalse(download_url) + self.assertFalse(pipelines) + self.assertEqual("Bad response from PurlDB, unable to get next job.", error_msg) + + scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() + self.assertFalse(scannable_uri_uuid) + self.assertFalse(download_url) + self.assertFalse(pipelines) + self.assertIn("Exception occured when calling `purldb.get_next_job()`:", error_msg) def test_scanpipe_pipes_purldb_poll_run_status(self): now = timezone.now() From 085d034251f4998c34b650a1017fdc0b1b473146 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 13 Mar 2024 16:33:40 -0700 Subject: [PATCH 17/30] Create test for package-scan-worker Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 26 +++++++++++++-- scanpipe/tests/pipes/test_purldb.py | 6 ++++ scanpipe/tests/test_commands.py | 32 +++++++++++++++---- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 817cc54a67..48aed9fa8f 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -24,12 +24,12 @@ from django.core.management.base import BaseCommand -from scanpipe.management.commands import CreateProjectCommandMixin +from scanpipe.management.commands import CreateProjectCommandMixin, AddInputCommandMixin from scanpipe.pipes import output from scanpipe.pipes import purldb -class Command(CreateProjectCommandMixin, BaseCommand): +class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand): help = "Create a ScanPipe project." def add_arguments(self, parser): @@ -37,14 +37,31 @@ def add_arguments(self, parser): parser.add_argument( "--sleep", type=int, + default=0, + action='store', help="Number in seconds how long the loop should sleep for before polling.", ) + parser.add_argument( + "--max-loops", + dest="max_loops", + default=0, + action="store", + help="Limit the number of loops to a maximum number. " + "0 means no limit. Used only for testing." + ) + def handle(self, *args, **options): sleep = options["sleep"] run_async = options["async"] + max_loops = options["max_loops"] + loop_count = 0 while True: + if max_loops and int(loop_count) >= int(max_loops): + self.stdout.write("loop max reached") + break + time.sleep(sleep) # 1. Get download url from purldb @@ -64,7 +81,8 @@ def handle(self, *args, **options): name=name, pipelines=pipelines, input_urls=input_urls, - execute=run_async, + execute=True, + run_async=run_async, ) # 3. Poll project results @@ -97,3 +115,5 @@ def handle(self, *args, **options): scan_log=error_log, ) self.stderr.write(error_log) + + loop_count += 1 diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index bc0a1532f3..fe1650f2bd 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -486,3 +486,9 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): error_message = purldb.poll_run_status(project=self.project1) self.assertEqual("failed", error_message) self.project1.runs.all().delete() + + def test_scanpipe_pipes_purldb_create_project_name(self): + download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" + scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" + project_name = purldb.create_project_name(download_url, scannable_uri_uuid) + self.assertEqual("httpsregistrynpmjsorgasdf-asdf-101tgz-52b2930d", project_name) diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 960f98bded..f5cef2cc95 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -687,10 +687,30 @@ def test_scanpipe_management_command_mixin_create_project_command_mixin_create_p run_async=True ) + @mock.patch("scanpipe.pipes.purldb.request_post") + @mock.patch("scanpipe.pipes.purldb.request_get") + def test_scanpipe_management_command_package_scan_worker(self, mock_request_get, mock_request_post): + scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" + mock_request_get.return_value = { + "scannable_uri_uuid": scannable_uri_uuid, + "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", + "pipelines": ["scan_codebase"] + } + mock_request_post.return_value = { + 'status': f'scan indexed for scannable uri {scannable_uri_uuid}' + } -class PackageScanWorkerManagementCommandTest(TestCase): - def test_package_scan_worker_management_command_create_project_name(self): - download_url = "https://registry.npmjs.org/asdf/-/asdf-1.0.1.tgz" - scannable_uri_uuid = "52b2930d-6e85-4b3e-ba3e-17dd9a618650" - project_name = purldb.create_project_name(download_url, scannable_uri_uuid) - self.assertEqual("httpsregistrynpmjsorgasdf-asdf-101tgz-52b2930d", project_name) + options = [ + "--max-loops", + 1, + ] + out = StringIO() + with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): + call_command("package-scan-worker", *options, stdout=out) + + out_value = out.getvalue() + self.assertIn("Project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e created", out_value) + self.assertIn("File(s) downloaded to the project inputs directory:", out_value) + self.assertIn("asdf-1.2.2.tgz", out_value) + self.assertIn("scan_codebase successfully executed on project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e", out_value) + mock_request_post.assert_called_once() From 38bf97321db0ecb24f499c1de643aa14714ba4d6 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 14 Mar 2024 14:14:23 -0700 Subject: [PATCH 18/30] Create failure tests for package-scan-worker * Remove `scanscan_project_url` from scanpipe.pipes.purldb.update_status() Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 11 +- scanpipe/pipes/purldb.py | 2 - scanpipe/tests/pipes/test_purldb.py | 12 +- scanpipe/tests/test_commands.py | 249 ++++++++++++++---- 4 files changed, 214 insertions(+), 60 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 48aed9fa8f..43eb625ffd 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -24,7 +24,8 @@ from django.core.management.base import BaseCommand -from scanpipe.management.commands import CreateProjectCommandMixin, AddInputCommandMixin +from scanpipe.management.commands import AddInputCommandMixin +from scanpipe.management.commands import CreateProjectCommandMixin from scanpipe.pipes import output from scanpipe.pipes import purldb @@ -38,7 +39,7 @@ def add_arguments(self, parser): "--sleep", type=int, default=0, - action='store', + action="store", help="Number in seconds how long the loop should sleep for before polling.", ) @@ -48,7 +49,7 @@ def add_arguments(self, parser): default=0, action="store", help="Limit the number of loops to a maximum number. " - "0 means no limit. Used only for testing." + "0 means no limit. Used only for testing.", ) def handle(self, *args, **options): @@ -65,7 +66,9 @@ def handle(self, *args, **options): time.sleep(sleep) # 1. Get download url from purldb - scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() + scannable_uri_uuid, download_url, pipelines, error_msg = ( + purldb.get_next_job() + ) if error_msg: self.stderr.write(error_msg) continue diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index b4c5bf8b4f..3e5bcd9c68 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -534,7 +534,6 @@ def update_status( scannable_uri_uuid, status, scan_log="", - scan_project_url="", timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): @@ -543,7 +542,6 @@ def update_status( "scannable_uri_uuid": scannable_uri_uuid, "scan_status": status, "scan_log": scan_log, - "scan_project_url": scan_project_url, } response = request_post( url=f"{api_url}scan_queue/update_status/", diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index fe1650f2bd..4cb785bc93 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -386,13 +386,9 @@ def test_scanpipe_pipes_purldb_get_next_job( "scannable_uri_uuid": expected_scannable_uri_uuid, "pipelines": expected_pipelines, }, - { - "download_url": "", - "scannable_uri_uuid": "", - "pipelines": [] - }, + {"download_url": "", "scannable_uri_uuid": "", "pipelines": []}, None, - Exception() + Exception(), ] results = purldb.get_next_job() self.assertTrue(results) @@ -418,7 +414,9 @@ def test_scanpipe_pipes_purldb_get_next_job( self.assertFalse(scannable_uri_uuid) self.assertFalse(download_url) self.assertFalse(pipelines) - self.assertIn("Exception occured when calling `purldb.get_next_job()`:", error_msg) + self.assertIn( + "Exception occured when calling `purldb.get_next_job()`:", error_msg + ) def test_scanpipe_pipes_purldb_poll_run_status(self): now = timezone.now() diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index f5cef2cc95..c9c39996d1 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -25,23 +25,22 @@ from io import StringIO from pathlib import Path from unittest import mock -import sys -from contextlib import redirect_stderr, redirect_stdout + from django.apps import apps -from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from django.core.management import CommandError from django.core.management import call_command +from django.core.management.base import BaseCommand from django.test import TestCase from django.test import override_settings from django.utils import timezone +from scanpipe.management import commands from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import Run from scanpipe.pipes import purldb -from scanpipe.management import commands scanpipe_app = apps.get_app_config("scanpipe") @@ -66,6 +65,7 @@ def raise_interrupt(run_pk): class ScanPipeManagementCommandTest(TestCase): pipeline_name = "analyze_docker_image" pipeline_class = scanpipe_app.pipelines.get(pipeline_name) + purldb_update_status_url = f"{purldb.PURLDB_API_URL}scan_queue/update_status/" def test_scanpipe_management_command_create_project_base(self): out = StringIO() @@ -600,16 +600,183 @@ def test_scanpipe_management_command_create_user(self): with self.assertRaisesMessage(CommandError, expected): call_command("create-user", "--no-input", username) + @mock.patch("scanpipe.pipes.purldb.request_post") + @mock.patch("scanpipe.pipes.purldb.request_get") + def test_scanpipe_management_command_package_scan_worker( + self, mock_request_get, mock_request_post + ): + scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" + mock_request_get.return_value = { + "scannable_uri_uuid": scannable_uri_uuid, + "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", + "pipelines": ["scan_codebase"], + } + mock_request_post.return_value = { + "status": f"scan indexed for scannable uri {scannable_uri_uuid}" + } + + options = [ + "--max-loops", + 1, + ] + out = StringIO() + with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): + call_command("package-scan-worker", *options, stdout=out) + + out_value = out.getvalue() + self.assertIn( + "Project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e created", out_value + ) + self.assertIn("File(s) downloaded to the project inputs directory:", out_value) + self.assertIn("asdf-1.2.2.tgz", out_value) + self.assertIn( + "scan_codebase successfully executed on project " + "httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e", + out_value, + ) + mock_request_post.assert_called_once() + mock_request_post_call = mock_request_post.mock_calls[0] + mock_request_post_call_kwargs = mock_request_post_call.kwargs + self.assertEqual( + self.purldb_update_status_url, mock_request_post_call_kwargs["url"] + ) + expected_data = { + "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", + "scan_status": "scanned", + } + self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) + self.assertTrue(mock_request_post_call_kwargs["files"]["scan_file"]) + + @mock.patch("scanpipe.pipes.purldb.request_post") + @mock.patch("scanpipe.pipes.purldb.request_get") + def test_scanpipe_management_command_package_scan_worker_failure( + self, mock_request_get, mock_request_post + ): + scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" + mock_request_get.return_value = { + "scannable_uri_uuid": scannable_uri_uuid, + "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", + "pipelines": ["scan_codebase"], + } + mock_request_post.return_value = { + "status": f"scan failed for scannable uri {scannable_uri_uuid}" + } + + options = [ + "--max-loops", + 1, + ] + out = StringIO() + with mock.patch("scanpipe.tasks.execute_pipeline_task", task_failure): + call_command("package-scan-worker", *options, stdout=out, stderr=out) + + out_value = out.getvalue() + self.assertIn("Exception occured during scan project:", out_value) + self.assertIn("Error during scan_codebase execution:", out_value) + self.assertIn("Error log", out_value) + mock_request_post.assert_called_once() + mock_request_post_call = mock_request_post.mock_calls[0] + mock_request_post_call_kwargs = mock_request_post_call.kwargs + print(mock_request_post_call_kwargs) + self.assertEqual( + self.purldb_update_status_url, mock_request_post_call_kwargs["url"] + ) + expected_data = { + "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", + "scan_status": "failed", + "scan_log": "Exception occured during scan project:\n\n" + "Error during scan_codebase execution:\nError log", + } + self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) + + @mock.patch("scanpipe.pipes.purldb.request_post") + @mock.patch("scanpipe.pipes.purldb.request_get") + def test_scanpipe_management_command_package_scan_worker_can_continue_after_failure( + self, mock_request_get, mock_request_post + ): + scannable_uri_uuid1 = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" + scannable_uri_uuid2 = "0bbdcf88-ad07-4970-9272-7d5f4c82cc7b" + mock_request_get.side_effect = [ + { + "scannable_uri_uuid": scannable_uri_uuid1, + "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", + "pipelines": ["scan_codebase"], + }, + { + "scannable_uri_uuid": scannable_uri_uuid2, + "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.1.tgz", + "pipelines": ["scan_codebase"], + }, + ] + + mock_request_post.side_effect = [ + { + "status": f"updated scannable uri {scannable_uri_uuid1} " + "scan_status to failed" + }, + {"status": f"scan indexed for scannable uri {scannable_uri_uuid2}"}, + ] + + options = [ + "--max-loops", + 2, + ] + out = StringIO() + with mock.patch("scanpipe.tasks.execute_pipeline_task", task_failure): + call_command("package-scan-worker", *options, stdout=out, stderr=out) + + out_value = out.getvalue() + self.assertIn( + "Project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e created", out_value + ) + self.assertIn( + "Project httpsregistrynpmjsorgasdf-asdf-121tgz-0bbdcf88 created", out_value + ) + self.assertIn("- asdf-1.2.2.tgz", out_value) + self.assertIn("- asdf-1.2.1.tgz", out_value) + self.assertIn("Exception occured during scan project:", out_value) + self.assertIn("Error during scan_codebase execution:", out_value) + self.assertIn("Error log", out_value) + + update_status_url = f"{purldb.PURLDB_API_URL}scan_queue/update_status/" + calls = [ + mock.call( + url=update_status_url, + timeout=60, + data={ + "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", + "scan_status": "failed", + "scan_log": "Exception occured during scan project:\n\n" + "Error during scan_codebase execution:\nError log", + }, + ), + mock.call( + url=update_status_url, + timeout=60, + data={ + "scannable_uri_uuid": "0bbdcf88-ad07-4970-9272-7d5f4c82cc7b", + "scan_status": "failed", + "scan_log": "Exception occured during scan project:\n\n" + "Error during scan_codebase execution:\nError log", + }, + ), + ] + mock_request_post.assert_has_calls(calls) + class ScanPipeManagementCommandMixinTest(TestCase): - class CreateProjectCommand(commands.CreateProjectCommandMixin, commands.AddInputCommandMixin, BaseCommand): + class CreateProjectCommand( + commands.CreateProjectCommandMixin, commands.AddInputCommandMixin, BaseCommand + ): pass create_project_command = CreateProjectCommand() pipeline_name = "analyze_docker_image" pipeline_class = scanpipe_app.pipelines.get(pipeline_name) - def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_base(self): + def test_scanpipe_management_command_mixin_create_project_base( + self, + ): expected = "This field cannot be blank." with self.assertRaisesMessage(CommandError, expected): self.create_project_command.create_project(name="") @@ -621,22 +788,32 @@ def test_scanpipe_management_command_mixin_create_project_command_mixin_create_p with self.assertRaisesMessage(CommandError, expected): self.create_project_command.create_project(name="my_project") - def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_notes(self): + def test_scanpipe_management_command_mixin_create_project_notes( + self, + ): notes = "Some notes about my project" - project = self.create_project_command.create_project(name="my_project", notes=notes) + project = self.create_project_command.create_project( + name="my_project", notes=notes + ) self.assertEqual(notes, project.notes) - def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_pipelines(self): + def test_scanpipe_management_command_mixin_create_project_pipelines( + self, + ): expected = "non-existing is not a valid pipeline" with self.assertRaisesMessage(CommandError, expected): - self.create_project_command.create_project(name="my_project", pipelines=["non-existing"]) + self.create_project_command.create_project( + name="my_project", pipelines=["non-existing"] + ) pipelines = [ self.pipeline_name, "analyze_root_filesystem_or_vm_image:group1,group2", "scan_package", ] - project = self.create_project_command.create_project(name="my_project", pipelines=pipelines) + project = self.create_project_command.create_project( + name="my_project", pipelines=pipelines + ) expected = [ self.pipeline_name, "analyze_root_filesystem_or_vm_image", @@ -646,23 +823,31 @@ def test_scanpipe_management_command_mixin_create_project_command_mixin_create_p run = project.runs.get(pipeline_name="analyze_root_filesystem_or_vm_image") self.assertEqual(["group1", "group2"], run.selected_groups) - def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_inputs(self): + def test_scanpipe_management_command_mixin_create_project_inputs( + self, + ): expected = "non-existing not found or not a file" with self.assertRaisesMessage(CommandError, expected): - self.create_project_command.create_project(name="my_project", input_files=["non-existing"]) + self.create_project_command.create_project( + name="my_project", input_files=["non-existing"] + ) parent_path = Path(__file__).parent input_files = [ str(parent_path / "test_commands.py"), str(parent_path / "test_models.py:tag"), ] - project = self.create_project_command.create_project(name="my_project", input_files=input_files) + project = self.create_project_command.create_project( + name="my_project", input_files=input_files + ) expected = sorted(["test_commands.py", "test_models.py"]) self.assertEqual(expected, sorted(project.input_files)) tagged_source = project.inputsources.get(filename="test_models.py") self.assertEqual("tag", tagged_source.tag) - def test_scanpipe_management_command_mixin_create_project_command_mixin_create_project_execute(self): + def test_scanpipe_management_command_mixin_create_project_execute( + self, + ): expected = "The execute argument requires one or more pipelines." with self.assertRaisesMessage(CommandError, expected): self.create_project_command.create_project(name="my_project", execute=True) @@ -670,9 +855,7 @@ def test_scanpipe_management_command_mixin_create_project_command_mixin_create_p pipeline = "load_inventory" with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): project = self.create_project_command.create_project( - name="my_project", - pipelines=[pipeline], - execute=True + name="my_project", pipelines=[pipeline], execute=True ) run = project.runs.first() self.assertTrue(run.task_succeeded) @@ -684,33 +867,5 @@ def test_scanpipe_management_command_mixin_create_project_command_mixin_create_p name="other_project", pipelines=[pipeline], execute=True, - run_async=True + run_async=True, ) - - @mock.patch("scanpipe.pipes.purldb.request_post") - @mock.patch("scanpipe.pipes.purldb.request_get") - def test_scanpipe_management_command_package_scan_worker(self, mock_request_get, mock_request_post): - scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" - mock_request_get.return_value = { - "scannable_uri_uuid": scannable_uri_uuid, - "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", - "pipelines": ["scan_codebase"] - } - mock_request_post.return_value = { - 'status': f'scan indexed for scannable uri {scannable_uri_uuid}' - } - - options = [ - "--max-loops", - 1, - ] - out = StringIO() - with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): - call_command("package-scan-worker", *options, stdout=out) - - out_value = out.getvalue() - self.assertIn("Project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e created", out_value) - self.assertIn("File(s) downloaded to the project inputs directory:", out_value) - self.assertIn("asdf-1.2.2.tgz", out_value) - self.assertIn("scan_codebase successfully executed on project httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e", out_value) - mock_request_post.assert_called_once() From bc1fbede3a4595d972edfe97b1d82311b142f2b5 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 14 Mar 2024 16:05:20 -0700 Subject: [PATCH 19/30] Save scannable_uri_uuid to project extra data Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 3 +++ scanpipe/tests/test_commands.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 43eb625ffd..6ca94e87c7 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -87,6 +87,9 @@ def handle(self, *args, **options): execute=True, run_async=run_async, ) + project.update_extra_data( + {"scannable_uri_uuid": scannable_uri_uuid} + ) # 3. Poll project results # TODO: see if we can block waiting for a signal when the diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index c9c39996d1..613bab8cff 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -606,9 +606,10 @@ def test_scanpipe_management_command_package_scan_worker( self, mock_request_get, mock_request_post ): scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" + download_url = "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz" mock_request_get.return_value = { "scannable_uri_uuid": scannable_uri_uuid, - "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", + "download_url": download_url, "pipelines": ["scan_codebase"], } mock_request_post.return_value = { @@ -634,6 +635,11 @@ def test_scanpipe_management_command_package_scan_worker( "httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e", out_value, ) + + project_name = purldb.create_project_name(download_url, scannable_uri_uuid) + project = Project.objects.get(name=project_name) + self.assertEqual(scannable_uri_uuid, project.extra_data['scannable_uri_uuid']) + mock_request_post.assert_called_once() mock_request_post_call = mock_request_post.mock_calls[0] mock_request_post_call_kwargs = mock_request_post_call.kwargs @@ -674,10 +680,10 @@ def test_scanpipe_management_command_package_scan_worker_failure( self.assertIn("Exception occured during scan project:", out_value) self.assertIn("Error during scan_codebase execution:", out_value) self.assertIn("Error log", out_value) + mock_request_post.assert_called_once() mock_request_post_call = mock_request_post.mock_calls[0] mock_request_post_call_kwargs = mock_request_post_call.kwargs - print(mock_request_post_call_kwargs) self.assertEqual( self.purldb_update_status_url, mock_request_post_call_kwargs["url"] ) From c6d50e619ba07e9d53c7227207599bc8e9239fe7 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 14 Mar 2024 17:42:22 -0700 Subject: [PATCH 20/30] Bump matchcode-toolkit version Signed-off-by: Jono Yang --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8493e29a0a..6c1fbfde90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -93,7 +93,7 @@ install_requires = # Font Awesome fontawesomefree==6.5.1 # MatchCode-toolkit - matchcode-toolkit==4.0.0 + matchcode-toolkit==3.1.0 # Univers univers==30.11.0 # Markdown From ccfc673c6eac42cf4c9c7d915cdf6e42fa565f0a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 14 Mar 2024 21:03:07 -0700 Subject: [PATCH 21/30] Send summary along with scan results to purldb Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 6 ++-- scanpipe/pipes/purldb.py | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 6ca94e87c7..03c036b9c3 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -26,7 +26,6 @@ from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import CreateProjectCommandMixin -from scanpipe.pipes import output from scanpipe.pipes import purldb @@ -108,9 +107,10 @@ def handle(self, *args, **options): ) else: # 4. Get project results and send to PurlDB - scan_output_location = output.to_json(project) + scan_file_location = project.get_output_file_path(name="results", extension="json") + summary_file_location = project.get_output_file_path(name="summary", extension="json") purldb.send_results_to_purldb( - scannable_uri_uuid, scan_output_location + scannable_uri_uuid, scan_file_location, summary_file_location ) except Exception as e: diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 3e5bcd9c68..700e170fda 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -505,7 +505,8 @@ def get_next_job(): def send_results_to_purldb( scannable_uri_uuid, - scan_output_location, + scan_file_location, + summary_file_location, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): @@ -513,20 +514,22 @@ def send_results_to_purldb( Send project results to purldb for the package handeled by the ScannableURI with uuid of `scannable_uri_uuid` """ - with open(scan_output_location, "rb") as f: - data = { - "scannable_uri_uuid": scannable_uri_uuid, - "scan_status": "scanned", - } - files = { - "scan_file": f, - } - response = request_post( - url=f"{api_url}scan_queue/update_status/", - timeout=timeout, - data=data, - files=files, - ) + with open(scan_file_location, "rb") as scan_file: + with open(summary_file_location, "rb") as summary_file: + data = { + "scannable_uri_uuid": scannable_uri_uuid, + "scan_status": "scanned", + } + files = { + "scan_file": scan_file, + "summary_file": summary_file, + } + response = request_post( + url=f"{api_url}scan_queue/update_status/", + timeout=timeout, + data=data, + files=files, + ) return response From a7f48ee65d30177f2ae0b4de269d4e673f3535cd Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 15 Mar 2024 19:58:30 -0700 Subject: [PATCH 22/30] Rename arguments for send_results_to_purldb * Bump matchcode-toolkit version to 4.0.0 Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 7 ++++--- scanpipe/pipes/purldb.py | 12 ++++++------ setup.cfg | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 03c036b9c3..355ef78a83 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -27,6 +27,7 @@ from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import CreateProjectCommandMixin from scanpipe.pipes import purldb +from scanpipe.pipes import output class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand): @@ -107,10 +108,10 @@ def handle(self, *args, **options): ) else: # 4. Get project results and send to PurlDB - scan_file_location = project.get_output_file_path(name="results", extension="json") - summary_file_location = project.get_output_file_path(name="summary", extension="json") + scan_results_location = output.to_json(project) + scan_summary_location = project.get_latest_output(filename="summary") purldb.send_results_to_purldb( - scannable_uri_uuid, scan_file_location, summary_file_location + scannable_uri_uuid, scan_results_location, scan_summary_location ) except Exception as e: diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 700e170fda..85f406e735 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -505,8 +505,8 @@ def get_next_job(): def send_results_to_purldb( scannable_uri_uuid, - scan_file_location, - summary_file_location, + scan_results_location, + scan_summary_location, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): @@ -514,15 +514,15 @@ def send_results_to_purldb( Send project results to purldb for the package handeled by the ScannableURI with uuid of `scannable_uri_uuid` """ - with open(scan_file_location, "rb") as scan_file: - with open(summary_file_location, "rb") as summary_file: + with open(scan_results_location, "rb") as scan_results_file: + with open(scan_summary_location, "rb") as scan_summary_file: data = { "scannable_uri_uuid": scannable_uri_uuid, "scan_status": "scanned", } files = { - "scan_file": scan_file, - "summary_file": summary_file, + "scan_results_file": scan_results_file, + "scan_summary_file": scan_summary_file, } response = request_post( url=f"{api_url}scan_queue/update_status/", diff --git a/setup.cfg b/setup.cfg index 6c1fbfde90..8493e29a0a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -93,7 +93,7 @@ install_requires = # Font Awesome fontawesomefree==6.5.1 # MatchCode-toolkit - matchcode-toolkit==3.1.0 + matchcode-toolkit==4.0.0 # Univers univers==30.11.0 # Markdown From 4401aea842baa1a7a640a71b679f91ee5d7319f6 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Sat, 16 Mar 2024 00:56:52 -0700 Subject: [PATCH 23/30] Send project extra data to purldb with scan results Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 6 +++++- scanpipe/pipes/purldb.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 355ef78a83..f2d04ce1cb 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -108,10 +108,14 @@ def handle(self, *args, **options): ) else: # 4. Get project results and send to PurlDB + project.refresh_from_db() scan_results_location = output.to_json(project) scan_summary_location = project.get_latest_output(filename="summary") purldb.send_results_to_purldb( - scannable_uri_uuid, scan_results_location, scan_summary_location + scannable_uri_uuid, + scan_results_location, + scan_summary_location, + project.extra_data, ) except Exception as e: diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 85f406e735..b0663e7dc5 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -507,6 +507,7 @@ def send_results_to_purldb( scannable_uri_uuid, scan_results_location, scan_summary_location, + project_extra_data, timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL, ): @@ -519,6 +520,7 @@ def send_results_to_purldb( data = { "scannable_uri_uuid": scannable_uri_uuid, "scan_status": "scanned", + "project_extra_data": json.dumps(project_extra_data), } files = { "scan_results_file": scan_results_file, From faa0957acef526777de7e39044d0556ec6f71d0e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Sat, 16 Mar 2024 01:20:59 -0700 Subject: [PATCH 24/30] Print success message when results are sent Signed-off-by: Jono Yang --- scanpipe/management/commands/package-scan-worker.py | 10 ++++++++-- scanpipe/tests/test_commands.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index f2d04ce1cb..35e6754512 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -26,8 +26,8 @@ from scanpipe.management.commands import AddInputCommandMixin from scanpipe.management.commands import CreateProjectCommandMixin -from scanpipe.pipes import purldb from scanpipe.pipes import output +from scanpipe.pipes import purldb class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand): @@ -110,13 +110,19 @@ def handle(self, *args, **options): # 4. Get project results and send to PurlDB project.refresh_from_db() scan_results_location = output.to_json(project) - scan_summary_location = project.get_latest_output(filename="summary") + scan_summary_location = project.get_latest_output( + filename="summary" + ) purldb.send_results_to_purldb( scannable_uri_uuid, scan_results_location, scan_summary_location, project.extra_data, ) + self.stdout.write( + "Scan results and other data have been sent to PurlDB", + self.style.SUCCESS, + ) except Exception as e: error_log = f"Exception occured during scan project:\n\n{str(e)}" diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 613bab8cff..48d91b2965 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -638,7 +638,7 @@ def test_scanpipe_management_command_package_scan_worker( project_name = purldb.create_project_name(download_url, scannable_uri_uuid) project = Project.objects.get(name=project_name) - self.assertEqual(scannable_uri_uuid, project.extra_data['scannable_uri_uuid']) + self.assertEqual(scannable_uri_uuid, project.extra_data["scannable_uri_uuid"]) mock_request_post.assert_called_once() mock_request_post_call = mock_request_post.mock_calls[0] From 709d171654844fc62c330cc064752338e46d6de0 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 19 Mar 2024 13:28:00 -0700 Subject: [PATCH 25/30] Update package-scan-worker tests * Sen both traceback and exception message to purldb Signed-off-by: Jono Yang --- .../commands/package-scan-worker.py | 4 ++- scanpipe/tests/test_commands.py | 29 +++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/package-scan-worker.py index 35e6754512..24276c8419 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/package-scan-worker.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +import traceback import time from django.core.management.base import BaseCommand @@ -125,7 +126,8 @@ def handle(self, *args, **options): ) except Exception as e: - error_log = f"Exception occured during scan project:\n\n{str(e)}" + tb = traceback.format_exc() + error_log = f"Exception occured during scan project:\n\n{tb}" purldb.update_status( scannable_uri_uuid, status="failed", diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 48d91b2965..678a9587d4 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -63,6 +63,7 @@ def raise_interrupt(run_pk): class ScanPipeManagementCommandTest(TestCase): + data_location = Path(__file__).parent / "data" pipeline_name = "analyze_docker_image" pipeline_class = scanpipe_app.pipelines.get(pipeline_name) purldb_update_status_url = f"{purldb.PURLDB_API_URL}scan_queue/update_status/" @@ -600,21 +601,23 @@ def test_scanpipe_management_command_create_user(self): with self.assertRaisesMessage(CommandError, expected): call_command("create-user", "--no-input", username) + @mock.patch("scanpipe.models.Project.get_latest_output") @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("scanpipe.pipes.purldb.request_get") def test_scanpipe_management_command_package_scan_worker( - self, mock_request_get, mock_request_post + self, mock_request_get, mock_request_post, mock_get_latest_output ): scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" download_url = "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz" mock_request_get.return_value = { "scannable_uri_uuid": scannable_uri_uuid, "download_url": download_url, - "pipelines": ["scan_codebase"], + "pipelines": ["scan_single_package"], } mock_request_post.return_value = { "status": f"scan indexed for scannable uri {scannable_uri_uuid}" } + mock_get_latest_output.return_value = self.data_location / "scancode" / "is-npm-1.0.0_summary.json" options = [ "--max-loops", @@ -631,7 +634,7 @@ def test_scanpipe_management_command_package_scan_worker( self.assertIn("File(s) downloaded to the project inputs directory:", out_value) self.assertIn("asdf-1.2.2.tgz", out_value) self.assertIn( - "scan_codebase successfully executed on project " + "scan_single_package successfully executed on project " "httpsregistrynpmjsorgasdf-asdf-122tgz-97627c6e", out_value, ) @@ -649,9 +652,11 @@ def test_scanpipe_management_command_package_scan_worker( expected_data = { "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", "scan_status": "scanned", + "project_extra_data": '{"scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5"}' } self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) - self.assertTrue(mock_request_post_call_kwargs["files"]["scan_file"]) + self.assertTrue(mock_request_post_call_kwargs["files"]["scan_results_file"]) + self.assertTrue(mock_request_post_call_kwargs["files"]["scan_summary_file"]) @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("scanpipe.pipes.purldb.request_get") @@ -662,7 +667,7 @@ def test_scanpipe_management_command_package_scan_worker_failure( mock_request_get.return_value = { "scannable_uri_uuid": scannable_uri_uuid, "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", - "pipelines": ["scan_codebase"], + "pipelines": ["scan_single_package"], } mock_request_post.return_value = { "status": f"scan failed for scannable uri {scannable_uri_uuid}" @@ -678,7 +683,7 @@ def test_scanpipe_management_command_package_scan_worker_failure( out_value = out.getvalue() self.assertIn("Exception occured during scan project:", out_value) - self.assertIn("Error during scan_codebase execution:", out_value) + self.assertIn("Error during scan_single_package execution:", out_value) self.assertIn("Error log", out_value) mock_request_post.assert_called_once() @@ -691,7 +696,7 @@ def test_scanpipe_management_command_package_scan_worker_failure( "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", "scan_status": "failed", "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_codebase execution:\nError log", + "Error during scan_single_package execution:\nError log", } self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) @@ -706,12 +711,12 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail { "scannable_uri_uuid": scannable_uri_uuid1, "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.2.tgz", - "pipelines": ["scan_codebase"], + "pipelines": ["scan_single_package"], }, { "scannable_uri_uuid": scannable_uri_uuid2, "download_url": "https://registry.npmjs.org/asdf/-/asdf-1.2.1.tgz", - "pipelines": ["scan_codebase"], + "pipelines": ["scan_single_package"], }, ] @@ -741,7 +746,7 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail self.assertIn("- asdf-1.2.2.tgz", out_value) self.assertIn("- asdf-1.2.1.tgz", out_value) self.assertIn("Exception occured during scan project:", out_value) - self.assertIn("Error during scan_codebase execution:", out_value) + self.assertIn("Error during scan_single_package execution:", out_value) self.assertIn("Error log", out_value) update_status_url = f"{purldb.PURLDB_API_URL}scan_queue/update_status/" @@ -753,7 +758,7 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", "scan_status": "failed", "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_codebase execution:\nError log", + "Error during scan_single_package execution:\nError log", }, ), mock.call( @@ -763,7 +768,7 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail "scannable_uri_uuid": "0bbdcf88-ad07-4970-9272-7d5f4c82cc7b", "scan_status": "failed", "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_codebase execution:\nError log", + "Error during scan_single_package execution:\nError log", }, ), ] From 5c85dc6469f56d21c3947e4244fa5a2c40249300 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 19 Mar 2024 19:00:13 -0700 Subject: [PATCH 26/30] Update CHANGELOG.rst * Update management command test names Signed-off-by: Jono Yang --- CHANGELOG.rst | 11 +++ ...-worker.py => purldb-scan-queue-worker.py} | 4 +- scanpipe/tests/test_commands.py | 84 ++++++++++--------- 3 files changed, 59 insertions(+), 40 deletions(-) rename scanpipe/management/commands/{package-scan-worker.py => purldb-scan-queue-worker.py} (99%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3f6e43bcc4..d9ddc009e1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,6 +44,17 @@ v34.1.0 (unreleased) and display any available data. https://github.com/nexB/scancode.io/issues/1125 +- Create a new management command `purldb-scan-queue-worker`, that runs + scancode.io as a Package scan queue worker for PurlDB. + `purldb-scan-queue-worker` gets the next available Package to be scanned and + the list of pipeline names to be run on the Package from PurlDB, creates a + Project, fetches the Package, runs the specified pipelines, and returns the + results to PurlDB. + https://github.com/nexB/scancode.io/pull/1078 + https://github.com/nexB/purldb/issues/236 + +- Update matchcode-toolkit to v4.0.0 + v34.0.0 (2024-03-04) -------------------- diff --git a/scanpipe/management/commands/package-scan-worker.py b/scanpipe/management/commands/purldb-scan-queue-worker.py similarity index 99% rename from scanpipe/management/commands/package-scan-worker.py rename to scanpipe/management/commands/purldb-scan-queue-worker.py index 24276c8419..a2262567a3 100644 --- a/scanpipe/management/commands/package-scan-worker.py +++ b/scanpipe/management/commands/purldb-scan-queue-worker.py @@ -20,8 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -import traceback import time +import traceback from django.core.management.base import BaseCommand @@ -125,7 +125,7 @@ def handle(self, *args, **options): self.style.SUCCESS, ) - except Exception as e: + except Exception: tb = traceback.format_exc() error_log = f"Exception occured during scan project:\n\n{tb}" purldb.update_status( diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 678a9587d4..9a53c088b5 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -604,7 +604,7 @@ def test_scanpipe_management_command_create_user(self): @mock.patch("scanpipe.models.Project.get_latest_output") @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("scanpipe.pipes.purldb.request_get") - def test_scanpipe_management_command_package_scan_worker( + def test_scanpipe_management_command_purldb_scan_queue_worker( self, mock_request_get, mock_request_post, mock_get_latest_output ): scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" @@ -617,7 +617,9 @@ def test_scanpipe_management_command_package_scan_worker( mock_request_post.return_value = { "status": f"scan indexed for scannable uri {scannable_uri_uuid}" } - mock_get_latest_output.return_value = self.data_location / "scancode" / "is-npm-1.0.0_summary.json" + mock_get_latest_output.return_value = ( + self.data_location / "scancode" / "is-npm-1.0.0_summary.json" + ) options = [ "--max-loops", @@ -625,7 +627,7 @@ def test_scanpipe_management_command_package_scan_worker( ] out = StringIO() with mock.patch("scanpipe.tasks.execute_pipeline_task", task_success): - call_command("package-scan-worker", *options, stdout=out) + call_command("purldb-scan-queue-worker", *options, stdout=out) out_value = out.getvalue() self.assertIn( @@ -652,7 +654,8 @@ def test_scanpipe_management_command_package_scan_worker( expected_data = { "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", "scan_status": "scanned", - "project_extra_data": '{"scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5"}' + "project_extra_data": '{"scannable_uri_uuid": ' + '"97627c6e-9acb-43e0-b8df-28bd92f2b7e5"}', } self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) self.assertTrue(mock_request_post_call_kwargs["files"]["scan_results_file"]) @@ -660,7 +663,7 @@ def test_scanpipe_management_command_package_scan_worker( @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("scanpipe.pipes.purldb.request_get") - def test_scanpipe_management_command_package_scan_worker_failure( + def test_scanpipe_management_command_purldb_scan_queue_worker_failure( self, mock_request_get, mock_request_post ): scannable_uri_uuid = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" @@ -679,7 +682,7 @@ def test_scanpipe_management_command_package_scan_worker_failure( ] out = StringIO() with mock.patch("scanpipe.tasks.execute_pipeline_task", task_failure): - call_command("package-scan-worker", *options, stdout=out, stderr=out) + call_command("purldb-scan-queue-worker", *options, stdout=out, stderr=out) out_value = out.getvalue() self.assertIn("Exception occured during scan project:", out_value) @@ -692,17 +695,19 @@ def test_scanpipe_management_command_package_scan_worker_failure( self.assertEqual( self.purldb_update_status_url, mock_request_post_call_kwargs["url"] ) - expected_data = { - "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", - "scan_status": "failed", - "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_single_package execution:\nError log", - } - self.assertEqual(expected_data, mock_request_post_call_kwargs["data"]) + self.assertEqual( + "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", + mock_request_post_call_kwargs["data"]["scannable_uri_uuid"], + ) + self.assertEqual("failed", mock_request_post_call_kwargs["data"]["scan_status"]) + self.assertIn( + "Exception occured during scan project:", + mock_request_post_call_kwargs["data"]["scan_log"], + ) @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("scanpipe.pipes.purldb.request_get") - def test_scanpipe_management_command_package_scan_worker_can_continue_after_failure( + def test_scanpipe_management_command_purldb_scan_queue_worker_continue_after_fail( self, mock_request_get, mock_request_post ): scannable_uri_uuid1 = "97627c6e-9acb-43e0-b8df-28bd92f2b7e5" @@ -734,7 +739,7 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail ] out = StringIO() with mock.patch("scanpipe.tasks.execute_pipeline_task", task_failure): - call_command("package-scan-worker", *options, stdout=out, stderr=out) + call_command("purldb-scan-queue-worker", *options, stdout=out, stderr=out) out_value = out.getvalue() self.assertIn( @@ -750,29 +755,32 @@ def test_scanpipe_management_command_package_scan_worker_can_continue_after_fail self.assertIn("Error log", out_value) update_status_url = f"{purldb.PURLDB_API_URL}scan_queue/update_status/" - calls = [ - mock.call( - url=update_status_url, - timeout=60, - data={ - "scannable_uri_uuid": "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", - "scan_status": "failed", - "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_single_package execution:\nError log", - }, - ), - mock.call( - url=update_status_url, - timeout=60, - data={ - "scannable_uri_uuid": "0bbdcf88-ad07-4970-9272-7d5f4c82cc7b", - "scan_status": "failed", - "scan_log": "Exception occured during scan project:\n\n" - "Error during scan_single_package execution:\nError log", - }, - ), - ] - mock_request_post.assert_has_calls(calls) + mocked_post_calls = mock_request_post.call_args_list + self.assertEqual(2, len(mocked_post_calls)) + + mock_post_call1 = mocked_post_calls[0] + self.assertEqual(update_status_url, mock_post_call1.kwargs["url"]) + self.assertEqual( + "97627c6e-9acb-43e0-b8df-28bd92f2b7e5", + mock_post_call1.kwargs["data"]["scannable_uri_uuid"], + ) + self.assertEqual("failed", mock_post_call1.kwargs["data"]["scan_status"]) + self.assertIn( + "Exception occured during scan project:", + mock_post_call1.kwargs["data"]["scan_log"], + ) + + mock_post_call2 = mocked_post_calls[1] + self.assertEqual(update_status_url, mock_post_call2.kwargs["url"]) + self.assertEqual( + "0bbdcf88-ad07-4970-9272-7d5f4c82cc7b", + mock_post_call2.kwargs["data"]["scannable_uri_uuid"], + ) + self.assertEqual("failed", mock_post_call1.kwargs["data"]["scan_status"]) + self.assertIn( + "Exception occured during scan project:", + mock_post_call2.kwargs["data"]["scan_log"], + ) class ScanPipeManagementCommandMixinTest(TestCase): From 1d472d507c872f17ed75d2b5d63b31d912a75695 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 20 Mar 2024 17:00:59 -0700 Subject: [PATCH 27/30] Address review comments * Move lists of statuses in poll_until_success to their own variables * Remove unnecessary else statement * Use tuples as default values for `create_project` Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 6 +-- .../commands/purldb-scan-queue-worker.py | 2 - scanpipe/pipes/purldb.py | 37 +++++++++++-------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 1fb974f9a7..4874f099d3 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -349,9 +349,9 @@ class CreateProjectCommandMixin(ExecuteProjectCommandMixin): def create_project( self, name, - pipelines=[], - input_files=[], - input_urls=[], + pipelines=(), + input_files=(), + input_urls=(), copy_from="", notes="", execute=False, diff --git a/scanpipe/management/commands/purldb-scan-queue-worker.py b/scanpipe/management/commands/purldb-scan-queue-worker.py index a2262567a3..b6c9850ede 100644 --- a/scanpipe/management/commands/purldb-scan-queue-worker.py +++ b/scanpipe/management/commands/purldb-scan-queue-worker.py @@ -93,8 +93,6 @@ def handle(self, *args, **options): ) # 3. Poll project results - # TODO: see if we can block waiting for a signal when the - # project is done running error_log = purldb.poll_run_status( project=project, sleep=sleep, diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index b0663e7dc5..c8fa408b15 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -366,12 +366,12 @@ def poll_run_url_status(run_url, sleep=10): """ if poll_until_success(check=get_run_url_status, sleep=sleep, run_url=run_url): return True - else: - response = request_get(run_url) - if response: - log = response["log"] - msg = f"Matching run has stopped:\n\n{log}" - raise PurlDBException(msg) + + response = request_get(run_url) + if response: + log = response["log"] + msg = f"Matching run has stopped:\n\n{log}" + raise PurlDBException(msg) def poll_until_success(check, sleep=10, **kwargs): @@ -385,23 +385,28 @@ def poll_until_success(check, sleep=10, **kwargs): function. """ run_status = AbstractTaskFieldsModel.Status + # Continue looping if the run instance has the following statuses + CONTINUE_STATUSES = [ + run_status.NOT_STARTED, + run_status.QUEUED, + run_status.RUNNING, + ] + # Return False if the run instance has the following statuses + FAIL_STATUSES = [ + run_status.FAILURE, + run_status.STOPPED, + run_status.STALE, + ] + while True: status = check(**kwargs) if status == run_status.SUCCESS: return True - if status in [ - run_status.NOT_STARTED, - run_status.QUEUED, - run_status.RUNNING, - ]: + if status in CONTINUE_STATUSES: continue - if status in [ - run_status.FAILURE, - run_status.STOPPED, - run_status.STALE, - ]: + if status in FAIL_STATUSES: return False time.sleep(sleep) From 3c81a2d684b0121fdba6878ab305a9e464a7f290 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 21 Mar 2024 20:02:35 -0700 Subject: [PATCH 28/30] Refactor logic in purldb-scan-queue-worker * Rename get_next_job to get_next_download_url * Update tests Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 59 +++++-- .../commands/purldb-scan-queue-worker.py | 155 ++++++++++-------- scanpipe/pipes/purldb.py | 32 +--- scanpipe/tests/pipes/test_purldb.py | 66 ++++---- 4 files changed, 171 insertions(+), 141 deletions(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 4874f099d3..6879827f74 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -349,9 +349,9 @@ class CreateProjectCommandMixin(ExecuteProjectCommandMixin): def create_project( self, name, - pipelines=(), - input_files=(), - input_urls=(), + pipelines=None, + input_files=None, + input_urls=None, copy_from="", notes="", execute=False, @@ -370,22 +370,56 @@ def create_project( raise CommandError("\n".join(e.messages)) # Run validation before creating the project in the database - pipelines_data = extract_group_from_pipelines(pipelines) - pipelines_data = validate_pipelines(pipelines_data) - - input_files_data = self.extract_tag_from_input_files(input_files) - self.validate_input_files(input_files=input_files_data.keys()) - validate_copy_from(copy_from) + pipelines_data, input_files_data = self._validate_project_inputs( + pipelines=pipelines, input_files=input_files, copy_from=copy_from + ) project.save() self.project = project msg = f"Project {name} created with work directory {project.work_directory}" self.stdout.write(msg, self.style.SUCCESS) + self._add_project_inputs( + pipelines_data=pipelines_data, + input_files_data=input_files_data, + input_urls=input_urls, + copy_from=copy_from, + ) + + if execute: + self.execute_project(run_async=run_async) + + return project + + def _validate_project_inputs(self, pipelines, input_files, copy_from): + """ + Validate `pipelines`, `input_files`, and `copy_from`, returning a tuple + of dictionaries containing the pipeline data of `pipelines` and the + input files data from `input_files. + """ + pipelines_data = {} + input_files_data = {} + + if pipelines: + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + + if input_files: + input_files_data = self.extract_tag_from_input_files(input_files) + self.validate_input_files(input_files=input_files_data.keys()) + + if copy_from: + validate_copy_from(copy_from) + + return pipelines_data, input_files_data + + def _add_project_inputs( + self, pipelines_data, input_files_data, input_urls, copy_from + ): for pipeline_name, selected_groups in pipelines_data.items(): self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) - if input_files: + if input_files_data: self.handle_input_files(input_files_data) if input_urls: @@ -393,8 +427,3 @@ def create_project( if copy_from: self.handle_copy_codebase(copy_from) - - if execute: - self.execute_project(run_async=run_async) - - return project diff --git a/scanpipe/management/commands/purldb-scan-queue-worker.py b/scanpipe/management/commands/purldb-scan-queue-worker.py index b6c9850ede..60d3d29ba8 100644 --- a/scanpipe/management/commands/purldb-scan-queue-worker.py +++ b/scanpipe/management/commands/purldb-scan-queue-worker.py @@ -32,7 +32,7 @@ class Command(CreateProjectCommandMixin, AddInputCommandMixin, BaseCommand): - help = "Create a ScanPipe project." + help = "Get a Package to be scanned from PurlDB and return the results" def add_arguments(self, parser): super().add_arguments(parser) @@ -65,72 +65,99 @@ def handle(self, *args, **options): break time.sleep(sleep) + loop_count += 1 # 1. Get download url from purldb - scannable_uri_uuid, download_url, pipelines, error_msg = ( - purldb.get_next_job() - ) - if error_msg: - self.stderr.write(error_msg) + response = purldb.get_next_download_url() + if response: + scannable_uri_uuid = response["scannable_uri_uuid"] + download_url = response["download_url"] + pipelines = response["pipelines"] + else: + self.stderr.write("Bad response from PurlDB: unable to get next job.") continue - if not download_url or not scannable_uri_uuid: + if not (download_url and scannable_uri_uuid): self.stdout.write("No new job from PurlDB.") - else: - try: - # 2. Create and run project - name = purldb.create_project_name(download_url, scannable_uri_uuid) - input_urls = [download_url] - project = self.create_project( - name=name, - pipelines=pipelines, - input_urls=input_urls, - execute=True, - run_async=run_async, - ) - project.update_extra_data( - {"scannable_uri_uuid": scannable_uri_uuid} - ) - - # 3. Poll project results - error_log = purldb.poll_run_status( - project=project, - sleep=sleep, - ) - - if error_log: - # Send error response to PurlDB - purldb.update_status( - scannable_uri_uuid, - status="failed", - scan_log=error_log, - ) - else: - # 4. Get project results and send to PurlDB - project.refresh_from_db() - scan_results_location = output.to_json(project) - scan_summary_location = project.get_latest_output( - filename="summary" - ) - purldb.send_results_to_purldb( - scannable_uri_uuid, - scan_results_location, - scan_summary_location, - project.extra_data, - ) - self.stdout.write( - "Scan results and other data have been sent to PurlDB", - self.style.SUCCESS, - ) - - except Exception: - tb = traceback.format_exc() - error_log = f"Exception occured during scan project:\n\n{tb}" - purldb.update_status( - scannable_uri_uuid, - status="failed", - scan_log=error_log, - ) - self.stderr.write(error_log) + continue - loop_count += 1 + try: + # 2. Create and run project + project = create_scan_project( + command=self, + scannable_uri_uuid=scannable_uri_uuid, + download_url=download_url, + pipelines=pipelines, + run_async=run_async, + ) + + # 3. Poll project results + purldb.poll_run_status( + project=project, + sleep=sleep, + ) + + # 4. Get project results and send to PurlDB + send_scan_project_results( + project=project, scannable_uri_uuid=scannable_uri_uuid + ) + self.stdout.write( + "Scan results and other data have been sent to PurlDB", + self.style.SUCCESS, + ) + + except Exception: + tb = traceback.format_exc() + error_log = f"Exception occured during scan project:\n\n{tb}" + purldb.update_status( + scannable_uri_uuid, + status="failed", + scan_log=error_log, + ) + self.stderr.write(error_log) + + +def create_scan_project( + command, scannable_uri_uuid, download_url, pipelines, run_async=False +): + """ + Create and return a Project for the scan project request with ID of + `scannable_uri_uuid`, where the target at `download_url` is fetched, and the + pipelines from `pipelines` is then run. + + If `run_async` is True, the pipelines on the Project is run in a separate + thread. + """ + name = purldb.create_project_name(download_url, scannable_uri_uuid) + input_urls = [download_url] + project = command.create_project( + name=name, + pipelines=pipelines, + input_urls=input_urls, + execute=True, + run_async=run_async, + ) + project.update_extra_data({"scannable_uri_uuid": scannable_uri_uuid}) + return project + + +def send_scan_project_results(project, scannable_uri_uuid): + """ + Send the JSON summary and results of `project` to PurlDB for the scan + request `scannable_uri_uuid`. + + Raise a PurlDBException if there is an issue sending results to PurlDB. + """ + project.refresh_from_db() + scan_results_location = output.to_json(project) + scan_summary_location = project.get_latest_output(filename="summary") + response = purldb.send_results_to_purldb( + scannable_uri_uuid, + scan_results_location, + scan_summary_location, + project.extra_data, + ) + if not response: + raise purldb.PurlDBException( + "Bad response returned when sending results to PurlDB" + ) diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index c8fa408b15..1c5b40168d 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -476,7 +476,7 @@ def create_packages_from_match_results(project, match_results): ) -def _get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): +def get_next_download_url(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): """ Return the ScannableURI UUID, download URL, and pipelines for the next Package to be scanned from PurlDB @@ -488,24 +488,7 @@ def _get_next_job(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): timeout=timeout, ) if response: - scannable_uri_uuid = response["scannable_uri_uuid"] - download_url = response["download_url"] - pipelines = response["pipelines"] - return scannable_uri_uuid, download_url, pipelines - - -def get_next_job(): - scannable_uri_uuid = download_url = pipelines = None - msg = "" - try: - response = _get_next_job() - if response: - scannable_uri_uuid, download_url, pipelines = response - else: - msg = "Bad response from PurlDB, unable to get next job." - except Exception as e: - msg = f"Exception occured when calling `purldb.get_next_job()`:\n\n{str(e)}" - return scannable_uri_uuid, download_url, pipelines, msg + return response def send_results_to_purldb( @@ -570,15 +553,16 @@ def create_project_name(download_url, scannable_uri_uuid): def poll_run_status(project, sleep=10): """ - Poll the status of all runs of `project`. Return the log of the run if - the run has stopped, failed, or gone stale, otherwise return an empty - string. + Poll the status of all runs of `project`. Raise a PurlDBException with a + message containing the log of the run if the run has stopped, failed, or + gone stale, otherwise return None. """ runs = project.runs.all() for run in runs: if not poll_until_success(check=get_run_status, sleep=sleep, run=run): - return run.log - return "" + status = get_run_status(run) + msg = f"Run ended with status {status}:\n\n{run.log}" + raise PurlDBException(msg) def get_run_status(run, **kwargs): diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 4cb785bc93..8c2d5e0bb4 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -373,7 +373,7 @@ def test_scanpipe_pipes_purldb_get_run_url_status( @mock.patch("scanpipe.pipes.purldb.request_get") @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_get_next_job( + def test_scanpipe_pipes_purldb_get_next_download_url( self, mock_is_available, mock_request_get ): mock_is_available.return_value = True @@ -388,35 +388,22 @@ def test_scanpipe_pipes_purldb_get_next_job( }, {"download_url": "", "scannable_uri_uuid": "", "pipelines": []}, None, - Exception(), ] - results = purldb.get_next_job() + + results = purldb.get_next_download_url() self.assertTrue(results) - scannable_uri_uuid, download_url, pipelines, error_msg = results - self.assertEqual(expected_download_url, download_url) - self.assertEqual(expected_scannable_uri_uuid, scannable_uri_uuid) - self.assertEqual(expected_pipelines, pipelines) - self.assertEqual("", error_msg) - - scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() - self.assertFalse(scannable_uri_uuid) - self.assertFalse(download_url) - self.assertFalse(pipelines) - self.assertEqual("", error_msg) - - scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() - self.assertFalse(scannable_uri_uuid) - self.assertFalse(download_url) - self.assertFalse(pipelines) - self.assertEqual("Bad response from PurlDB, unable to get next job.", error_msg) - - scannable_uri_uuid, download_url, pipelines, error_msg = purldb.get_next_job() - self.assertFalse(scannable_uri_uuid) - self.assertFalse(download_url) - self.assertFalse(pipelines) - self.assertIn( - "Exception occured when calling `purldb.get_next_job()`:", error_msg - ) + self.assertEqual(expected_scannable_uri_uuid, results["scannable_uri_uuid"]) + self.assertEqual(expected_download_url, results["download_url"]) + self.assertEqual(expected_pipelines, results["pipelines"]) + + results = purldb.get_next_download_url() + self.assertTrue(results) + self.assertFalse(results["scannable_uri_uuid"]) + self.assertFalse(results["download_url"]) + self.assertFalse(results["pipelines"]) + + results = purldb.get_next_download_url() + self.assertFalse(results) def test_scanpipe_pipes_purldb_poll_run_status(self): now = timezone.now() @@ -429,8 +416,7 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): task_end_date=now, task_exitcode=0, ) - error_message = purldb.poll_run_status(project=self.project1) - self.assertEqual("", error_message) + purldb.poll_run_status(project=self.project1) self.project1.runs.all().delete() self.create_run( @@ -440,8 +426,9 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): task_exitcode=1, log="failed", ) - error_message = purldb.poll_run_status(project=self.project1) - self.assertEqual("failed", error_message) + with self.assertRaises(purldb.PurlDBException) as context: + purldb.poll_run_status(project=self.project1) + self.assertIn("failed", context.exception) self.project1.runs.all().delete() self.create_run( @@ -451,8 +438,9 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): task_exitcode=99, log="stopped", ) - error_message = purldb.poll_run_status(project=self.project1) - self.assertEqual("stopped", error_message) + with self.assertRaises(purldb.PurlDBException) as context: + purldb.poll_run_status(project=self.project1) + self.assertIn("stopped", context.exception) self.project1.runs.all().delete() self.create_run( @@ -462,8 +450,9 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): task_exitcode=88, log="stale", ) - error_message = purldb.poll_run_status(project=self.project1) - self.assertEqual("stale", error_message) + with self.assertRaises(purldb.PurlDBException) as context: + purldb.poll_run_status(project=self.project1) + self.assertIn("stale", context.exception) self.project1.runs.all().delete() # Test pipelines success, then failure @@ -481,8 +470,9 @@ def test_scanpipe_pipes_purldb_poll_run_status(self): task_exitcode=1, log="failed", ) - error_message = purldb.poll_run_status(project=self.project1) - self.assertEqual("failed", error_message) + with self.assertRaises(purldb.PurlDBException) as context: + purldb.poll_run_status(project=self.project1) + self.assertIn("failed", context.exception) self.project1.runs.all().delete() def test_scanpipe_pipes_purldb_create_project_name(self): From 27ad732b7bce6e369e33d62b701cbf5c2fcb5cb0 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 26 Mar 2024 15:58:21 -0700 Subject: [PATCH 29/30] Fix imports and tests after rebase Signed-off-by: Jono Yang --- scanpipe/pipes/__init__.py | 40 ++++ scanpipe/pipes/matchcode.py | 49 ++--- scanpipe/pipes/purldb.py | 125 +----------- scanpipe/tests/pipes/test_matchcode.py | 50 ++++- scanpipe/tests/pipes/test_purldb.py | 268 +------------------------ 5 files changed, 118 insertions(+), 414 deletions(-) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 7e8c737f61..b0148c319f 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -23,6 +23,7 @@ import difflib import logging import sys +import time import uuid from contextlib import suppress from datetime import datetime @@ -33,6 +34,7 @@ from django.db.models import Count from scanpipe import humanize_time +from scanpipe.models import AbstractTaskFieldsModel from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency @@ -411,3 +413,41 @@ def get_resource_diff_ratio(resource_a, resource_b): str_a=resource_a.file_content, str_b=resource_b.file_content, ) + + +def poll_until_success(check, sleep=10, **kwargs): + """ + Given a function `check`, which returns the status of a run, return True + when the run instance has completed successfully. + + Return False when the run instance has failed, stopped, or gone stale. + + The arguments for `check` need to be provided as keyword argument into this + function. + """ + run_status = AbstractTaskFieldsModel.Status + # Continue looping if the run instance has the following statuses + CONTINUE_STATUSES = [ + run_status.NOT_STARTED, + run_status.QUEUED, + run_status.RUNNING, + ] + # Return False if the run instance has the following statuses + FAIL_STATUSES = [ + run_status.FAILURE, + run_status.STOPPED, + run_status.STALE, + ] + + while True: + status = check(**kwargs) + if status == run_status.SUCCESS: + return True + + if status in CONTINUE_STATUSES: + continue + + if status in FAIL_STATUSES: + return False + + time.sleep(sleep) diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index 1e26a08602..678b066bc0 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -21,7 +21,6 @@ # Visit https://github.com/nexB/scancode.io for support and download. import logging -import time from collections import defaultdict from django.conf import settings @@ -29,9 +28,9 @@ import requests from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints -from scanpipe.models import AbstractTaskFieldsModel from scanpipe.pipes import codebase from scanpipe.pipes import flag +from scanpipe.pipes import poll_until_success from scanpipe.pipes.output import to_json @@ -210,7 +209,18 @@ def send_project_json_to_matchcode( return run_url -def poll_until_success(run_url, sleep=10): +def get_run_url_status(run_url, **kwargs): + """ + Given a `run_url`, which is a URL to a ScanCode.io Project run, return its + status, otherwise return None. + """ + response = request_get(run_url) + if response: + status = response["status"] + return status + + +def poll_run_url_status(run_url, sleep=10): """ Given a URL to a scancode.io run instance, `run_url`, return True when the run instance has completed successfully. @@ -218,31 +228,14 @@ def poll_until_success(run_url, sleep=10): Raise a MatchCodeIOException when the run instance has failed, stopped, or gone stale. """ - run_status = AbstractTaskFieldsModel.Status - while True: - response = request_get(run_url) - if response: - status = response["status"] - if status == run_status.SUCCESS: - return True - - if status in [ - run_status.NOT_STARTED, - run_status.QUEUED, - run_status.RUNNING, - ]: - continue - - if status in [ - run_status.FAILURE, - run_status.STOPPED, - run_status.STALE, - ]: - log = response["log"] - msg = f"Matching run has stopped:\n\n{log}" - raise MatchCodeIOException(msg) - - time.sleep(sleep) + if poll_until_success(check=get_run_url_status, sleep=sleep, run_url=run_url): + return True + + response = request_get(run_url) + if response: + log = response["log"] + msg = f"Matching run has stopped:\n\n{log}" + raise MatchCodeIOException(msg) def get_match_results(run_url): diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 1c5b40168d..365f9c7325 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -31,11 +31,8 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import InvalidVersionRange -from django.utils.text import slugify -from scanpipe.models import AbstractTaskFieldsModel from scanpipe.pipes import LoopProgress -from scanpipe.pipes import flag -from scanpipe.pipes.output import to_json +from scanpipe.pipes import poll_until_success class PurlDBException(Exception): @@ -356,126 +353,6 @@ def find_packages(payload): return response.get("results") -def poll_run_url_status(run_url, sleep=10): - """ - Given a URL to a scancode.io run instance, `run_url`, return True when the - run instance has completed successfully. - - Raise a PurlDBException when the run instance has failed, stopped, or gone - stale. - """ - if poll_until_success(check=get_run_url_status, sleep=sleep, run_url=run_url): - return True - - response = request_get(run_url) - if response: - log = response["log"] - msg = f"Matching run has stopped:\n\n{log}" - raise PurlDBException(msg) - - -def poll_until_success(check, sleep=10, **kwargs): - """ - Given a function `check`, which returns the status of a run, return True - when the run instance has completed successfully. - - Return False when the run instance has failed, stopped, or gone stale. - - The arguments for `check` need to be provided as keyword argument into this - function. - """ - run_status = AbstractTaskFieldsModel.Status - # Continue looping if the run instance has the following statuses - CONTINUE_STATUSES = [ - run_status.NOT_STARTED, - run_status.QUEUED, - run_status.RUNNING, - ] - # Return False if the run instance has the following statuses - FAIL_STATUSES = [ - run_status.FAILURE, - run_status.STOPPED, - run_status.STALE, - ] - - while True: - status = check(**kwargs) - if status == run_status.SUCCESS: - return True - - if status in CONTINUE_STATUSES: - continue - - if status in FAIL_STATUSES: - return False - - time.sleep(sleep) - - -def get_run_url_status(run_url, **kwargs): - """ - Given a `run_url`, which is a URL to a ScanCode.io Project run, return its - status, otherwise return None. - """ - response = request_get(run_url) - if response: - status = response["status"] - return status - - -def get_match_results(run_url): - """ - Given the `run_url` for a pipeline running the matchcode matching pipeline, - return the match results for that run. - """ - response = request_get(run_url) - project_url = response["project"] - # `project_url` can have params, such as "?format=json" - if "?" in project_url: - project_url, _ = project_url.split("?") - project_url = project_url.rstrip("/") - results_url = project_url + "/results/" - return request_get(results_url) - - -def map_match_results(match_results): - """ - Given `match_results`, which is a mapping of ScanCode.io codebase results, - return a defaultdict(list) where the keys are the package_uid of matched - packages and the value is a list containing the paths of Resources - associated with the package_uid. - """ - resource_results = match_results.get("files", []) - resource_paths_by_package_uids = defaultdict(list) - for resource in resource_results: - for_packages = resource.get("for_packages", []) - for package_uid in for_packages: - resource_paths_by_package_uids[package_uid].append(resource["path"]) - return resource_paths_by_package_uids - - -def create_packages_from_match_results(project, match_results): - """ - Given `match_results`, which is a mapping of ScanCode.io codebase results, - use the Package data from it to create DiscoveredPackages for `project` and - associate the proper Resources of `project` to the DiscoveredPackages. - """ - from scanpipe.pipes.d2d import create_package_from_purldb_data - - resource_paths_by_package_uids = map_match_results(match_results) - matched_packages = match_results.get("packages", []) - for matched_package in matched_packages: - package_uid = matched_package["package_uid"] - resource_paths = resource_paths_by_package_uids[package_uid] - resources = project.codebaseresources.filter(path__in=resource_paths) - create_package_from_purldb_data( - project, - resources=resources, - package_data=matched_package, - status=flag.MATCHED_TO_PURLDB_PACKAGE, - ) - - def get_next_download_url(timeout=DEFAULT_TIMEOUT, api_url=PURLDB_API_URL): """ Return the ScannableURI UUID, download URL, and pipelines for the next diff --git a/scanpipe/tests/pipes/test_matchcode.py b/scanpipe/tests/pipes/test_matchcode.py index 75cc40e71a..3850cf6217 100644 --- a/scanpipe/tests/pipes/test_matchcode.py +++ b/scanpipe/tests/pipes/test_matchcode.py @@ -82,7 +82,32 @@ def mock_request_post_return(url, files, timeout): @mock.patch("scanpipe.pipes.matchcode.request_get") @mock.patch("scanpipe.pipes.matchcode.is_available") - def test_scanpipe_pipes_matchcode_poll_until_success( + def test_scanpipe_pipes_matchcode_get_run_url_status( + self, mock_is_available, mock_request_get + ): + mock_is_available.return_value = True + + request_get_check_response_loc = ( + self.data_location + / "matchcode" + / "match_to_matchcode" + / "request_get_check_response.json" + ) + with open(request_get_check_response_loc, "r") as f: + mock_request_get_check_return = json.load(f) + + mock_request_get.side_effect = [ + mock_request_get_check_return, + ] + + run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" + status = matchcode.get_run_url_status(run_url) + + self.assertEqual("success", status) + + @mock.patch("scanpipe.pipes.matchcode.request_get") + @mock.patch("scanpipe.pipes.matchcode.is_available") + def test_scanpipe_pipes_matchcode_poll_run_url_status( self, mock_is_available, mock_request_get ): run_status = AbstractTaskFieldsModel.Status @@ -109,7 +134,7 @@ def test_scanpipe_pipes_matchcode_poll_until_success( "status": run_status.SUCCESS, }, ] - return_value = matchcode.poll_until_success(run_url) + return_value = matchcode.poll_run_url_status(run_url) self.assertEqual(True, return_value) # Failure @@ -131,9 +156,14 @@ def test_scanpipe_pipes_matchcode_poll_until_success( "status": run_status.FAILURE, "log": "failure message", }, + { + "url": run_url, + "status": run_status.FAILURE, + "log": "failure message", + }, ] with self.assertRaises(Exception) as context: - matchcode.poll_until_success(run_url) + matchcode.poll_run_url_status(run_url) self.assertTrue("failure message" in str(context.exception)) # Stopped @@ -155,9 +185,14 @@ def test_scanpipe_pipes_matchcode_poll_until_success( "status": run_status.STOPPED, "log": "stop message", }, + { + "url": run_url, + "status": run_status.STOPPED, + "log": "stop message", + }, ] with self.assertRaises(Exception) as context: - matchcode.poll_until_success(run_url) + matchcode.poll_run_url_status(run_url) self.assertTrue("stop message" in str(context.exception)) # Stale @@ -179,9 +214,14 @@ def test_scanpipe_pipes_matchcode_poll_until_success( "status": run_status.STALE, "log": "stale message", }, + { + "url": run_url, + "status": run_status.STALE, + "log": "stale message", + }, ] with self.assertRaises(Exception) as context: - matchcode.poll_until_success(run_url) + matchcode.poll_run_url_status(run_url) self.assertTrue("stale message" in str(context.exception)) def test_scanpipe_pipes_matchcode_map_match_results(self): diff --git a/scanpipe/tests/pipes/test_purldb.py b/scanpipe/tests/pipes/test_purldb.py index 8c2d5e0bb4..5d4efe0cc1 100644 --- a/scanpipe/tests/pipes/test_purldb.py +++ b/scanpipe/tests/pipes/test_purldb.py @@ -114,263 +114,6 @@ def mock_request_post_return(url, data, headers, timeout): "1 PURLs were already present in PurlDB index queue", expected_log ) - @mock.patch("scanpipe.pipes.purldb.request_post") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_send_project_json_to_matchcode( - self, mock_is_available, mock_request_post - ): - mock_is_available.return_value = True - - def mock_request_post_return(url, files, timeout): - request_post_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_post_response.json" - ) - with open(request_post_response_loc, "r") as f: - return json.load(f) - - mock_request_post.side_effect = mock_request_post_return - - run_url = purldb.send_project_json_to_matchcode(self.project1) - expected_run_url = ( - "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - ) - self.assertEqual(expected_run_url, run_url) - - @mock.patch("scanpipe.pipes.purldb.request_get") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_poll_run_url_status( - self, mock_is_available, mock_request_get - ): - run_status = AbstractTaskFieldsModel.Status - - mock_is_available.return_value = True - - # Success - run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.SUCCESS, - }, - ] - return_value = purldb.poll_run_url_status(run_url) - self.assertEqual(True, return_value) - - # Failure - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.FAILURE, - "log": "failure message", - }, - { - "url": run_url, - "status": run_status.FAILURE, - "log": "failure message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_run_url_status(run_url) - self.assertTrue("failure message" in str(context.exception)) - - # Stopped - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.STOPPED, - "log": "stop message", - }, - { - "url": run_url, - "status": run_status.STOPPED, - "log": "stop message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_run_url_status(run_url) - self.assertTrue("stop message" in str(context.exception)) - - # Stale - mock_request_get.side_effect = [ - { - "url": run_url, - "status": run_status.NOT_STARTED, - }, - { - "url": run_url, - "status": run_status.QUEUED, - }, - { - "url": run_url, - "status": run_status.RUNNING, - }, - { - "url": run_url, - "status": run_status.STALE, - "log": "stale message", - }, - { - "url": run_url, - "status": run_status.STALE, - "log": "stale message", - }, - ] - with self.assertRaises(Exception) as context: - purldb.poll_run_url_status(run_url) - self.assertTrue("stale message" in str(context.exception)) - - def test_scanpipe_pipes_purldb_map_match_results(self): - request_post_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_post_response_loc, "r") as f: - match_results = json.load(f) - - resource_paths_by_package_uids = purldb.map_match_results(match_results) - expected = defaultdict(list) - expected_package_uid = ( - "pkg:maven/org.elasticsearch/elasticsearch-x-content@7.17.9" - "?classifier=sources&uuid=a8814800-8120-4f50-ba4f-08c443ccda8e" - ) - expected[expected_package_uid].append( - "elasticsearch-x-content-7.17.9-sources.jar" - ) - self.assertEqual(expected, resource_paths_by_package_uids) - - def test_scanpipe_pipes_purldb_create_packages_from_match_results(self): - r1 = make_resource_file( - self.project1, - path="elasticsearch-x-content-7.17.9-sources.jar", - sha1="30d21add57abe04beece3f28a079671dbc9043e4", - ) - r2 = make_resource_file( - self.project1, - path="something-else.json", - sha1="deadbeef", - ) - - request_get_results_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_get_results_response_loc, "r") as f: - match_results = json.load(f) - - self.assertEqual(0, self.project1.discoveredpackages.all().count()) - self.assertFalse(0, len(r1.for_packages)) - self.assertFalse(0, len(r2.for_packages)) - - purldb.create_packages_from_match_results(self.project1, match_results) - - self.assertEqual(1, self.project1.discoveredpackages.all().count()) - package = self.project1.discoveredpackages.first() - self.assertEqual([package.package_uid], r1.for_packages) - # This resource should not have a Package match - self.assertFalse(0, len(r2.for_packages)) - - @mock.patch("scanpipe.pipes.purldb.request_get") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_get_match_results( - self, mock_is_available, mock_request_get - ): - mock_is_available.return_value = True - - request_get_check_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_check_response.json" - ) - with open(request_get_check_response_loc, "r") as f: - mock_request_get_check_return = json.load(f) - - request_get_results_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_results_response.json" - ) - with open(request_get_results_response_loc, "r") as f: - mock_request_get_results_return = json.load(f) - mock_request_get.side_effect = [ - mock_request_get_check_return, - mock_request_get_results_return, - ] - - run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - match_results = purldb.get_match_results(run_url) - - self.assertEqual(mock_request_get_results_return, match_results) - - @mock.patch("scanpipe.pipes.purldb.request_get") - @mock.patch("scanpipe.pipes.purldb.is_available") - def test_scanpipe_pipes_purldb_get_run_url_status( - self, mock_is_available, mock_request_get - ): - mock_is_available.return_value = True - - request_get_check_response_loc = ( - self.data_location - / "purldb" - / "match_to_purldb" - / "request_get_check_response.json" - ) - with open(request_get_check_response_loc, "r") as f: - mock_request_get_check_return = json.load(f) - - mock_request_get.side_effect = [ - mock_request_get_check_return, - ] - - run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/" - status = purldb.get_run_url_status(run_url) - - self.assertEqual("success", status) - @mock.patch("scanpipe.pipes.purldb.request_get") @mock.patch("scanpipe.pipes.purldb.is_available") def test_scanpipe_pipes_purldb_get_next_download_url( @@ -405,6 +148,17 @@ def test_scanpipe_pipes_purldb_get_next_download_url( results = purldb.get_next_download_url() self.assertFalse(results) + def test_scanpipe_pipes_purldb_get_run_status(self): + now = timezone.now() + run = self.create_run( + pipeline="succeed", + task_start_date=now, + task_end_date=now, + task_exitcode=0, + ) + status = purldb.get_run_status(run=run) + self.assertEqual("success", status) + def test_scanpipe_pipes_purldb_poll_run_status(self): now = timezone.now() From 2fb7e122b2f499dac7ff6fbaa6340fd399f4d983 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 26 Mar 2024 20:23:25 -0700 Subject: [PATCH 30/30] Move Command methods into functions Signed-off-by: Jono Yang --- scanpipe/management/commands/__init__.py | 368 ++++++++++++++--------- 1 file changed, 220 insertions(+), 148 deletions(-) diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 6879827f74..e8395c4740 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -189,67 +189,29 @@ def extract_tag_from_input_files(input_files): For example: "/path/to/file.zip:tag" """ - input_files_data = {} - for file in input_files: - if ":" in file: - key, value = file.split(":", maxsplit=1) - input_files_data.update({key: value}) - else: - input_files_data.update({file: ""}) - return input_files_data + return extract_tag_from_input_files(input_files=input_files) def handle_input_files(self, input_files_data): """Copy provided `input_files` to the project's `input` directory.""" - copied = [] - - for file_location, tag in input_files_data.items(): - self.project.copy_input_from(file_location) - filename = Path(file_location).name - copied.append(filename) - self.project.add_input_source( - filename=filename, - is_uploaded=True, - tag=tag, - ) - - msg = f"File{pluralize(copied)} copied to the project inputs directory:" - self.stdout.write(msg, self.style.SUCCESS) - msg = "\n".join(["- " + filename for filename in copied]) - self.stdout.write(msg) + handle_input_files( + project=self.project, input_files_data=input_files_data, command=self + ) @staticmethod def validate_input_files(input_files): """Raise an error if one of the provided `input_files` entry does not exist.""" - for file_location in input_files: - file_path = Path(file_location) - if not file_path.is_file(): - raise CommandError(f"{file_location} not found or not a file") + validate_input_files(input_files=input_files) def handle_input_urls(self, input_urls): """ Fetch provided `input_urls` and stores it in the project's `input` directory. """ - downloads, errors = fetch_urls(input_urls) - - if downloads: - self.project.add_downloads(downloads) - msg = "File(s) downloaded to the project inputs directory:" - self.stdout.write(msg, self.style.SUCCESS) - msg = "\n".join(["- " + downloaded.filename for downloaded in downloads]) - self.stdout.write(msg) - - if errors: - msg = "Could not fetch URL(s):\n" - msg += "\n".join(["- " + url for url in errors]) - self.stderr.write(msg) + handle_input_urls(project=self.project, input_urls=input_urls, command=self) def handle_copy_codebase(self, copy_from): """Copy `codebase_path` tree to the project's `codebase` directory.""" - project_codebase = self.project.codebase_path - msg = f"{copy_from} content copied in {project_codebase}" - self.stdout.write(msg, self.style.SUCCESS) - shutil.copytree(src=copy_from, dst=project_codebase, dirs_exist_ok=True) + handle_copy_codebase(project=self.project, copy_from=copy_from, command=self) def validate_copy_from(copy_from): @@ -293,6 +255,210 @@ def validate_pipelines(pipelines_data): return pipelines_data +def extract_tag_from_input_files(input_files): + """ + Add support for the ":tag" suffix in file location. + + For example: "/path/to/file.zip:tag" + """ + input_files_data = {} + for file in input_files: + if ":" in file: + key, value = file.split(":", maxsplit=1) + input_files_data.update({key: value}) + else: + input_files_data.update({file: ""}) + return input_files_data + + +def validate_input_files(input_files): + """Raise an error if one of the provided `input_files` entry does not exist.""" + for file_location in input_files: + file_path = Path(file_location) + if not file_path.is_file(): + raise CommandError(f"{file_location} not found or not a file") + + +def validate_project_inputs(pipelines, input_files, copy_from): + """ + Validate `pipelines`, `input_files`, and `copy_from`, returning a tuple + of dictionaries containing the pipeline data of `pipelines` and the + input files data from `input_files. + """ + pipelines_data = {} + input_files_data = {} + + if pipelines: + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + + if input_files: + input_files_data = extract_tag_from_input_files(input_files) + validate_input_files(input_files=input_files_data.keys()) + + if copy_from: + validate_copy_from(copy_from) + + return pipelines_data, input_files_data + + +def handle_input_files(project, input_files_data, command=None): + """Copy provided `input_files` to the project's `input` directory.""" + copied = [] + + for file_location, tag in input_files_data.items(): + project.copy_input_from(file_location) + filename = Path(file_location).name + copied.append(filename) + project.add_input_source( + filename=filename, + is_uploaded=True, + tag=tag, + ) + + if command: + msg = f"File{pluralize(copied)} copied to the project inputs directory:" + command.stdout.write(msg, command.style.SUCCESS) + msg = "\n".join(["- " + filename for filename in copied]) + command.stdout.write(msg) + + +def handle_input_urls(project, input_urls, command=None): + """ + Fetch provided `input_urls` and stores it in the project's `input` + directory. + """ + downloads, errors = fetch_urls(input_urls) + + if downloads: + project.add_downloads(downloads) + msg = "File(s) downloaded to the project inputs directory:" + if command: + command.stdout.write(msg, command.style.SUCCESS) + msg = "\n".join(["- " + downloaded.filename for downloaded in downloads]) + command.stdout.write(msg) + + if errors and command: + msg = "Could not fetch URL(s):\n" + msg += "\n".join(["- " + url for url in errors]) + command.stderr.write(msg) + + +def handle_copy_codebase(project, copy_from, command=None): + """Copy `codebase_path` tree to the project's `codebase` directory.""" + project_codebase = project.codebase_path + if command: + msg = f"{copy_from} content copied in {project_codebase}" + command.stdout.write(msg, command.style.SUCCESS) + shutil.copytree(src=copy_from, dst=project_codebase, dirs_exist_ok=True) + + +def add_project_inputs( + project, pipelines_data, input_files_data, input_urls, copy_from, command=None +): + for pipeline_name, selected_groups in pipelines_data.items(): + project.add_pipeline(pipeline_name, selected_groups=selected_groups) + + if input_files_data: + handle_input_files( + project=project, input_files_data=input_files_data, command=command + ) + + if input_urls: + handle_input_urls(project=project, input_urls=input_urls, command=command) + + if copy_from: + handle_copy_codebase(project=project, copy_from=copy_from, command=command) + + +def execute_project(project, run_async=False, command=None): + run = project.get_next_run() + + if not run: + raise CommandError(f"No pipelines to run on project {project}") + + if run_async: + if not settings.SCANCODEIO_ASYNC: + msg = "SCANCODEIO_ASYNC=False is not compatible with --async option." + raise CommandError(msg) + + run.start() + if command: + msg = f"{run.pipeline_name} added to the tasks queue for execution." + command.stdout.write(msg, command.style.SUCCESS) + else: + command.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") + + try: + tasks.execute_pipeline_task(run.pk) + except KeyboardInterrupt: + run.set_task_stopped() + raise CommandError("Pipeline execution stopped.") + except Exception as e: + run.set_task_ended(exitcode=1, output=str(e)) + raise CommandError(e) + + run.refresh_from_db() + + if run.task_succeeded and command: + msg = f"{run.pipeline_name} successfully executed on " f"project {project}" + command.stdout.write(msg, command.style.SUCCESS) + else: + msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" + raise CommandError(msg) + + +def create_project( + name, + pipelines=None, + input_files=None, + input_urls=None, + copy_from="", + notes="", + execute=False, + run_async=False, + command=None, +): + if execute and not pipelines: + raise CommandError("The execute argument requires one or more pipelines.") + + project = Project(name=name) + if notes: + project.notes = notes + + try: + project.full_clean(exclude=["slug"]) + except ValidationError as e: + raise CommandError("\n".join(e.messages)) + + # Run validation before creating the project in the database + pipelines_data, input_files_data = validate_project_inputs( + pipelines=pipelines, input_files=input_files, copy_from=copy_from + ) + + project.save() + if command: + command.project = project + + if command: + msg = f"Project {name} created with work directory {project.work_directory}" + command.stdout.write(msg, command.style.SUCCESS) + + add_project_inputs( + project=project, + pipelines_data=pipelines_data, + input_files_data=input_files_data, + input_urls=input_urls, + copy_from=copy_from, + command=command, + ) + + if execute: + execute_project(project=project, run_async=run_async, command=command) + + return project + + class ExecuteProjectCommandMixin: def add_arguments(self, parser): super().add_arguments(parser) @@ -307,42 +473,7 @@ def add_arguments(self, parser): ) def execute_project(self, run_async=False): - run = self.project.get_next_run() - - if not run: - raise CommandError(f"No pipelines to run on project {self.project}") - - if run_async: - if not settings.SCANCODEIO_ASYNC: - msg = "SCANCODEIO_ASYNC=False is not compatible with --async option." - raise CommandError(msg) - - run.start() - msg = f"{run.pipeline_name} added to the tasks queue for execution." - self.stdout.write(msg, self.style.SUCCESS) - else: - self.stdout.write(f"Start the {run.pipeline_name} pipeline execution...") - - try: - tasks.execute_pipeline_task(run.pk) - except KeyboardInterrupt: - run.set_task_stopped() - raise CommandError("Pipeline execution stopped.") - except Exception as e: - run.set_task_ended(exitcode=1, output=str(e)) - raise CommandError(e) - - run.refresh_from_db() - - if run.task_succeeded: - msg = ( - f"{run.pipeline_name} successfully executed on " - f"project {self.project}" - ) - self.stdout.write(msg, self.style.SUCCESS) - else: - msg = f"Error during {run.pipeline_name} execution:\n{run.task_output}" - raise CommandError(msg) + execute_project(project=self.project, run_async=run_async, command=self) class CreateProjectCommandMixin(ExecuteProjectCommandMixin): @@ -357,73 +488,14 @@ def create_project( execute=False, run_async=False, ): - if execute and not pipelines: - raise CommandError("The execute argument requires one or more pipelines.") - - project = Project(name=name) - if notes: - project.notes = notes - - try: - project.full_clean(exclude=["slug"]) - except ValidationError as e: - raise CommandError("\n".join(e.messages)) - - # Run validation before creating the project in the database - pipelines_data, input_files_data = self._validate_project_inputs( - pipelines=pipelines, input_files=input_files, copy_from=copy_from - ) - - project.save() - self.project = project - msg = f"Project {name} created with work directory {project.work_directory}" - self.stdout.write(msg, self.style.SUCCESS) - - self._add_project_inputs( - pipelines_data=pipelines_data, - input_files_data=input_files_data, + return create_project( + name=name, + pipelines=pipelines, + input_files=input_files, input_urls=input_urls, copy_from=copy_from, + notes=notes, + execute=execute, + run_async=run_async, + command=self, ) - - if execute: - self.execute_project(run_async=run_async) - - return project - - def _validate_project_inputs(self, pipelines, input_files, copy_from): - """ - Validate `pipelines`, `input_files`, and `copy_from`, returning a tuple - of dictionaries containing the pipeline data of `pipelines` and the - input files data from `input_files. - """ - pipelines_data = {} - input_files_data = {} - - if pipelines: - pipelines_data = extract_group_from_pipelines(pipelines) - pipelines_data = validate_pipelines(pipelines_data) - - if input_files: - input_files_data = self.extract_tag_from_input_files(input_files) - self.validate_input_files(input_files=input_files_data.keys()) - - if copy_from: - validate_copy_from(copy_from) - - return pipelines_data, input_files_data - - def _add_project_inputs( - self, pipelines_data, input_files_data, input_urls, copy_from - ): - for pipeline_name, selected_groups in pipelines_data.items(): - self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) - - if input_files_data: - self.handle_input_files(input_files_data) - - if input_urls: - self.handle_input_urls(input_urls) - - if copy_from: - self.handle_copy_codebase(copy_from)