diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 184725f7f1..71f1bd88be 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,13 +4,18 @@ Changelog v33.2.0 (unreleased) -------------------- +- Add ability to "group" pipeline steps to control their inclusion in a pipeline run. + The groups can be selected in the UI, or provided using the + "pipeline_name:group1,group2" syntax in CLI and REST API. + https://github.com/nexB/scancode.io/issues/1045 + - Refine pipeline choices in the "Add pipeline" modal based on the project context. * When there is at least one existing pipeline in the project, the modal now includes all addon pipelines along with the existing pipeline for selection. * In cases where no pipelines are assigned to the project, the modal displays all base (non-addon) pipelines for user selection. - https://github.com/nexB/scancode.io/issues/ + https://github.com/nexB/scancode.io/issues/1071 v33.1.0 (2024-02-02) -------------------- diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index e370172054..466a255a59 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -84,6 +84,11 @@ Optional arguments: - ``--pipeline PIPELINES`` Pipelines names to add on the project. +.. tip:: + Use the "pipeline_name:group1,group2" syntax to select steps groups: + + ``--pipeline map_deploy_to_develop:Java,JavaScript`` + - ``--input-file INPUTS_FILES`` Input file locations to copy in the :guilabel:`input/` work directory. @@ -190,6 +195,11 @@ add the docker pipeline to your project:: $ scanpipe add-pipeline --project foo analyze_docker_image +.. tip:: + Use the "pipeline_name:group1,group2" syntax to select steps groups: + + ``--pipeline map_deploy_to_develop:Java,JavaScript`` + `$ scanpipe execute --project PROJECT` -------------------------------------- @@ -201,6 +211,7 @@ Optional arguments: - ``--async`` Add the pipeline run to the tasks queue for execution by a worker instead of running in the current thread. + `$ scanpipe show-pipeline --project PROJECT` -------------------------------------------- diff --git a/docs/rest-api.rst b/docs/rest-api.rst index b9e65f3c52..bf544c2b05 100644 --- a/docs/rest-api.rst +++ b/docs/rest-api.rst @@ -278,6 +278,11 @@ Data: - ``pipeline``: The pipeline name - ``execute_now``: ``true`` or ``false`` +.. tip:: + Use the "pipeline_name:group1,group2" syntax to select steps groups: + + ``"pipeline": "map_deploy_to_develop:Java,JavaScript"`` + Using cURL: .. code-block:: console diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py index 1add425497..f27c5ceb99 100644 --- a/scanpipe/api/views.py +++ b/scanpipe/api/views.py @@ -247,10 +247,11 @@ def add_pipeline(self, request, *args, **kwargs): pipeline = request.data.get("pipeline") if pipeline: - pipeline = scanpipe_app.get_new_pipeline_name(pipeline) - if pipeline in scanpipe_app.pipelines: + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline) + pipeline_name = scanpipe_app.get_new_pipeline_name(pipeline_name) + if pipeline_name in scanpipe_app.pipelines: execute_now = request.data.get("execute_now") - project.add_pipeline(pipeline, execute_now) + project.add_pipeline(pipeline_name, execute_now, selected_groups=groups) return Response({"status": "Pipeline added."}) message = {"status": f"{pipeline} is not a valid pipeline."} diff --git a/scanpipe/apps.py b/scanpipe/apps.py index a74314a309..8f46096a96 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -189,6 +189,17 @@ def get_new_pipeline_name(pipeline_name): return new_name return pipeline_name + @staticmethod + def extract_group_from_pipeline(pipeline): + pipeline_name = pipeline + groups = None + + if ":" in pipeline: + pipeline_name, value = pipeline.split(":", maxsplit=1) + groups = value.split(",") if value else [] + + return pipeline_name, groups + def get_scancode_licenses(self): """ Load licenses-related information from the ScanCode-toolkit ``licensedcode`` diff --git a/scanpipe/forms.py b/scanpipe/forms.py index 8580e95783..f0c65c991d 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -100,6 +100,14 @@ def handle_inputs(self, project): project.add_input_source(download_url=url) +class GroupChoiceField(forms.MultipleChoiceField): + widget = forms.CheckboxSelectMultiple + + def valid_value(self, value): + """Accept all values.""" + return True + + class PipelineBaseForm(forms.Form): pipeline = forms.ChoiceField( choices=scanpipe_app.get_pipeline_choices(), @@ -110,12 +118,14 @@ class PipelineBaseForm(forms.Form): initial=True, required=False, ) + selected_groups = GroupChoiceField(required=False) def handle_pipeline(self, project): pipeline = self.cleaned_data["pipeline"] execute_now = self.cleaned_data["execute_now"] + selected_groups = self.cleaned_data.get("selected_groups", []) if pipeline: - project.add_pipeline(pipeline, execute_now) + project.add_pipeline(pipeline, execute_now, selected_groups) class ProjectForm(InputsBaseForm, PipelineBaseForm, forms.ModelForm): @@ -127,6 +137,7 @@ class Meta: "input_urls", "pipeline", "execute_now", + "selected_groups", ] def __init__(self, *args, **kwargs): @@ -158,32 +169,11 @@ def save(self, project): class AddPipelineForm(PipelineBaseForm): pipeline = forms.ChoiceField( + choices=scanpipe_app.get_pipeline_choices(), widget=forms.RadioSelect(), required=True, ) - def __init__(self, project_runs=None, *args, **kwargs): - super().__init__(*args, **kwargs) - - # The pipeline choices are determined based on the project context: - # 1. If no pipelines are assigned to the project: - # Include all base (non-addon) pipelines. - # 2. If at least one pipeline already exists on the project: - # Include all addon pipelines and the existing pipeline (useful for - # potential re-runs in debug mode). - project_run_names = {run.pipeline_name for run in project_runs or []} - - pipeline_choices = [ - (name, pipeline_class.get_summary()) - for name, pipeline_class in scanpipe_app.pipelines.items() - # no pipelines are assigned to the project - if (not project_runs and not pipeline_class.is_addon) - # at least one pipeline already exists on the project - or (project_runs and (name in project_run_names or pipeline_class.is_addon)) - ] - - self.fields["pipeline"].choices = pipeline_choices - def save(self, project): self.handle_pipeline(project) return project diff --git a/scanpipe/management/commands/__init__.py b/scanpipe/management/commands/__init__.py index 3cbc6b529f..6c3c1051e9 100644 --- a/scanpipe/management/commands/__init__.py +++ b/scanpipe/management/commands/__init__.py @@ -152,14 +152,22 @@ def add_arguments(self, parser): action="append", dest="input_files", default=list(), - help="Input file locations to copy in the input/ work directory.", + help=( + "Input file locations to copy in the input/ work directory. " + 'Use the "filename:tag" syntax to tag input files such as ' + '"path/filename:tag"' + ), ) parser.add_argument( "--input-url", action="append", dest="input_urls", default=list(), - help="Input URLs to download in the input/ work directory.", + help=( + "Input URLs to download in the input/ work directory. " + 'Use the "url#tag" syntax to tag downloaded files such as ' + '"https://url.com/filename#tag"' + ), ) parser.add_argument( "--copy-codebase", @@ -251,19 +259,32 @@ def validate_copy_from(copy_from): raise CommandError(f"{copy_from} is not a directory") -def validate_pipelines(pipeline_names): +def extract_group_from_pipelines(pipelines): + """ + Add support for the ":group1,group2" suffix in pipeline data. + + For example: "map_deploy_to_develop:Java,JavaScript" + """ + pipelines_data = {} + for pipeline in pipelines: + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline) + pipelines_data[pipeline_name] = groups + return pipelines_data + + +def validate_pipelines(pipelines_data): """Raise an error if one of the `pipeline_names` is not available.""" # Backward compatibility with old pipeline names. - pipeline_names = [ - scanpipe_app.get_new_pipeline_name(pipeline_name) - for pipeline_name in pipeline_names - ] + pipelines_data = { + scanpipe_app.get_new_pipeline_name(pipeline_name): groups + for pipeline_name, groups in pipelines_data.items() + } - for pipeline_name in pipeline_names: + for pipeline_name in pipelines_data.keys(): if pipeline_name not in scanpipe_app.pipelines: raise CommandError( f"{pipeline_name} is not a valid pipeline. \n" f"Available: {', '.join(scanpipe_app.pipelines.keys())}" ) - return pipeline_names + return pipelines_data diff --git a/scanpipe/management/commands/add-pipeline.py b/scanpipe/management/commands/add-pipeline.py index 417d849226..8d4bcd9401 100644 --- a/scanpipe/management/commands/add-pipeline.py +++ b/scanpipe/management/commands/add-pipeline.py @@ -23,6 +23,7 @@ from django.template.defaultfilters import pluralize from scanpipe.management.commands import ProjectCommand +from scanpipe.management.commands import extract_group_from_pipelines from scanpipe.management.commands import validate_pipelines @@ -38,13 +39,16 @@ def add_arguments(self, parser): help="One or more pipeline names.", ) - def handle(self, *pipeline_names, **options): - super().handle(*pipeline_names, **options) + def handle(self, *pipelines, **options): + super().handle(*pipelines, **options) - pipeline_names = validate_pipelines(pipeline_names) - for pipeline_name in pipeline_names: - self.project.add_pipeline(pipeline_name) + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + for pipeline_name, selected_groups in pipelines_data.items(): + self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) + + pipeline_names = pipelines_data.keys() msg = ( f"Pipeline{pluralize(pipeline_names)} {', '.join(pipeline_names)} " f"added to the project" diff --git a/scanpipe/management/commands/create-project.py b/scanpipe/management/commands/create-project.py index af3464b05f..e1816aa9e6 100644 --- a/scanpipe/management/commands/create-project.py +++ b/scanpipe/management/commands/create-project.py @@ -26,6 +26,7 @@ from django.core.management.base import BaseCommand from scanpipe.management.commands import AddInputCommandMixin +from scanpipe.management.commands import extract_group_from_pipelines from scanpipe.management.commands import validate_copy_from from scanpipe.management.commands import validate_pipelines from scanpipe.models import Project @@ -43,8 +44,9 @@ def add_arguments(self, parser): dest="pipelines", default=list(), help=( - "Pipelines names to add to the project." - "The pipelines are added and executed based on their given order." + "Pipelines names to add to the project. " + "The pipelines are added and executed based on their given order. " + 'Groups can be provided using the "pipeline_name:group1,group2" syntax.' ), ) parser.add_argument( @@ -68,7 +70,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): name = options["name"] - pipeline_names = options["pipelines"] + pipelines = options["pipelines"] input_files = options["input_files"] input_urls = options["input_urls"] copy_from = options["copy_codebase"] @@ -84,22 +86,24 @@ def handle(self, *args, **options): raise CommandError("\n".join(e.messages)) # Run validation before creating the project in the database - pipeline_names = validate_pipelines(pipeline_names) + pipelines_data = extract_group_from_pipelines(pipelines) + pipelines_data = validate_pipelines(pipelines_data) + input_files_data = self.extract_tag_from_input_files(input_files) self.validate_input_files(input_files=input_files_data.keys()) validate_copy_from(copy_from) - if execute and not pipeline_names: + if execute and not pipelines: raise CommandError("The --execute option requires one or more pipelines.") project.save() + self.project = project msg = f"Project {name} created with work directory {project.work_directory}" self.stdout.write(msg, self.style.SUCCESS) - for pipeline_name in pipeline_names: - project.add_pipeline(pipeline_name) + for pipeline_name, selected_groups in pipelines_data.items(): + self.project.add_pipeline(pipeline_name, selected_groups=selected_groups) - self.project = project if input_files: self.handle_input_files(input_files_data) diff --git a/scanpipe/management/commands/show-pipeline.py b/scanpipe/management/commands/show-pipeline.py index e3e8fa4ecd..d180286db9 100644 --- a/scanpipe/management/commands/show-pipeline.py +++ b/scanpipe/management/commands/show-pipeline.py @@ -32,4 +32,7 @@ def handle(self, *args, **options): for run in self.project.runs.all(): status_code = self.get_run_status_code(run) - self.stdout.write(f" [{status_code}] {run.pipeline_name}") + output = f" [{status_code}] {run.pipeline_name}" + if run.selected_groups: + output += f" ({','.join(run.selected_groups)})" + self.stdout.write(output) diff --git a/scanpipe/migrations/0052_run_selected_groups.py b/scanpipe/migrations/0052_run_selected_groups.py new file mode 100644 index 0000000000..db5951b486 --- /dev/null +++ b/scanpipe/migrations/0052_run_selected_groups.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.1 on 2024-02-02 11:42 + +import scanpipe.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("scanpipe", "0051_rename_pipelines_data"), + ] + + operations = [ + migrations.AddField( + model_name="run", + name="selected_groups", + field=models.JSONField( + blank=True, + null=True, + validators=[scanpipe.models.validate_none_or_list], + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index d7efabdf71..bcf7eadd29 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -691,7 +691,9 @@ def clone( if copy_pipelines: for run in self.runs.all(): - cloned_project.add_pipeline(run.pipeline_name, execute_now) + cloned_project.add_pipeline( + run.pipeline_name, execute_now, selected_groups=run.selected_groups + ) if copy_subscriptions: for subscription in self.webhooksubscriptions.all(): @@ -1025,7 +1027,7 @@ def add_uploads(self, uploads): for uploaded_file in uploads: self.add_upload(uploaded_file) - def add_pipeline(self, pipeline_name, execute_now=False): + def add_pipeline(self, pipeline_name, execute_now=False, selected_groups=None): """ Create a new Run instance with the provided `pipeline` on the current project. @@ -1039,10 +1041,13 @@ def add_pipeline(self, pipeline_name, execute_now=False): if not pipeline_class: raise ValueError(f"Unknown pipeline: {pipeline_name}") + validate_none_or_list(selected_groups) + run = Run.objects.create( project=self, pipeline_name=pipeline_name, description=pipeline_class.get_summary(), + selected_groups=selected_groups, ) # Do not start the pipeline execution, even if explicitly requested, @@ -1585,6 +1590,11 @@ def queued_or_running(self): return self.filter(task_id__isnull=False, task_end_date__isnull=True) +def validate_none_or_list(value): + if value is not None and not isinstance(value, list): + raise ValidationError("Value must be a list.") + + class Run(UUIDPKModel, ProjectRelatedModel, AbstractTaskFieldsModel): """The Database representation of a pipeline execution.""" @@ -1596,6 +1606,9 @@ class Run(UUIDPKModel, ProjectRelatedModel, AbstractTaskFieldsModel): scancodeio_version = models.CharField(max_length=30, blank=True) description = models.TextField(blank=True) current_step = models.CharField(max_length=256, blank=True) + selected_groups = models.JSONField( + null=True, blank=True, validators=[validate_none_or_list] + ) objects = RunQuerySet.as_manager() diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 874148cc4a..11ad697aec 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -23,7 +23,6 @@ import inspect import logging import traceback -import warnings from contextlib import contextmanager from functools import wraps from pydoc import getdoc @@ -43,6 +42,19 @@ class InputFileError(Exception): """InputFile is missing or cannot be downloaded.""" +def group(*groups): + """Mark a function as part of a particular group.""" + + def decorator(obj): + if hasattr(obj, "groups"): + obj.groups = obj.groups.union(groups) + else: + setattr(obj, "groups", set(groups)) + return obj + + return decorator + + class BasePipeline: """Base class for all pipelines.""" @@ -63,19 +75,28 @@ def steps(cls): raise NotImplementedError @classmethod - def get_steps(cls): + def get_steps(cls, groups=None): """ - Raise a deprecation warning when the steps are defined as a tuple instead of - a classmethod. + Return the list of steps defined in the ``steps`` class method. + + If the optional ``groups`` parameter is provided, only include steps labeled + with groups that intersect with the provided list. If a step has no groups or + if ``groups`` is not specified, include the step in the result. """ - if callable(cls.steps): - return cls.steps() + if not callable(cls.steps): + raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.") - warnings.warn( - f"Defining ``steps`` as a tuple is deprecated in {cls} " - f"Use a ``steps(cls)`` classmethod instead." - ) - return cls.steps + steps = cls.steps() + + if groups is not None: + steps = tuple( + step + for step in steps + if not getattr(step, "groups", []) + or set(getattr(step, "groups")).intersection(groups) + ) + + return steps @classmethod def get_doc(cls): @@ -86,7 +107,12 @@ def get_doc(cls): def get_graph(cls): """Return a graph of steps.""" return [ - {"name": step.__name__, "doc": getdoc(step)} for step in cls.get_steps() + { + "name": step.__name__, + "doc": getdoc(step), + "groups": getattr(step, "groups", []), + } + for step in cls.get_steps() ] @classmethod @@ -97,6 +123,7 @@ def get_info(cls): "summary": summary, "description": description, "steps": cls.get_graph(), + "available_groups": cls.get_available_groups(), } @classmethod @@ -104,6 +131,16 @@ def get_summary(cls): """Get the doc string summary.""" return cls.get_info()["summary"] + @classmethod + def get_available_groups(cls): + return sorted( + set( + group_name + for step in cls.get_steps() + for group_name in getattr(step, "groups", []) + ) + ) + def log(self, message): """Log the given `message` to the current module logger and Run instance.""" now_as_localtime = timezone.localtime(timezone.now()) @@ -115,7 +152,8 @@ def log(self, message): def execute(self): """Execute each steps in the order defined on this pipeline class.""" self.log(f"Pipeline [{self.pipeline_name}] starting") - steps = self.get_steps() + + steps = self.get_steps(groups=self.run.selected_groups) if self.download_inputs: steps = (self.__class__.download_missing_inputs,) + steps diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 54ad7a5be3..1351dcfa43 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -22,6 +22,7 @@ from scanpipe import pipes from scanpipe.pipelines import Pipeline +from scanpipe.pipelines import group from scanpipe.pipes import d2d from scanpipe.pipes import flag from scanpipe.pipes import matchcode @@ -151,10 +152,7 @@ def fingerprint_codebase_directories(self): matchcode.fingerprint_codebase_directories(self.project, to_codebase_only=True) def flag_whitespace_files(self): - """ - Flag whitespace files with size less than or equal - to 100 byte as ignored. - """ + """Flag whitespace files with size less than or equal to 100 byte as ignored.""" d2d.flag_whitespace_files(project=self.project) def map_about_files(self): @@ -178,18 +176,22 @@ def match_archives_to_purldb(self): logger=self.log, ) + @group("Java") def find_java_packages(self): """Find the java package of the .java source files.""" d2d.find_java_packages(self.project, logger=self.log) + @group("Java") def map_java_to_class(self): """Map a .class compiled file to its .java source.""" d2d.map_java_to_class(project=self.project, logger=self.log) + @group("Java") def map_jar_to_source(self): """Map .jar files to their related source directory.""" d2d.map_jar_to_source(project=self.project, logger=self.log) + @group("JavaScript") def map_javascript(self): """ Map a packed or minified JavaScript, TypeScript, CSS and SCSS @@ -221,18 +223,22 @@ def match_resources_to_purldb(self): logger=self.log, ) + @group("JavaScript") def map_javascript_post_purldb_match(self): """Map minified javascript file based on existing PurlDB match.""" d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log) + @group("JavaScript") def map_javascript_path(self): """Map javascript file based on path.""" d2d.map_javascript_path(project=self.project, logger=self.log) + @group("JavaScript") def map_javascript_colocation(self): """Map JavaScript files based on neighborhood file mapping.""" d2d.map_javascript_colocation(project=self.project, logger=self.log) + @group("JavaScript") def map_thirdparty_npm_packages(self): """Map thirdparty package using package.json metadata.""" d2d.map_thirdparty_npm_packages(project=self.project, logger=self.log) diff --git a/scanpipe/templates/scanpipe/modals/add_pipeline_modal.html b/scanpipe/templates/scanpipe/modals/add_pipeline_modal.html index 5a2cf3e68d..d5e7e65f23 100644 --- a/scanpipe/templates/scanpipe/modals/add_pipeline_modal.html +++ b/scanpipe/templates/scanpipe/modals/add_pipeline_modal.html @@ -23,13 +23,25 @@ {% endif %}
- {% for label, summary in add_pipeline_form.pipeline.field.choices %} + {% for pipeline_name, pipeline_info in pipeline_choices %}
-

{{ summary }}

+

{{ pipeline_info.summary }}

+ {% if pipeline_info.available_groups %} +
+ {% for group in pipeline_info.available_groups %} + + {% endfor %} +
+ {% endif %}
{% endfor %}
diff --git a/scanpipe/templates/scanpipe/modals/run_modal_content.html b/scanpipe/templates/scanpipe/modals/run_modal_content.html index 646fb9823c..5820779312 100644 --- a/scanpipe/templates/scanpipe/modals/run_modal_content.html +++ b/scanpipe/templates/scanpipe/modals/run_modal_content.html @@ -15,6 +15,19 @@ {% include "scanpipe/includes/run_status_tag.html" with run=run only %}
+ {% if run.selected_groups %} +
+
+ Selected groups + + {% for group in run.selected_groups %} + {% if not forloop.first %},{% endif %} + {{ group }} + {% endfor %} + +
+
+ {% endif %} {% if run.status == run.Status.NOT_STARTED or run.status == run.Status.QUEUED %} diff --git a/scanpipe/templates/scanpipe/project_form.html b/scanpipe/templates/scanpipe/project_form.html index 2759f4b415..6f4aa1c888 100644 --- a/scanpipe/templates/scanpipe/project_form.html +++ b/scanpipe/templates/scanpipe/project_form.html @@ -53,7 +53,15 @@

- +
+ + +
{{ form.pipeline }} @@ -62,10 +70,7 @@

- + {{ form.selected_groups }}
@@ -98,15 +103,26 @@

Pipelines:

-
+

{{ pipeline_info.summary }}

{% if pipeline_info.description %} -

{{ pipeline_info.description|linebreaks }}

+

+ {{ pipeline_info.description|linebreaksbr }} +

{% endif %}
{% for step in pipeline_info.steps %} {{ step.name }} + {% if step.groups %} +
+ {% for group in step.groups %} + {{ group }} + {% endfor %} +
+ {% endif %}
{{ step.doc }}
{% if not forloop.last %}
↓
{% endif %} {% endfor %} @@ -125,28 +141,78 @@

Pipelines:

{% block scripts %} + + {{ pipelines_available_groups|json_script:"pipelines_available_groups" }} + {% endblock %} \ No newline at end of file diff --git a/scanpipe/tests/pipelines/with_groups.py b/scanpipe/tests/pipelines/with_groups.py new file mode 100644 index 0000000000..1b00e3f16b --- /dev/null +++ b/scanpipe/tests/pipelines/with_groups.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from scanpipe.pipelines import Pipeline +from scanpipe.pipelines import group + + +class WithGroups(Pipeline): + """Include "grouped" steps.""" + + @classmethod + def steps(cls): + return ( + cls.grouped_with_foo_and_bar, + cls.grouped_with_bar, + cls.grouped_with_excluded, + cls.no_groups, + ) + + @group("foo", "bar") + def grouped_with_foo_and_bar(self): + """Step1 doc.""" + pass + + @group("bar") + def grouped_with_bar(self): + """Step2 doc.""" + pass + + @group("excluded") + def grouped_with_excluded(self): + """Step2 doc.""" + pass + + def no_groups(self): + """Step2 doc.""" + pass diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 8e6a5bbfa5..859c490e4c 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -527,7 +527,7 @@ def test_scanpipe_api_project_action_results_download(self): def test_scanpipe_api_project_action_pipelines(self): url = reverse("project-pipelines") response = self.csrf_client.get(url) - expected = ["name", "summary", "description", "steps"] + expected = ["name", "summary", "description", "steps", "available_groups"] self.assertEqual(expected, list(response.data[0].keys())) def test_scanpipe_api_project_action_resources(self): @@ -764,6 +764,16 @@ def test_scanpipe_api_project_action_add_pipeline_old_name_compatibility(self): self.assertEqual({"status": "Pipeline added."}, response.data) self.assertEqual("analyze_docker_image", self.project1.runs.get().pipeline_name) + def test_scanpipe_api_project_action_add_pipeline_groups(self): + url = reverse("project-add-pipeline", args=[self.project1.uuid]) + data = { + "pipeline": "analyze_docker_image:group1,group2", + "execute_now": False, + } + response = self.csrf_client.post(url, data=data) + self.assertEqual({"status": "Pipeline added."}, response.data) + self.assertEqual("analyze_docker_image", self.project1.runs.get().pipeline_name) + def test_scanpipe_api_project_action_add_input(self): url = reverse("project-add-input", args=[self.project1.uuid]) response = self.csrf_client.get(url) diff --git a/scanpipe/tests/test_apps.py b/scanpipe/tests/test_apps.py index 6b650eb9f4..f6df16b299 100644 --- a/scanpipe/tests/test_apps.py +++ b/scanpipe/tests/test_apps.py @@ -150,3 +150,26 @@ def test_scanpipe_apps_get_new_pipeline_name(self): self.assertEqual( "analyze_docker_image", scanpipe_app.get_new_pipeline_name("docker") ) + + def test_scanpipe_apps_extract_group_from_pipeline(self): + pipeline = "map_deploy_to_develop" + + pipeline_str = pipeline + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline_str) + self.assertEqual(pipeline, pipeline_name) + self.assertEqual(None, groups) + + pipeline_str = "map_deploy_to_develop:" + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline_str) + self.assertEqual(pipeline, pipeline_name) + self.assertEqual([], groups) + + pipeline_str = "map_deploy_to_develop:group1" + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline_str) + self.assertEqual(pipeline, pipeline_name) + self.assertEqual(["group1"], groups) + + pipeline_str = "map_deploy_to_develop:group1,group2" + pipeline_name, groups = scanpipe_app.extract_group_from_pipeline(pipeline_str) + self.assertEqual(pipeline, pipeline_name) + self.assertEqual(["group1", "group2"], groups) diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 754994d365..e7b23317a9 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -100,7 +100,7 @@ def test_scanpipe_management_command_create_project_pipelines(self): "--pipeline", self.pipeline_name, "--pipeline", - "analyze_root_filesystem_or_vm_image", + "analyze_root_filesystem_or_vm_image:group1,group2", "--pipeline", "scan_package", # old name backward compatibility ] @@ -113,6 +113,8 @@ def test_scanpipe_management_command_create_project_pipelines(self): "scan_single_package", ] self.assertEqual(expected, [run.pipeline_name for run in project.runs.all()]) + run = project.runs.get(pipeline_name="analyze_root_filesystem_or_vm_image") + self.assertEqual(["group1", "group2"], run.selected_groups) def test_scanpipe_management_command_create_project_inputs(self): out = StringIO() @@ -254,7 +256,7 @@ def test_scanpipe_management_command_add_pipeline(self): pipelines = [ self.pipeline_name, - "analyze_root_filesystem_or_vm_image", + "analyze_root_filesystem_or_vm_image:group1,group2", "scan_package", # old name backward compatibility ] @@ -276,6 +278,8 @@ def test_scanpipe_management_command_add_pipeline(self): "scan_single_package", ] self.assertEqual(expected, [run.pipeline_name for run in project.runs.all()]) + run = project.runs.get(pipeline_name="analyze_root_filesystem_or_vm_image") + self.assertEqual(["group1", "group2"], run.selected_groups) options = ["--project", project.name, "non-existing"] expected = "non-existing is not a valid pipeline" @@ -301,13 +305,15 @@ def test_scanpipe_management_command_show_pipeline(self): ) self.assertEqual(expected, out.getvalue()) - project.runs.filter(pipeline_name=pipeline_names[0]).update(task_exitcode=0) + project.runs.filter(pipeline_name=pipeline_names[0]).update( + task_exitcode=0, selected_groups=["group1", "group2"] + ) project.runs.filter(pipeline_name=pipeline_names[1]).update(task_exitcode=1) out = StringIO() call_command("show-pipeline", *options, stdout=out) expected = ( - " [SUCCESS] analyze_docker_image\n" + " [SUCCESS] analyze_docker_image (group1,group2)\n" " [FAILURE] analyze_root_filesystem_or_vm_image\n" ) self.assertEqual(expected, out.getvalue()) diff --git a/scanpipe/tests/test_forms.py b/scanpipe/tests/test_forms.py index ca78328e24..dfcad40289 100644 --- a/scanpipe/tests/test_forms.py +++ b/scanpipe/tests/test_forms.py @@ -27,7 +27,6 @@ import requests -from scanpipe.forms import AddPipelineForm from scanpipe.forms import EditInputSourceTagForm from scanpipe.forms import InputsBaseForm from scanpipe.forms import ProjectForm @@ -92,26 +91,6 @@ def test_scanpipe_forms_project_form_pipeline_choices(self): self.assertIn(main_pipeline, choices) self.assertNotIn(addon_pipeline, choices) - def test_scanpipe_forms_add_pipeline_form_pipeline_choices(self): - main_pipeline1 = "scan_codebase" - main_pipeline2 = "scan_single_package" - addon_pipeline = "find_vulnerabilities" - - form = AddPipelineForm() - choices = form.fields["pipeline"].choices - pipeline_choices = [choice[0] for choice in choices] - self.assertIn(main_pipeline1, pipeline_choices) - self.assertIn(main_pipeline2, pipeline_choices) - self.assertNotIn(addon_pipeline, pipeline_choices) - - self.project1.add_pipeline(pipeline_name=main_pipeline1) - form = AddPipelineForm(project_runs=self.project1.runs.all()) - choices = form.fields["pipeline"].choices - pipeline_choices = [choice[0] for choice in choices] - self.assertIn(main_pipeline1, pipeline_choices) - self.assertNotIn(main_pipeline2, pipeline_choices) - self.assertIn(addon_pipeline, pipeline_choices) - def test_scanpipe_forms_project_settings_form_update_name_and_notes(self): data = { "name": "new name", diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 6ac3c50738..436f04de54 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -36,6 +36,7 @@ from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import call_command from django.db import DataError @@ -228,7 +229,7 @@ def test_scanpipe_project_model_clone(self): self.project1.update(settings={"extract_recursively": True}) new_file_path1 = self.project1.input_path / "file.zip" new_file_path1.touch() - run1 = self.project1.add_pipeline("analyze_docker_image") + run1 = self.project1.add_pipeline("analyze_docker_image", selected_groups=["g"]) run2 = self.project1.add_pipeline("find_vulnerabilities") subscription1 = self.project1.add_webhook_subscription("http://domain.url") @@ -264,6 +265,7 @@ def test_scanpipe_project_model_clone(self): [run.pipeline_name for run in runs], ) self.assertNotEqual(run1.pk, runs[0].pk) + self.assertEqual(run1.selected_groups, runs[0].selected_groups) self.assertNotEqual(run2.pk, runs[1].pk) self.assertEqual(1, len(cloned_project2.webhooksubscriptions.all())) cloned_subscription = cloned_project2.webhooksubscriptions.get() @@ -770,6 +772,20 @@ def test_scanpipe_run_model_set_current_step(self): run1 = Run.objects.get(pk=run1.pk) self.assertEqual("", run1.current_step) + def test_scanpipe_run_model_selected_groups(self): + run1 = Run.objects.create(project=self.project1) + self.assertEqual(None, run1.selected_groups) + + # Empty list has not the same behavior as None + run1.update(selected_groups=[]) + self.assertEqual([], run1.selected_groups) + + run1.update(selected_groups=["foo"]) + self.assertEqual(["foo"], run1.selected_groups) + + run1.update(selected_groups=["foo", "bar"]) + self.assertEqual(["foo", "bar"], run1.selected_groups) + def test_scanpipe_run_model_pipeline_class_property(self): run1 = Run.objects.create(project=self.project1, pipeline_name="do_nothing") self.assertEqual(DoNothing, run1.pipeline_class) @@ -2157,6 +2173,22 @@ def test_scanpipe_project_model_add_pipeline_start_method(self, mock_execute_tas run1.start() mock_execute_task.assert_called_once() + def test_scanpipe_project_model_add_pipeline_selected_groups(self): + project1 = Project.objects.create(name="Analysis") + pipeline_name = "scan_codebase" + + run1 = project1.add_pipeline(pipeline_name, selected_groups=[]) + self.assertEqual([], run1.selected_groups) + + run2 = project1.add_pipeline(pipeline_name, selected_groups=["foo"]) + self.assertEqual(["foo"], run2.selected_groups) + + run3 = project1.add_pipeline(pipeline_name, selected_groups=["foo", "bar"]) + self.assertEqual(["foo", "bar"], run3.selected_groups) + + with self.assertRaises(ValidationError): + project1.add_pipeline(pipeline_name, selected_groups={}) + def test_scanpipe_project_model_add_info(self): project1 = Project.objects.create(name="Analysis") message = project1.add_info(description="This is an info") diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index bf2eec4717..1045b0ad96 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -24,7 +24,6 @@ import os import sys import tempfile -import warnings from pathlib import Path from unittest import mock from unittest import skipIf @@ -51,6 +50,7 @@ from scanpipe.tests.pipelines.do_nothing import DoNothing from scanpipe.tests.pipelines.profile_step import ProfileStep from scanpipe.tests.pipelines.steps_as_attribute import StepsAsAttribute +from scanpipe.tests.pipelines.with_groups import WithGroups from_docker_image = os.environ.get("FROM_DOCKER_IMAGE") @@ -67,9 +67,10 @@ def test_scanpipe_pipelines_class_get_info(self): "description": "Description section of the doc string.", "summary": "Do nothing, in 2 steps.", "steps": [ - {"name": "step1", "doc": "Step1 doc."}, - {"name": "step2", "doc": "Step2 doc."}, + {"name": "step1", "doc": "Step1 doc.", "groups": []}, + {"name": "step2", "doc": "Step2 doc.", "groups": []}, ], + "available_groups": [], } self.assertEqual(expected, DoNothing.get_info()) @@ -77,8 +78,9 @@ def test_scanpipe_pipelines_class_get_info(self): "summary": "Profile a step using the @profile decorator.", "description": "", "steps": [ - {"name": "step", "doc": ""}, + {"name": "step", "doc": "", "groups": []}, ], + "available_groups": [], } self.assertEqual(expected, ProfileStep.get_info()) @@ -213,8 +215,8 @@ class SubSubClass(DoNothing): def test_scanpipe_pipelines_class_get_graph(self): expected = [ - {"doc": "Step1 doc.", "name": "step1"}, - {"doc": "Step2 doc.", "name": "step2"}, + {"name": "step1", "doc": "Step1 doc.", "groups": []}, + {"name": "step2", "doc": "Step2 doc.", "groups": []}, ] self.assertEqual(expected, DoNothing.get_graph()) @@ -242,17 +244,41 @@ def test_scanpipe_pipelines_class_get_steps(self): ) self.assertEqual(expected, DoNothing.get_steps()) - expected = (StepsAsAttribute.step1,) - with warnings.catch_warnings(record=True) as caught_warnings: - self.assertEqual(expected, StepsAsAttribute.get_steps()) - self.assertEqual(len(caught_warnings), 1) - caught_warning = caught_warnings[0] + with self.assertRaises(TypeError) as cm: + StepsAsAttribute.get_steps() + expected = "Use a ``steps(cls)`` classmethod to declare the steps." + self.assertEqual(expected, str(cm.exception)) + def test_scanpipe_pipelines_class_get_steps_with_groups(self): expected = ( - f"Defining ``steps`` as a tuple is deprecated in {StepsAsAttribute} " - f"Use a ``steps(cls)`` classmethod instead." + WithGroups.grouped_with_foo_and_bar, + WithGroups.grouped_with_bar, + WithGroups.grouped_with_excluded, + WithGroups.no_groups, ) - self.assertEqual(expected, str(caught_warning.message)) + self.assertEqual(expected, WithGroups.get_steps()) + + expected = (WithGroups.no_groups,) + self.assertEqual(expected, WithGroups.get_steps(groups=[])) + self.assertEqual(expected, WithGroups.get_steps(groups=["not"])) + + expected = ( + WithGroups.grouped_with_foo_and_bar, + WithGroups.grouped_with_bar, + WithGroups.no_groups, + ) + self.assertEqual(expected, WithGroups.get_steps(groups=["bar"])) + self.assertEqual(expected, WithGroups.get_steps(groups=["foo", "bar"])) + + expected = ( + WithGroups.grouped_with_foo_and_bar, + WithGroups.no_groups, + ) + self.assertEqual(expected, WithGroups.get_steps(groups=["foo"])) + + def test_scanpipe_pipelines_class_get_available_groups(self): + self.assertEqual(["bar", "excluded", "foo"], WithGroups.get_available_groups()) + self.assertEqual([], DoNothing.get_available_groups()) def test_scanpipe_pipelines_class_env_loaded_from_config_file(self): project1 = Project.objects.create(name="Analysis") diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 068cc616fd..55d8430d65 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -325,6 +325,26 @@ def test_scanpipe_views_project_details_add_pipelines(self): self.assertEqual("analyze_docker_image", run.pipeline_name) self.assertIsNone(run.task_start_date) + def test_scanpipe_views_project_details_get_pipeline_choices(self): + main_pipeline1 = "scan_codebase" + main_pipeline2 = "scan_single_package" + addon_pipeline = "find_vulnerabilities" + + choices = ProjectDetailView.get_pipeline_choices(pipeline_runs=[]) + pipeline_choices = [choice[0] for choice in choices] + self.assertIn(main_pipeline1, pipeline_choices) + self.assertIn(main_pipeline2, pipeline_choices) + self.assertNotIn(addon_pipeline, pipeline_choices) + + self.project1.add_pipeline(pipeline_name=main_pipeline1) + choices = ProjectDetailView.get_pipeline_choices( + pipeline_runs=self.project1.runs.all() + ) + pipeline_choices = [choice[0] for choice in choices] + self.assertIn(main_pipeline1, pipeline_choices) + self.assertNotIn(main_pipeline2, pipeline_choices) + self.assertIn(addon_pipeline, pipeline_choices) + def test_scanpipe_views_project_details_add_labels(self): url = self.project1.get_absolute_url() data = { diff --git a/scanpipe/views.py b/scanpipe/views.py index d71ba89eb8..244dd44de9 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -578,11 +578,16 @@ class ProjectCreateView(ConditionalLoginRequired, FormAjaxMixin, generic.CreateV def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["pipelines"] = { + pipelines = { key: pipeline_class.get_info() for key, pipeline_class in scanpipe_app.pipelines.items() if not pipeline_class.is_addon } + pipelines_available_groups = { + name: info["available_groups"] for name, info in pipelines.items() + } + context["pipelines"] = pipelines + context["pipelines_available_groups"] = pipelines_available_groups return context @@ -693,7 +698,8 @@ def get_context_data(self, **kwargs): { "input_sources": project.get_inputs_with_source(), "labels": list(project.labels.all()), - "add_pipeline_form": AddPipelineForm(project_runs=pipeline_runs), + "add_pipeline_form": AddPipelineForm(), + "pipeline_choices": self.get_pipeline_choices(pipeline_runs), "add_inputs_form": AddInputsForm(), "add_labels_form": AddLabelsForm(), "edit_input_tag_from": EditInputSourceTagForm(), @@ -743,6 +749,27 @@ def post(self, request, *args, **kwargs): return redirect(project) + @staticmethod + def get_pipeline_choices(pipeline_runs): + """ + Determine pipeline choices based on the project context: + 1. If no pipelines are assigned to the project: + Include all base (non-addon) pipelines. + 2. If at least one pipeline already exists on the project: + Include all addon pipelines and the existing pipeline (useful for + potential re-runs in debug mode). + """ + project_run_names = {run.pipeline_name for run in pipeline_runs or []} + pipeline_choices = [ + (name, pipeline_class.get_info()) + for name, pipeline_class in scanpipe_app.pipelines.items() + # no pipelines are assigned to the project + if (not pipeline_runs and not pipeline_class.is_addon) + # at least one pipeline already exists on the project + or pipeline_runs and (name in project_run_names or pipeline_class.is_addon) + ] + return pipeline_choices + class ProjectSettingsView(ConditionalLoginRequired, UpdateView): model = Project