diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ffa09b1fda..dbed9fc993 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -60,6 +60,9 @@ v34.5.0 (unreleased) - Remove the ``extract_recursively`` option from the Project configuration. https://github.com/nexB/scancode.io/issues/1236 +- Add support for a ``ignored_dependency_scopes`` field on the Project configuration. + https://github.com/nexB/scancode.io/issues/1197 + - Add support for storing the scancode-config.yml file in codebase. The scancode-config.yml file can be provided as a project input, or can be located in the codebase/ immediate subdirectories. This allows to provide the configuration diff --git a/docs/conf.py b/docs/conf.py index 45829cb767..a9aea67f61 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,6 +67,9 @@ # a list of builtin themes. html_theme = "sphinx_rtd_theme" +# The style name to use for Pygments highlighting of source code. +pygments_style = "emacs" + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/docs/project-configuration.rst b/docs/project-configuration.rst index d972f6d7cb..d4a308a2da 100644 --- a/docs/project-configuration.rst +++ b/docs/project-configuration.rst @@ -40,7 +40,12 @@ Content of a ``scancode-config.yml`` file: product_version: '1.0' ignored_patterns: - '*.tmp' - - tests/* + - 'tests/*' + ignored_dependency_scopes: + - package_type: npm + scope: devDependencies + - package_type: pypi + scope: tests See the :ref:`project_configuration_settings` section for the details about each setting. @@ -49,7 +54,6 @@ setting. You can generate the project configuration file from the :ref:`user_interface_project_settings` UI. - .. _project_configuration_settings: Settings @@ -86,3 +90,41 @@ within the project. .. warning:: Be cautious when specifying patterns to avoid unintended exclusions. + +ignored_dependency_scopes +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Specify certain dependency scopes to be ignored for a given package type. +This allows you to exclude dependencies from being created or resolved based on their +scope. + +**Guidelines:** + +- **Exact Matches Only:** The scope names must be specified exactly as they appear. + Wildcards and partial matches are not supported. +- **Scope Specification:** List each scope name you wish to ignore. + +**Examples:** + +To exclude all ``devDependencies`` for ``npm`` packages and ``tests`` for ``pypi`` +packages, define the following in your ``scancode-config.yml`` configuration file: + +.. code-block:: yaml + + ignored_dependency_scopes: + - package_type: npm + scope: devDependencies + - package_type: pypi + scope: tests + +If you prefer to use the :ref:`user_interface_project_settings` form, list each +ignored scope using the `package_type:scope` syntax, **one per line**, such as: + +.. code-block:: text + + npm:devDependencies + pypi:tests + +.. warning:: + Be precise when listing scope names to avoid unintended exclusions. + Ensure the scope names are correct and reflect your project requirements. diff --git a/scanpipe/forms.py b/scanpipe/forms.py index 9774290afb..781341b4fe 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -276,7 +276,68 @@ def prepare_value(self, value): return value -ignored_patterns_help_markdown = """ +class KeyValueListField(forms.CharField): + """ + A Django form field that displays as a textarea and converts each line of + "key:value" input into a list of dictionaries with customizable keys. + + Each line of the textarea input is split into key-value pairs, + removing leading/trailing whitespace and empty lines. The resulting list of + dictionaries is then stored as the field value. + """ + + widget = forms.Textarea + + def __init__(self, *args, key_name="key", value_name="value", **kwargs): + """Initialize the KeyValueListField with custom key and value names.""" + self.key_name = key_name + self.value_name = value_name + super().__init__(*args, **kwargs) + + def to_python(self, value): + """ + Split the textarea input into lines, convert each line to a dictionary, + and remove empty lines. + """ + if not value: + return None + + items = [] + for line in value.splitlines(): + line = line.strip() + if not line: + continue + parts = line.split(":", 1) + if len(parts) != 2: + raise ValidationError( + f"Invalid input line: '{line}'. " + f"Each line must contain exactly one ':' character." + ) + key, value = parts + key = key.strip() + value = value.strip() + if not key or not value: + raise ValidationError( + f"Invalid input line: '{line}'. " + f"Both key and value must be non-empty." + ) + items.append({self.key_name: key, self.value_name: value}) + + return items + + def prepare_value(self, value): + """ + Join the list of dictionaries into a string with newlines, + using the "key:value" format. + """ + if value is not None and isinstance(value, list): + value = "\n".join( + f"{item[self.key_name]}:{item[self.value_name]}" for item in value + ) + return value + + +ignored_patterns_help = """ Provide one or more path patterns to be ignored, one per line. Each pattern should follow the syntax of Unix shell-style wildcards: @@ -295,10 +356,19 @@ def prepare_value(self, value): Be cautious when specifying patterns to avoid unintended exclusions. """ +ignored_dependency_scopes_help = """ +Specify certain dependency scopes to be ignored for a given package type. + +This allows you to exclude dependencies from being created or resolved based on their +scope using the `package_type:scope` syntax, **one per line**. +For example: `npm:devDependencies` +""" + class ProjectSettingsForm(forms.ModelForm): settings_fields = [ "ignored_patterns", + "ignored_dependency_scopes", "attribution_template", "product_name", "product_version", @@ -306,7 +376,7 @@ class ProjectSettingsForm(forms.ModelForm): ignored_patterns = ListTextarea( label="Ignored patterns", required=False, - help_text=convert_markdown_to_html(ignored_patterns_help_markdown.strip()), + help_text=convert_markdown_to_html(ignored_patterns_help.strip()), widget=forms.Textarea( attrs={ "class": "textarea is-dynamic", @@ -315,6 +385,20 @@ class ProjectSettingsForm(forms.ModelForm): }, ), ) + ignored_dependency_scopes = KeyValueListField( + label="Ignored dependency scopes", + required=False, + help_text=convert_markdown_to_html(ignored_dependency_scopes_help.strip()), + widget=forms.Textarea( + attrs={ + "class": "textarea is-dynamic", + "rows": 2, + "placeholder": "npm:devDependencies\npypi:tests", + }, + ), + key_name="package_type", + value_name="scope", + ) attribution_template = forms.CharField( label="Attribution template", required=False, diff --git a/scanpipe/models.py b/scanpipe/models.py index 47a18f24bf..2c19c2d2bf 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -27,6 +27,7 @@ import shutil import uuid from collections import Counter +from collections import defaultdict from contextlib import suppress from itertools import groupby from operator import itemgetter @@ -759,7 +760,7 @@ def get_input_config_file(self): Priority order: 1. If a config file exists directly in the input/ directory, return it. 2. If exactly one config file exists in a codebase/ immediate subdirectory, - return it. + return it. 3. If multiple config files are found in subdirectories, report an error. """ config_filename = settings.SCANCODEIO_CONFIG_FILE @@ -830,6 +831,29 @@ def get_env(self, field_name=None): return env + def get_ignored_dependency_scopes_index(self): + """ + Return a dictionary index of the ``ignored_dependency_scopes`` setting values + defined in this Project env. + """ + ignored_dependency_scopes = self.get_env(field_name="ignored_dependency_scopes") + if not ignored_dependency_scopes: + return {} + + ignored_scope_index = defaultdict(list) + for entry in ignored_dependency_scopes: + ignored_scope_index[entry.get("package_type")].append(entry.get("scope")) + + return dict(ignored_scope_index) + + @cached_property + def ignored_dependency_scopes_index(self): + """ + Return the computed value of get_ignored_dependency_scopes_index. + The value is only generated once and cached for further calls. + """ + return self.get_ignored_dependency_scopes_index() + def clear_tmp_directory(self): """ Delete the whole content of the tmp/ directory. diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 984d3c50dd..61e2d2a79d 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -216,6 +216,25 @@ def create_local_files_package(project, defaults, codebase_resources=None): return update_or_create_package(project, package_data, codebase_resources) +def ignore_dependency_scope(project, dependency_data): + """ + Return True if the dependency should be ignored, i.e.: not created. + The ignored scopes are defined on the project ``ignored_dependency_scopes`` setting + field. + """ + ignored_scope_index = project.ignored_dependency_scopes_index + if not ignored_scope_index: + return False + + dependency_package_type = dependency_data.get("package_type") + dependency_scope = dependency_data.get("scope") + if dependency_package_type and dependency_scope: + if dependency_scope in ignored_scope_index.get(dependency_package_type, []): + return True # Ignore this dependency entry. + + return False + + def update_or_create_dependency( project, dependency_data, @@ -239,6 +258,9 @@ def update_or_create_dependency( dependency = None dependency_uid = dependency_data.get("dependency_uid") + if ignore_dependency_scope(project, dependency_data): + return # Do not create the DiscoveredDependency record. + if not dependency_uid: dependency_data["dependency_uid"] = uuid.uuid4() else: diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index 79c1be6144..4ba42eaeed 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -64,6 +64,18 @@ {{ form.ignored_patterns.help_text|safe|linebreaksbr }} + +
+ +
+ {{ form.ignored_dependency_scopes }} +
+
+ {{ form.ignored_dependency_scopes.help_text|safe|linebreaksbr }} +
+
diff --git a/scanpipe/tests/data/settings/scancode-config.yml b/scanpipe/tests/data/settings/scancode-config.yml index d9b2ad96cc..48115a469f 100644 --- a/scanpipe/tests/data/settings/scancode-config.yml +++ b/scanpipe/tests/data/settings/scancode-config.yml @@ -1,4 +1,10 @@ +product_name: My Product Name +product_version: '1.0' ignored_patterns: - - "*.img" - - "docs/*" - - "*/tests/*" + - '*.tmp' + - 'tests/*' +ignored_dependency_scopes: + - package_type: npm + scope: devDependencies + - package_type: pypi + scope: tests diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index 4b045e0d91..eef90fd715 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -192,6 +192,30 @@ def test_scanpipe_pipes_update_or_create_dependency(self): dependency = pipes.update_or_create_dependency(p1, dependency_data) self.assertEqual("install", dependency.scope) + def test_scanpipe_pipes_update_or_create_dependency_ignored_dependency_scopes(self): + p1 = Project.objects.create(name="Analysis") + make_resource_file(p1, "daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO") + pipes.update_or_create_package(p1, package_data1) + + p1.settings = { + "ignored_dependency_scopes": [{"package_type": "pypi", "scope": "tests"}] + } + p1.save() + + dependency_data = dict(dependency_data1) + self.assertFalse(pipes.ignore_dependency_scope(p1, dependency_data)) + dependency = pipes.update_or_create_dependency(p1, dependency_data) + for field_name, value in dependency_data.items(): + self.assertEqual(value, getattr(dependency, field_name), msg=field_name) + dependency.delete() + + # Matching the ignored setting + dependency_data["package_type"] = "pypi" + dependency_data["scope"] = "tests" + self.assertTrue(pipes.ignore_dependency_scope(p1, dependency_data)) + dependency = pipes.update_or_create_dependency(p1, dependency_data) + self.assertIsNone(dependency) + def test_scanpipe_pipes_get_or_create_relation(self): p1 = Project.objects.create(name="Analysis") from1 = make_resource_file(p1, "from/a.txt") diff --git a/scanpipe/tests/test_forms.py b/scanpipe/tests/test_forms.py index 3c33bcf2fa..eb6d59dabe 100644 --- a/scanpipe/tests/test_forms.py +++ b/scanpipe/tests/test_forms.py @@ -133,6 +133,7 @@ def test_scanpipe_forms_project_settings_form_update_project_settings(self): expected = { "ignored_patterns": ["*.ext", "dir/*"], + "ignored_dependency_scopes": None, "product_name": "", "product_version": "", "attribution_template": "", @@ -143,6 +144,60 @@ def test_scanpipe_forms_project_settings_form_update_project_settings(self): } self.assertEqual(expected, project.get_env()) + def test_scanpipe_forms_project_settings_form_ignored_dependency_scopes(self): + data = { + "name": self.project1.name, + "ignored_dependency_scopes": "", + } + form = ProjectSettingsForm(data=data, instance=self.project1) + self.assertTrue(form.is_valid()) + + data["ignored_dependency_scopes"] = "bad" + form = ProjectSettingsForm(data=data, instance=self.project1) + self.assertFalse(form.is_valid()) + expected = { + "ignored_dependency_scopes": [ + "Invalid input line: 'bad'. Each line must contain exactly one ':' " + "character." + ] + } + self.assertEqual(expected, form.errors) + + data["ignored_dependency_scopes"] = "npm:" + form = ProjectSettingsForm(data=data, instance=self.project1) + self.assertFalse(form.is_valid()) + + expected = { + "ignored_dependency_scopes": [ + "Invalid input line: 'npm:'. Both key and value must be non-empty." + ] + } + self.assertEqual(expected, form.errors) + + data["ignored_dependency_scopes"] = "npm:devDependencies\npypi:tests" + form = ProjectSettingsForm(data=data, instance=self.project1) + self.assertTrue(form.is_valid()) + + project = form.save() + expected = { + "ignored_patterns": None, + "ignored_dependency_scopes": [ + {"package_type": "npm", "scope": "devDependencies"}, + {"package_type": "pypi", "scope": "tests"}, + ], + "attribution_template": "", + "product_name": "", + "product_version": "", + } + self.assertEqual(expected, project.settings) + expected = { + "ignored_dependency_scopes": [ + {"package_type": "npm", "scope": "devDependencies"}, + {"package_type": "pypi", "scope": "tests"}, + ] + } + self.assertEqual(expected, project.get_env()) + def test_scanpipe_forms_edit_input_source_tag_form(self): data = {} form = EditInputSourceTagForm(data=data) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index fbba10f3ab..16e4b95dbd 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -668,16 +668,26 @@ def test_scanpipe_project_get_env(self): copy_input(test_config_file, self.project1.input_path) expected = { - "ignored_patterns": ["*.img", "docs/*", "*/tests/*"], + "product_name": "My Product Name", + "product_version": "1.0", + "ignored_patterns": ["*.tmp", "tests/*"], + "ignored_dependency_scopes": [ + {"package_type": "npm", "scope": "devDependencies"}, + {"package_type": "pypi", "scope": "tests"}, + ], } self.assertEqual(expected, self.project1.get_env()) config = {"ignored_patterns": None} self.project1.settings = config self.project1.save() - expected = { - "ignored_patterns": ["*.img", "docs/*", "*/tests/*"], - } + self.assertEqual(expected, self.project1.get_env()) + + config = {"ignored_patterns": ["*.txt"], "product_name": "Product1"} + self.project1.settings = config + self.project1.save() + expected["product_name"] = "Product1" + expected["ignored_patterns"] = ["*.txt"] self.assertEqual(expected, self.project1.get_env()) def test_scanpipe_project_get_env_invalid_yml_content(self): @@ -692,6 +702,27 @@ def test_scanpipe_project_get_env_invalid_yml_content(self): self.assertIn("Failed to load configuration from", error.description) self.assertIn("The file format is invalid.", error.description) + def test_scanpipe_project_get_ignored_dependency_scopes_index(self): + self.project1.settings = { + "ignored_dependency_scopes": [{"package_type": "pypi", "scope": "tests"}] + } + expected = {"pypi": ["tests"]} + self.assertEqual(expected, self.project1.ignored_dependency_scopes_index) + self.assertEqual(expected, self.project1.get_ignored_dependency_scopes_index()) + + self.project1.settings = { + "ignored_dependency_scopes": [ + {"package_type": "pypi", "scope": "tests"}, + {"package_type": "pypi", "scope": "build"}, + {"package_type": "npm", "scope": "devDependencies"}, + ] + } + # Since this is a cache property, it still returns the previous value + self.assertEqual(expected, self.project1.ignored_dependency_scopes_index) + # The following function call always build and return the index + expected = {"npm": ["devDependencies"], "pypi": ["tests", "build"]} + self.assertEqual(expected, self.project1.get_ignored_dependency_scopes_index()) + def test_scanpipe_project_model_labels(self): self.project1.labels.add("label1", "label2") self.assertEqual(2, UUIDTaggedItem.objects.count())