Add support for ignored_dependency_scopes field for configuration (#1235)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -60,6 +60,9 @@ v34.5.0 (unreleased)
 - Remove the ``extract_recursively`` option from the Project configuration.
   https://github.com/nexB/scancode.io/issues/1236
 
+- Add support for a ``ignored_dependency_scopes`` field on the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1197
+
 - Add support for storing the scancode-config.yml file in codebase.
   The scancode-config.yml file can be provided as a project input, or can be located
   in the codebase/ immediate subdirectories. This allows to provide the configuration

docs/conf.py

@@ -67,6 +67,9 @@
 # a list of builtin themes.
 html_theme = "sphinx_rtd_theme"
 
+# The style name to use for Pygments highlighting of source code.
+pygments_style = "emacs"
+
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".

docs/project-configuration.rst

@@ -40,7 +40,12 @@ Content of a ``scancode-config.yml`` file:
     product_version: '1.0'
     ignored_patterns:
       - '*.tmp'
-      - tests/*
+      - 'tests/*'
+    ignored_dependency_scopes:
+     - package_type: npm
+       scope: devDependencies
+     - package_type: pypi
+       scope: tests
 
 See the :ref:`project_configuration_settings` section for the details about each
 setting.
@@ -49,7 +54,6 @@ setting.
     You can generate the project configuration file from the
     :ref:`user_interface_project_settings` UI.
 
-
 .. _project_configuration_settings:
 
 Settings
@@ -86,3 +90,41 @@ within the project.
 
 .. warning::
     Be cautious when specifying patterns to avoid unintended exclusions.
+
+ignored_dependency_scopes
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Specify certain dependency scopes to be ignored for a given package type.
+This allows you to exclude dependencies from being created or resolved based on their
+scope.
+
+**Guidelines:**
+
+- **Exact Matches Only:** The scope names must be specified exactly as they appear.
+  Wildcards and partial matches are not supported.
+- **Scope Specification:** List each scope name you wish to ignore.
+
+**Examples:**
+
+To exclude all ``devDependencies`` for ``npm`` packages and ``tests`` for ``pypi``
+packages, define the following in your ``scancode-config.yml`` configuration file:
+
+.. code-block:: yaml
+
+    ignored_dependency_scopes:
+      - package_type: npm
+        scope: devDependencies
+      - package_type: pypi
+        scope: tests
+
+If you prefer to use the :ref:`user_interface_project_settings` form, list each
+ignored scope using the `package_type:scope` syntax, **one per line**, such as:
+
+.. code-block:: text
+
+    npm:devDependencies
+    pypi:tests
+
+.. warning::
+    Be precise when listing scope names to avoid unintended exclusions.
+    Ensure the scope names are correct and reflect your project requirements.

scanpipe/forms.py

@@ -276,7 +276,68 @@ def prepare_value(self, value):
         return value
 
 
-ignored_patterns_help_markdown = """
+class KeyValueListField(forms.CharField):
+    """
+    A Django form field that displays as a textarea and converts each line of
+    "key:value" input into a list of dictionaries with customizable keys.
+
+    Each line of the textarea input is split into key-value pairs,
+    removing leading/trailing whitespace and empty lines. The resulting list of
+    dictionaries is then stored as the field value.
+    """
+
+    widget = forms.Textarea
+
+    def __init__(self, *args, key_name="key", value_name="value", **kwargs):
+        """Initialize the KeyValueListField with custom key and value names."""
+        self.key_name = key_name
+        self.value_name = value_name
+        super().__init__(*args, **kwargs)
+
+    def to_python(self, value):
+        """
+        Split the textarea input into lines, convert each line to a dictionary,
+        and remove empty lines.
+        """
+        if not value:
+            return None
+
+        items = []
+        for line in value.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(":", 1)
+            if len(parts) != 2:
+                raise ValidationError(
+                    f"Invalid input line: '{line}'. "
+                    f"Each line must contain exactly one ':' character."
+                )
+            key, value = parts
+            key = key.strip()
+            value = value.strip()
+            if not key or not value:
+                raise ValidationError(
+                    f"Invalid input line: '{line}'. "
+                    f"Both key and value must be non-empty."
+                )
+            items.append({self.key_name: key, self.value_name: value})
+
+        return items
+
+    def prepare_value(self, value):
+        """
+        Join the list of dictionaries into a string with newlines,
+        using the "key:value" format.
+        """
+        if value is not None and isinstance(value, list):
+            value = "\n".join(
+                f"{item[self.key_name]}:{item[self.value_name]}" for item in value
+            )
+        return value
+
+
+ignored_patterns_help = """
 Provide one or more path patterns to be ignored, one per line.
 
 Each pattern should follow the syntax of Unix shell-style wildcards:
@@ -295,18 +356,27 @@ def prepare_value(self, value):
 Be cautious when specifying patterns to avoid unintended exclusions.
 """
 
+ignored_dependency_scopes_help = """
+Specify certain dependency scopes to be ignored for a given package type.
+
+This allows you to exclude dependencies from being created or resolved based on their
+scope using the `package_type:scope` syntax, **one per line**.
+For example: `npm:devDependencies`
+"""
+
 
 class ProjectSettingsForm(forms.ModelForm):
     settings_fields = [
         "ignored_patterns",
+        "ignored_dependency_scopes",
         "attribution_template",
         "product_name",
         "product_version",
     ]
     ignored_patterns = ListTextarea(
         label="Ignored patterns",
         required=False,
-        help_text=convert_markdown_to_html(ignored_patterns_help_markdown.strip()),
+        help_text=convert_markdown_to_html(ignored_patterns_help.strip()),
         widget=forms.Textarea(
             attrs={
                 "class": "textarea is-dynamic",
@@ -315,6 +385,20 @@ class ProjectSettingsForm(forms.ModelForm):
             },
         ),
     )
+    ignored_dependency_scopes = KeyValueListField(
+        label="Ignored dependency scopes",
+        required=False,
+        help_text=convert_markdown_to_html(ignored_dependency_scopes_help.strip()),
+        widget=forms.Textarea(
+            attrs={
+                "class": "textarea is-dynamic",
+                "rows": 2,
+                "placeholder": "npm:devDependencies\npypi:tests",
+            },
+        ),
+        key_name="package_type",
+        value_name="scope",
+    )
     attribution_template = forms.CharField(
         label="Attribution template",
         required=False,

scanpipe/models.py

@@ -27,6 +27,7 @@
 import shutil
 import uuid
 from collections import Counter
+from collections import defaultdict
 from contextlib import suppress
 from itertools import groupby
 from operator import itemgetter
@@ -759,7 +760,7 @@ def get_input_config_file(self):
         Priority order:
         1. If a config file exists directly in the input/ directory, return it.
         2. If exactly one config file exists in a codebase/ immediate subdirectory,
-           return it.
+        return it.
         3. If multiple config files are found in subdirectories, report an error.
         """
         config_filename = settings.SCANCODEIO_CONFIG_FILE
@@ -830,6 +831,29 @@ def get_env(self, field_name=None):
 
         return env
 
+    def get_ignored_dependency_scopes_index(self):
+        """
+        Return a dictionary index of the ``ignored_dependency_scopes`` setting values
+        defined in this Project env.
+        """
+        ignored_dependency_scopes = self.get_env(field_name="ignored_dependency_scopes")
+        if not ignored_dependency_scopes:
+            return {}
+
+        ignored_scope_index = defaultdict(list)
+        for entry in ignored_dependency_scopes:
+            ignored_scope_index[entry.get("package_type")].append(entry.get("scope"))
+
+        return dict(ignored_scope_index)
+
+    @cached_property
+    def ignored_dependency_scopes_index(self):
+        """
+        Return the computed value of get_ignored_dependency_scopes_index.
+        The value is only generated once and cached for further calls.
+        """
+        return self.get_ignored_dependency_scopes_index()
+
     def clear_tmp_directory(self):
         """
         Delete the whole content of the tmp/ directory.

scanpipe/pipes/__init__.py

@@ -216,6 +216,25 @@ def create_local_files_package(project, defaults, codebase_resources=None):
     return update_or_create_package(project, package_data, codebase_resources)
 
 
+def ignore_dependency_scope(project, dependency_data):
+    """
+    Return True if the dependency should be ignored, i.e.: not created.
+    The ignored scopes are defined on the project ``ignored_dependency_scopes`` setting
+    field.
+    """
+    ignored_scope_index = project.ignored_dependency_scopes_index
+    if not ignored_scope_index:
+        return False
+
+    dependency_package_type = dependency_data.get("package_type")
+    dependency_scope = dependency_data.get("scope")
+    if dependency_package_type and dependency_scope:
+        if dependency_scope in ignored_scope_index.get(dependency_package_type, []):
+            return True  # Ignore this dependency entry.
+
+    return False
+
+
 def update_or_create_dependency(
     project,
     dependency_data,
@@ -239,6 +258,9 @@ def update_or_create_dependency(
     dependency = None
     dependency_uid = dependency_data.get("dependency_uid")
 
+    if ignore_dependency_scope(project, dependency_data):
+        return  # Do not create the DiscoveredDependency record.
+
     if not dependency_uid:
         dependency_data["dependency_uid"] = uuid.uuid4()
     else:

scanpipe/templates/scanpipe/project_settings.html

@@ -64,6 +64,18 @@
                     {{ form.ignored_patterns.help_text|safe|linebreaksbr }}
                   </div>
                 </div>
+
+                <div class="field">
+                  <label class="label" for="{{ form.ignored_dependency_scopes.id_for_label }}">
+                    {{ form.ignored_dependency_scopes.label }}
+                  </label>
+                  <div class="control">
+                    {{ form.ignored_dependency_scopes }}
+                  </div>
+                  <div class="help">
+                    {{ form.ignored_dependency_scopes.help_text|safe|linebreaksbr }}
+                  </div>
+                </div>
               </div>
             </div>
 

scanpipe/tests/data/settings/scancode-config.yml

@@ -1,4 +1,10 @@
+product_name: My Product Name
+product_version: '1.0'
 ignored_patterns:
-  - "*.img"
-  - "docs/*"
-  - "*/tests/*"
+  - '*.tmp'
+  - 'tests/*'
+ignored_dependency_scopes:
+ - package_type: npm
+   scope: devDependencies
+ - package_type: pypi
+   scope: tests

scanpipe/tests/pipes/test_pipes.py

@@ -192,6 +192,30 @@ def test_scanpipe_pipes_update_or_create_dependency(self):
         dependency = pipes.update_or_create_dependency(p1, dependency_data)
         self.assertEqual("install", dependency.scope)
 
+    def test_scanpipe_pipes_update_or_create_dependency_ignored_dependency_scopes(self):
+        p1 = Project.objects.create(name="Analysis")
+        make_resource_file(p1, "daglib-0.3.2.tar.gz-extract/daglib-0.3.2/PKG-INFO")
+        pipes.update_or_create_package(p1, package_data1)
+
+        p1.settings = {
+            "ignored_dependency_scopes": [{"package_type": "pypi", "scope": "tests"}]
+        }
+        p1.save()
+
+        dependency_data = dict(dependency_data1)
+        self.assertFalse(pipes.ignore_dependency_scope(p1, dependency_data))
+        dependency = pipes.update_or_create_dependency(p1, dependency_data)
+        for field_name, value in dependency_data.items():
+            self.assertEqual(value, getattr(dependency, field_name), msg=field_name)
+        dependency.delete()
+
+        # Matching the ignored setting
+        dependency_data["package_type"] = "pypi"
+        dependency_data["scope"] = "tests"
+        self.assertTrue(pipes.ignore_dependency_scope(p1, dependency_data))
+        dependency = pipes.update_or_create_dependency(p1, dependency_data)
+        self.assertIsNone(dependency)
+
     def test_scanpipe_pipes_get_or_create_relation(self):
         p1 = Project.objects.create(name="Analysis")
         from1 = make_resource_file(p1, "from/a.txt")