Add support for scancode-config.yml in codebase #1236 (#1243)

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -57,6 +57,15 @@ v34.5.0 (unreleased)
 - Remove the ``scancode_license_score`` option from the Project configuration.
   https://github.com/nexB/scancode.io/issues/1231
 
+- Remove the ``extract_recursively`` option from the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1236
+
+- Add support for storing the scancode-config.yml file in codebase.
+  The scancode-config.yml file can be provided as a project input, or can be located
+  in the codebase/ immediate subdirectories. This allows to provide the configuration
+  file as part of an input archive or a git clone for example.
+  https://github.com/nexB/scancode.io/issues/1236
+
 - Add support for CycloneDX SBOM component properties as generated by external tools.
   For example, the ``ResolvedUrl`` generated by cdxgen is now imported as the package
   ``download_url``.

docs/custom-pipelines.rst

@@ -124,9 +124,6 @@ the file's directory in the :ref:`scancodeio_settings_pipelines_dirs`.
                 cls.report_licenses_with_resources,
             )
 
-        # Set to True to extract recursively nested archives in archives.
-        extract_recursively = False
-
         # See https://jinja.palletsprojects.com/en/3.0.x/templates/ for documentation
         report_template = """
         {% for matched_text, paths in resources.items() -%}

docs/project-configuration.rst

@@ -38,7 +38,6 @@ Content of a ``scancode-config.yml`` file:
 
     product_name: My Product Name
     product_version: '1.0'
-    extract_recursively: yes
     ignored_patterns:
       - '*.tmp'
       - tests/*
@@ -66,15 +65,6 @@ product_version
 
 The product version of this project, as specified within the DejaCode application.
 
-extract_recursively
-^^^^^^^^^^^^^^^^^^^
-
-Extract nested archives-in-archives recursively.
-**This is enabled by default on new projects.**
-
-You can turn off this behavior in favor of a shallow extraction using
-``extract_recursively: no`` in your configuration file.
-
 ignored_patterns
 ^^^^^^^^^^^^^^^^
 

scanpipe/forms.py

@@ -298,19 +298,11 @@ def prepare_value(self, value):
 
 class ProjectSettingsForm(forms.ModelForm):
     settings_fields = [
-        "extract_recursively",
         "ignored_patterns",
         "attribution_template",
         "product_name",
         "product_version",
     ]
-    extract_recursively = forms.BooleanField(
-        label="Extract recursively",
-        required=False,
-        initial=True,
-        help_text="Extract nested archives-in-archives recursively",
-        widget=forms.CheckboxInput(attrs={"class": "checkbox mr-1"}),
-    )
     ignored_patterns = ListTextarea(
         label="Ignored patterns",
         required=False,

scanpipe/models.py

@@ -753,12 +753,42 @@ def get_codebase_config_directory(self):
 
     def get_input_config_file(self):
         """
-        Return the ``scancode-config.yml`` file from the input/ directory if
-        available.
-        """
-        config_file = self.input_path / settings.SCANCODEIO_CONFIG_FILE
-        if config_file.exists():
-            return config_file
+        Return the ``scancode-config.yml`` file from the input/ directory
+        or from the codebase/ immediate subdirectories.
+
+        Priority order:
+        1. If a config file exists directly in the input/ directory, return it.
+        2. If exactly one config file exists in a codebase/ immediate subdirectory,
+           return it.
+        3. If multiple config files are found in subdirectories, report an error.
+        """
+        config_filename = settings.SCANCODEIO_CONFIG_FILE
+
+        # Check for the config file in the root of the input/ directory.
+        root_config_file = self.input_path / config_filename
+        if root_config_file.exists():
+            return root_config_file
+
+        # Search for config files in immediate codebase/ subdirectories.
+        subdir_config_files = list(self.codebase_path.glob(f"*/{config_filename}"))
+
+        # If exactly one config file is found in codebase/ subdirectories, return it.
+        if len(subdir_config_files) == 1:
+            return subdir_config_files[0]
+
+        # If multiple config files are found, report an error.
+        if len(subdir_config_files) > 1:
+            self.add_warning(
+                f"More than one {config_filename} found. "
+                f"Could not determine which one to use.",
+                model="Project",
+                details={
+                    "resources": [
+                        str(path.relative_to(self.work_path))
+                        for path in subdir_config_files
+                    ]
+                },
+            )
 
     def get_settings_as_yml(self):
         """Return the ``settings`` file content as yml, suitable for a config file."""
@@ -774,17 +804,23 @@ def get_enabled_settings(self):
 
     def get_env(self, field_name=None):
         """
-        Return the project environment loaded from the ``.scancode/config.yml`` config
-        file, when available, and overriden by the ``settings`` model field.
+        Return the project environment loaded from the ``scancode-config.yml`` config
+        file, when available, and overridden by the ``settings`` model field.
 
         ``field_name`` can be provided to get a single entry from the env.
         """
         env = {}
 
         # 1. Load settings from config file when available.
         if config_file := self.get_input_config_file():
-            with suppress(saneyaml.YAMLError):
+            logger.info(f"Loading env from {config_file.relative_to(self.work_path)}")
+            try:
                 env = saneyaml.load(config_file.read_text())
+            except saneyaml.YAMLError:
+                self.add_error(
+                    f'Failed to load configuration from "{config_file}". '
+                    f"The file format is invalid."
+                )
 
         # 2. Update with defined values from the Project ``settings`` field.
         env.update(self.get_enabled_settings())

scanpipe/pipelines/__init__.py

@@ -295,12 +295,16 @@ def extract_archives(self):
 
         extract_errors = scancode.extract_archives(
             location=self.project.codebase_path,
-            recurse=self.env.get("extract_recursively", True),
+            recurse=True,
         )
 
         if extract_errors:
             self.add_error("\n".join(extract_errors))
 
+        # Reload the project env post-extraction as the scancode-config.yml file
+        # may be located in one of the extracted archives.
+        self.env = self.project.get_env()
+
 
 def is_pipeline(obj):
     """

scanpipe/pipelines/deploy_to_develop.py

@@ -135,6 +135,10 @@ def extract_inputs_to_codebase_directory(self):
         if errors:
             self.add_error("\n".join(errors))
 
+        # Reload the project env post-extraction as the scancode-config.yml file
+        # may be located in one of the extracted archives.
+        self.env = self.project.get_env()
+
     def collect_and_create_codebase_resources(self):
         """Collect and create codebase resources."""
         pipes.collect_and_create_codebase_resources(self.project)

scanpipe/pipelines/root_filesystem.py

@@ -63,6 +63,10 @@ def extract_input_files_to_codebase_directory(self):
         if errors:
             self.add_error("\n".join(errors))
 
+        # Reload the project env post-extraction as the scancode-config.yml file
+        # may be located in one of the extracted archives.
+        self.env = self.project.get_env()
+
     def find_root_filesystems(self):
         """Find root filesystems in the project's codebase/."""
         self.root_filesystems = list(rootfs.RootFs.from_project_codebase(self.project))

scanpipe/pipelines/scan_single_package.py

@@ -101,6 +101,10 @@ def extract_input_to_codebase_directory(self):
         if extract_errors:
             self.add_error("\n".join(extract_errors))
 
+        # Reload the project env post-extraction as the scancode-config.yml file
+        # may be located in one of the extracted archives.
+        self.env = self.project.get_env()
+
     def run_scan(self):
         """Scan extracted codebase/ content."""
         scan_output_path = self.project.get_output_file_path("scancode", "json")

scanpipe/templates/scanpipe/includes/project_settings_menu.html

@@ -5,11 +5,6 @@
         <i class="fa-solid fa-gear mr-2"></i>General
       </a>
     </li>
-    <li>
-      <a href="#extraction">
-        <i class="fa-solid fa-file-zipper mr-2"></i>Extraction
-      </a>
-    </li>
     <li>
       <a href="#ignored">
         <i class="fa-solid fa-forward mr-2"></i>Ignored