Merge branch 'main' into resolve-dependencies-from-lockfile

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

.gitignore

@@ -47,7 +47,6 @@ local
 /.python-version
 /.pytest_cache/
 /scancodeio.egg-info/
-policies.yml
 *.rdb
 *.aof
 .vscode

CHANGELOG.rst

@@ -1,7 +1,21 @@
 Changelog
 =========
 
-v34.6.3 (unreleased)
+v34.6.4 (unreleased)
+--------------------
+
+- Add all "classify" plugin fields from scancode-toolkit on the CodebaseResource model.
+  https://github.com/nexB/scancode.io/issues/1275
+
+- Refine the extraction errors reporting to include the resource path for rendering
+  link to the related resources in the UI.
+  https://github.com/nexB/scancode.io/issues/1273
+
+- Add a ``flush-projects`` management command, to Delete all project data and their
+  related work directories created more than a specified number of days ago.
+  https://github.com/nexB/scancode.io/issues/1289
+
+v34.6.3 (2024-06-21)
 --------------------
 
 - Use the ``--option=value`` syntax for args entries in place of ``--option value``
@@ -11,6 +25,9 @@ v34.6.3 (unreleased)
 - Fix an issue in the d2d JavaScript mapper.
   https://github.com/nexB/scancode.io/pull/1274
 
+- Add support for a ``ignored_vulnerabilities`` field on the Project configuration.
+  https://github.com/nexB/scancode.io/issues/1271
+
 v34.6.2 (2024-06-18)
 --------------------
 

docs/command-line-interface.rst

@@ -298,6 +298,27 @@ Optional arguments:
 - ``--no-input`` Does not prompt the user for input of any kind.
 
 
+.. _cli_flush_projects:
+
+`$ scanpipe flush-projects`
+---------------------------
+
+Delete all project data and their related work directories created more than a
+specified number of days ago.
+
+Optional arguments:
+
+- ``---retain-days RETAIN_DAYS`` Specify the number of days to retain data.
+  All data older than this number of days will be deleted.
+  **Defaults to 0 (delete all data)**.
+
+  For example, to delete all projects created more than one week ago::
+
+    scanpipe flush-projects --retain-days 7
+
+- ``--no-input`` Does not prompt the user for input of any kind.
+
+
 .. _cli_create_user:
 
 `$ scanpipe create-user <username>`

docs/faq.rst

@@ -249,3 +249,32 @@ Note that only the HTTPS type of URL is supported::
 A GitHub repository URL example::
 
     https://github.com/username/repository.git
+
+How can I cleanup my ScanCode.io installation, removing all projects and related data?
+--------------------------------------------------------------------------------------
+
+You can use the :ref:`cli_flush_projects` command to perform bulk deletion of projects
+and their associated data stored on disk::
+
+    $ scanpipe flush-projects
+
+**Confirmation will be required before deletion.**
+
+To automate this process, such as running it from a cron job, you can use the
+``--no-input`` option to skip confirmation prompts.
+
+Additionally, you can retain specific projects and their data based on their
+creation date using the ``--retain-days`` option.
+
+Here's an example of a crontab entry that runs daily and flushes all projects and
+data older than 7 days::
+
+    @daily scanpipe flush-projects --retain-days 7 --no-input
+
+.. note:: If you are use Docker for running ScanCode.io, you can run the scanpipe
+  ``flush-projects`` command using::
+
+    docker compose run --rm web scanpipe flush-projects
+
+  See :ref:`command_line_interface` chapter for more information about the scanpipe
+  command.

docs/project-configuration.rst

@@ -52,16 +52,20 @@ Content of a ``scancode-config.yml`` file:
     product_name: My Product Name
     product_version: '1.0'
     ignored_patterns:
-      - '*.tmp'
-      - 'tests/*'
+     - '*.tmp'
+     - 'tests/*'
     ignored_dependency_scopes:
      - package_type: npm
        scope: devDependencies
      - package_type: pypi
        scope: tests
+    ignored_vulnerabilities:
+     - VCID-q4q6-yfng-aaag
+     - CVE-2024-27351
+     - GHSA-vm8q-m57g-pff3
 
-See the :ref:`project_configuration_settings` section for the details about each
-setting.
+See the following :ref:`project_configuration_settings` section for the details about
+each setting.
 
 .. tip::
     You can generate the project configuration file from the
@@ -125,10 +129,10 @@ packages, define the following in your ``scancode-config.yml`` configuration fil
 .. code-block:: yaml
 
     ignored_dependency_scopes:
-      - package_type: npm
-        scope: devDependencies
-      - package_type: pypi
-        scope: tests
+     - package_type: npm
+       scope: devDependencies
+     - package_type: pypi
+       scope: tests
 
 If you prefer to use the :ref:`user_interface_project_settings` form, list each
 ignored scope using the `package_type:scope` syntax, **one per line**, such as:
@@ -141,3 +145,21 @@ ignored scope using the `package_type:scope` syntax, **one per line**, such as:
 .. warning::
     Be precise when listing scope names to avoid unintended exclusions.
     Ensure the scope names are correct and reflect your project requirements.
+
+ignored_vulnerabilities
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Provide one or more vulnerability id to be ignored, **one per line**.
+
+You can provide ``VCID`` from VulnerableCode or any aliases such as ``CVE`` or
+``GHSA``.
+
+.. code-block:: yaml
+
+    ignored_vulnerabilities:
+     - VCID-q4q6-yfng-aaag
+     - CVE-2024-27351
+     - GHSA-vm8q-m57g-pff3
+     - OSV-2020-871
+     - BIT-django-2024-24680
+     - PYSEC-2024-28

docs/scancode-config.yml

@@ -9,12 +9,18 @@
 # - Use ``*`` to match multiple characters.
 # - Use ``?`` to match a single character.
 #ignored_patterns:
-#  - '*.tmp'
-#  - 'tests/*'
+# - '*.tmp'
+# - 'tests/*'
 
 # Specify certain dependency scopes to be ignored for a given package type.
 #ignored_dependency_scopes:
 # - package_type: npm
 #   scope: devDependencies
 # - package_type: pypi
 #   scope: tests
+
+# Specify certain vulnerabilities to be ignored using VCID, CVE, or any aliases.
+#ignored_vulnerabilities:
+# - VCID-q4q6-yfng-aaag
+# - CVE-2024-27351
+# - GHSA-vm8q-m57g-pff3

etc/thirdparty/virtualenv.pyz

@@ -1,7 +1,7 @@
 about_resource: virtualenv.pyz
 name: get-virtualenv
-version: 20.25.1
-download_url: https://github.com/pypa/get-virtualenv/raw/20.25.1/public/virtualenv.pyz
+version: 20.26.3
+download_url: https://github.com/pypa/get-virtualenv/raw/20.26.3/public/virtualenv.pyz
 description: virtualenv is a tool to create isolated Python environments.
 homepage_url: https://github.com/pypa/virtualenv
 license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new
@@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others
 redistribute: yes
 attribute: yes
 track_changes: yes
-package_url: pkg:github/pypa/get-virtualenv@20.25.1#public/virtualenv.pyz
+package_url: pkg:github/pypa/get-virtualenv@20.26.3#public/virtualenv.pyz

etc/thirdparty/virtualenv.pyz.ABOUT

@@ -28,7 +28,7 @@
 
 import git
 
-VERSION = "34.6.2"
+VERSION = "34.6.3"
 
 PROJECT_DIR = Path(__file__).resolve().parent
 ROOT_DIR = PROJECT_DIR.parent

scancodeio/__init__.py

@@ -315,6 +315,10 @@ class Meta:
             "is_text",
             "is_archive",
             "is_media",
+            "is_legal",
+            "is_manifest",
+            "is_readme",
+            "is_top_level",
             "is_key_file",
             "detected_license_expression",
             "detected_license_expression_spdx",