Add support for external licenses in scans

CHANGELOG.rst

@@ -46,14 +46,17 @@ License detection:
   matches in a larger license detecion. This remove a larger number of false
   positive or ambiguous license detections.
 
-
 - The data structure of the JSON output has changed for licenses. We now
   return match details once for each matched license expression rather than
   once for each license in a matched expression. There is a new top-level
   "license_references" attribute that contains the data details for each
   detected license only once. This data can contain the reference license text
   as an option.
 
+- We can now detect licenses using custom license texts and license rules.
+  These can be provided as a one off in a directory or packaged as a plugin
+  for consistent reuse and deployment.
+
 v31.2.1 - 2022-10-05
 ----------------------------------
 

azure-pipelines.yml

@@ -32,8 +32,7 @@ jobs:
                   --ignore=tests/licensedcode/test_detection_datadriven1.py \
                   --ignore=tests/licensedcode/test_detection_datadriven2.py \
                   --ignore=tests/licensedcode/test_detection_datadriven3.py \
-                  --ignore=tests/licensedcode/test_detection_datadriven4.py \
-                  tests/licensedcode
+                  --ignore=tests/licensedcode/test_detection_datadriven4.py
 
               license_datadriven1_2: |
                 venv/bin/pytest -n 3 -vvs --test-suite=all \
@@ -78,6 +77,18 @@ jobs:
                 venv/bin/pytest -n 3 -vvs --test-suite=all \
                   tests/licensedcode/test_zzzz_cache.py
 
+              # this test runs in isolation because it modifies the actual
+              # license index with additional licenses provided by a plugin
+              # and we use the special --test-suite=plugins marker for these
+              # tests
+              additional_license_combined: |
+                venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_1/
+                venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_2/
+                venv/bin/scancode-reindex-licenses \
+                  --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/
+                venv/bin/pytest -vvs --test-suite=plugins \
+                  tests/licensedcode/test_additional_license.py
+
     - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython

conftest.py

@@ -38,6 +38,7 @@
 ################################################################################
 SLOW_TEST = 'scanslow'
 VALIDATION_TEST = 'scanvalidate'
+PLUGINS_TEST = 'scanplugins'
 
 
 def pytest_configure(config):
@@ -53,8 +54,14 @@ def pytest_configure(config):
         ': Mark a ScanCode test as a validation test, super slow, long running test.',
     )
 
+    config.addinivalue_line(
+        'markers',
+        PLUGINS_TEST +
+        ': Mark a ScanCode test as a special CI test to tests installing additional plugins.',
+    )
+
 
-TEST_SUITES = 'standard', 'all', 'validate'
+TEST_SUITES = ('standard', 'all', 'validate', 'plugins',)
 
 
 def pytest_addoption(parser):
@@ -72,9 +79,11 @@ def pytest_addoption(parser):
         help='Select which test suite to run: '
              '"standard" runs the standard test suite designed to run reasonably fast. '
              '"all" runs "standard" and "slow" (long running) tests. '
-             '"validate" runs all the tests. '
+             '"validate" runs all the tests, except the "plugins" tests. '
+             '"plugins" runs special plugins tests. Needs extra setup, and is used only in the CI. '
              'Use the @pytest.mark.scanslow marker to mark a test as "slow" test. '
              'Use the @pytest.mark.scanvalidate marker to mark a test as a "validate" test.'
+             'Use the @pytest.mark.scanplugins marker to mark a test as a "plugins" test.'
         )
 
 ################################################################################
@@ -87,13 +96,19 @@ def pytest_collection_modifyitems(config, items):
     test_suite = config.getvalue('test_suite')
     run_everything = test_suite == 'validate'
     run_slow_test = test_suite in ('all', 'validate')
+    run_only_plugins = test_suite == 'plugins'
 
     tests_to_run = []
     tests_to_skip = []
 
     for item in items:
         is_validate = bool(item.get_closest_marker(VALIDATION_TEST))
         is_slow = bool(item.get_closest_marker(SLOW_TEST))
+        is_plugins = bool(item.get_closest_marker(PLUGINS_TEST))
+
+        if is_plugins and not run_only_plugins:
+            tests_to_skip.append(item)
+            continue
 
         if is_validate and not run_everything:
             tests_to_skip.append(item)

docs/source/cli-reference/core-options.rst

@@ -83,6 +83,58 @@ Comparing Progress Message Options
     ..
         [ToDo] Research and Write Better
 
+``--additional-license-directory`` Option
+-----------------------------------------
+
+    .. admonition:: Dependency
+
+        The option ``--additional-license-directory`` requires the option ``-reindex--licenses``.
+
+    The ``--additional-license-directory`` option allows the user to include additional directories
+    of licenses to use in license detection.
+
+    This command only needs to be run once for each set of additional directories; in all subsequent
+    runs of Scancode with the same directories all the licenses in the directories will be cached.
+
+    The directory structure should look something like this::
+
+        licenses/
+        ├── privateLicense1/
+        │   ├── license/
+        │   │   ├── privateLicense1.LICENSE
+        │   │   └── privateLicense1.yml
+        │   └── rule/
+        │       ├── privateLicense1.RULE
+        │       └── privateLicense1.yml
+        └── privateLicense2/
+            ├── license/
+            │   ├── privateLicense2.LICENSE
+            │   └── privateLicense2.yml
+            └── rule/
+                ├── privateLicense2.RULE
+                └── privateLicense2.yml
+
+    Here is an example of reindexing the license cache using the ``--additional-license-directory PATH`` option with a single directory.
+    Note that ``--reindex-licenses`` **must** come after ``--additional-license-directory``::
+
+        scancode --additional-license-directory /home/user/external_licenses/license1 --reindex-licenses
+
+    You can also include multiple directories like so::
+
+        scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external2 --reindex-licenses
+
+    If you want to continue running scans with ``/home/user/external_licenses/external1`` and ``/home/user/external_licenses/external2``,
+    you can simply run scans after reindexing with those directories and they will be included. ::
+
+        scancode -clpieu --json-pp output.json samples
+
+    However, if you wanted to run a scan with a new set of directories, such as ``home/user/external_licenses/external1``
+    and ``home/user/external_licenses/external3``, you would need to reindex the license index with those directories as parameters. ::
+
+        scancode --additional-license-directory /home/user/external_licenses/external1 --additional-license-directory /home/user/external_licenses/external3 --reindex-licenses
+
+    ..
+
 ----
 
 ``--from-json`` Option

docs/source/cli-reference/output-format.rst

@@ -183,7 +183,7 @@ following options.
               "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
               "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style",
               "spdx_license_key": null,
-              "spdx_url": "",
+              "spdx_url": null,
               "start_line": 9,
               "end_line": 15,
               "matched_rule": {

docs/source/cli-reference/synopsis.rst

@@ -234,7 +234,7 @@ A sample JSON output for an individual file will look like::
           "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
           "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style",
           "spdx_license_key": null,
-          "spdx_url": "",
+          "spdx_url": null,
           "start_line": 9,
           "end_line": 15,
           "matched_rule": {

docs/source/how-to-guides/index.rst

@@ -8,3 +8,4 @@
 
    add_new_license
    add_new_license_detection_rule
+   install_new_license_plugin

docs/source/how-to-guides/install_new_license_plugin.rst

@@ -0,0 +1,139 @@
+.. _install_new_license_plugin:
+
+How to Install External License Plugins to Use in License Detection
+===================================================================
+
+Users can install external licenses and rules in the form of plugins. These
+licenses and rules are then used in license detection.
+
+How to create a plugin containing external licenses and/or rules
+----------------------------------------------------------------
+
+To create a plugin with external licenses or rules, we must create a Python package
+containing the license and/or rule files. Python packages can have many different
+file structures. You can find an example package in
+``tests/licensedcode/data/example_external_licenses/licenses_to_install1``.
+
+This is the basic structure of the example plugin::
+
+    licenses_to_install1/
+    ├── src/
+    │   └── licenses_to_install1/
+    │       ├── licenses/
+    │       │   ├── example-installed-1.LICENSE
+    │       │   └── example-installed-1.yaml
+    |       ├── rules/
+    │       │   ├── example-installed-1.RULE
+    │       │   └── example-installed-1.yaml
+    │       └── __init__.py
+    ├── gpl-1.0.LICENSE
+    ├── MANIFEST.in
+    ├── setup.cfg
+    └── setup.py
+
+Key points to note
+------------------
+
+Entry points definition in ``setup.py``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+First, in ``setup.py``, you must provide an entry point called ``scancode_location_provider``.
+This allows ScanCode Toolkit to discover the plugin and use it in license detection.
+Here is the definition of ``entry_points`` in ``setup.py``::
+
+    entry_points={
+            'scancode_location_provider': [
+                'licenses_to_install1 = licenses_to_install1:LicensesToInstall1Paths',
+            ],
+        },
+
+The ``scancode_location_provider`` entry point maps to a list with information about the plugin.
+The variable ``licenses_to_install1`` is the name of the entry point. All entry point names
+**must** start with the prefix ``licenses``, or else ScanCode Toolkit will not use them in
+license detection.
+
+Directory structure
+^^^^^^^^^^^^^^^^^^^
+
+``licenses_to_install1`` is set to ``licenses_to_install1:LicensesToInstall1Paths``.
+Note that in ``src``, we have another directory called ``licenses_to_install1`` and in
+``licenses_to_install1/__init__.py``, we define the class ``LicensesToInstall1Paths``.
+These two values make up the entry point definition.
+
+``LicensesToInstall1Paths`` is a subclass of ``LocationProviderPlugin`` and
+implements the method ``get_locations()``. The class you define in ``__init__.py``
+must also subclass ``LocationProviderPlugin`` and implement this method.
+
+Finally, the same directory containing the class definition must also contain the
+licenses and/or rules. Licenses must be contained in a directory called ``licenses`` and rules
+must be contained in a directory called ``rules``.
+
+See :ref:`add_new_license_for_det` and :ref:`add_new_license_det_rule` to understand
+the structure of license and rule files, respectively.
+
+After creating this plugin, you can upload it to PyPI so that others can use it, or you can
+leave it as a local directory.
+
+Installing and using the plugin
+-------------------------------
+To use the plugin in license detection, all you need to do is install it using ``pip``.
+Once it is installed, the contained licenses and rules will automatically be used in
+license detection assuming the plugin follows the correct directory structure conventions.
+
+Writing tests for new installed licenses
+----------------------------------------
+
+Look at ``tests/licensedcode/data/example_external_licenses/licenses_to_install1`` to see
+an example of a plugin with tests. The tests are contained in the ``tests`` directory::
+
+   licenses_to_install1/
+    ├── src/
+    │   └── licenses_to_install1/
+    │       ├── licenses/
+    │       │   ├── example-installed-1.LICENSE
+    │       │   └── example-installed-1.yaml
+    │       ├── rules/
+    │       │   ├── example-installed-1.RULE
+    │       │   └── example-installed-1.yaml
+    │       └── __init__.py/
+    ├── tests/
+    │    ├── data/
+    │    │   ├── example-installed-1.txt
+    │    │   └── example-installed-1.txt.yml
+    │    └── test_detection_datadriven.py
+    ├── gpl-1.0.LICENSE
+    ├── MANIFEST.in
+    ├── setup.cfg
+    └── setup.py
+
+To write your own tests, first make sure ``setup.py`` includes ``scancode-toolkit``
+as a dependency::
+
+    ...
+    install_requires=[
+        'scancode-toolkit',
+    ],
+    ...
+
+Then you can define a test class and call the ``build_tests`` method defined in
+``licensedcode_test_utils``, passing in the test directory and the test class as parameters::
+
+    TEST_DIR = abspath(join(dirname(__file__), 'data'))
+
+
+    class TestLicenseDataDriven1(unittest.TestCase):
+        pass
+
+
+    licensedcode_test_utils.build_tests(
+        TEST_DIR,
+        clazz=TestLicenseDataDriven1, regen=scancode_config.REGEN_TEST_FIXTURES)
+
+The ``tests/data`` directory contains a pair of files for each license:
+a license text file and a YAML file specifying the expected license expressions from the test.
+
+Finally, to run the test, do the following:
+
+1. Create a virtual environment to install the package into.
+2. Install the package using ``pip``, e.g. ``pip install ./licenses_to_install1``.
+3. Run the tests, e.g. ``py.test tests/test_detection_datadriven.py``.

docs/source/rst_snippets/core_options.rst

@@ -14,6 +14,11 @@ All "Core" Scan Options
 --reindex-licenses       Force a check and possible reindexing of the
                          cached license index.
 
+--additional-license-directory PATH
+
+          Include paths to directories containing additional licenses and rules to use
+          in license detection. This can be used multiple times for multiple directories.
+
 --from-json              Load codebase from an existing JSON scan
 
 --max-in-memory INTEGER  Maximum number of files and directories scan

docs/source/rst_snippets/note_snippets/core_indep.rst

@@ -1,3 +1,4 @@
 .. note::
 
-    All the Core Options are independent options, i.e. They don't depend on other options.
+    All the Core Options except for ``--additional-license-directory`` are independent options, i.e.
+    They don't depend on other options.

setup-mini.cfg

@@ -146,6 +146,7 @@ packages =
 [options.entry_points]
 console_scripts =
     scancode = scancode.cli:scancode
+    scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

setup.cfg

@@ -146,6 +146,7 @@ packages =
 [options.entry_points]
 console_scripts =
     scancode = scancode.cli:scancode
+    scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form: