Follow license reference to another file

src/licensedcode/models.py

@@ -953,6 +953,10 @@ def validate(self, licensing=None):
                     except InvalidRule as e:
                         yield f'Failed to parse and validate license_expression: {e}'
 
+            if self.referenced_filenames:
+                if len(set(self.referenced_filenames)) != len(self.referenced_filenames):
+                    yield 'referenced_filenames cannot contain duplicates.'
+
     def license_keys(self, unique=True):
         """
         Return a list of license keys for this rule.

src/licensedcode/plugin_license.py

@@ -11,6 +11,7 @@
 
 import attr
 
+from commoncode import fileutils
 from commoncode.cliutils import PluggableCommandLineOption
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
@@ -116,3 +117,73 @@ def get_scanner(
             license_text_diagnostics=license_text_diagnostics,
             license_url_template=license_url_template
         )
+
+    def process_codebase(self, codebase, **kwargs):
+
+        if codebase.has_single_resource:
+            return
+
+        for resource in codebase.walk(topdown=False):
+            match_reference_license(resource,codebase)
+
+
+def match_reference_license(resource, codebase):
+    """
+    Return the ``resource`` Resource updating and saving it in place, after adding new
+    license matches (licenses and license_expressions) following their Rule
+    ``referenced_filenames`` if any. Return None if this is not a file Resource.
+    """
+    if not resource.is_file:
+        return
+
+    licenses = resource.licenses
+    license_expressions = resource.license_expressions
+    if not licenses:
+        return 
+
+    referenced_licenses = []
+    referenced_license_expressions = []
+    referenced_filenames = get_referenced_filenames(licenses)
+    modified = False
+
+    for referenced_filename in referenced_filenames:
+        new_resource = find_referenced_resource(referenced_filename=referenced_filename, resource=resource, codebase=codebase)
+        if new_resource:
+            modified = True
+            referenced_licenses.extend(new_resource.licenses)
+            referenced_license_expressions.extend(new_resource.license_expressions)
+
+    licenses.extend(referenced_licenses)
+    license_expressions.extend(referenced_license_expressions)
+
+    if modified:
+        codebase.save_resource(resource)
+    return resource
+
+
+def get_referenced_filenames(license_matches):
+    """
+    Return a list of unique referenced filenames found in the rules of a list of ``license_matches``
+    """
+    referenced_filenames = []
+    for license_match in license_matches:
+        referenced_files = license_match['matched_rule']['referenced_filenames']
+        for referenced_filename in referenced_files:
+            if not referenced_filename in referenced_filenames:
+                referenced_filenames.append(referenced_filename)
+
+    return referenced_filenames
+
+
+def find_referenced_resource(referenced_filename, resource, codebase, **kwargs):
+    """
+    Return a Resource matching the ``referenced_filename`` path or filename given a ``resource`` in ``codebase``.
+    Return None if the ``referenced_filename`` cannot be found in the same directory as the base ``resource``.
+    ``referenced_filename`` is the path or filename referenced in a LicenseMatch of ``resource``,
+    """
+    parent = resource.parent(codebase)
+
+    for child in parent.children(codebase):
+        path = child.path
+        if path.endswith(referenced_filename) or fileutils.file_base_name(child.path) == referenced_filename:
+            return child

tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json

@@ -0,0 +1,165 @@
+{
+  "headers": [
+    {
+      "tool_name": "scancode-toolkit",
+      "options": {
+        "input": "<path>",
+        "--json": "<file>",
+        "--license": true,
+        "--license-text": true,
+        "--license-text-diagnostics": true,
+        "--strip-root": true
+      },
+      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
+      "message": null,
+      "errors": [],
+      "extra_data": {
+        "files_count": 2
+      }
+    }
+  ],
+  "files": [
+    {
+      "path": "LICENSE",
+      "type": "file",
+      "licenses": [
+        {
+          "key": "mit",
+          "score": 100.0,
+          "name": "MIT License",
+          "short_name": "MIT License",
+          "category": "Permissive",
+          "is_exception": false,
+          "is_unknown": false,
+          "owner": "MIT",
+          "homepage_url": "http://opensource.org/licenses/mit-license.php",
+          "text_url": "http://opensource.org/licenses/mit-license.php",
+          "reference_url": "https://scancode-licensedb.aboutcode.org/mit",
+          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE",
+          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.yml",
+          "spdx_license_key": "MIT",
+          "spdx_url": "https://spdx.org/licenses/MIT",
+          "start_line": 1,
+          "end_line": 1,
+          "matched_rule": {
+            "identifier": "mit_66.RULE",
+            "license_expression": "mit",
+            "licenses": [
+              "mit"
+            ],
+            "referenced_filenames": [],
+            "is_license_text": false,
+            "is_license_notice": true,
+            "is_license_reference": false,
+            "is_license_tag": false,
+            "is_license_intro": false,
+            "has_unknown": false,
+            "matcher": "1-hash",
+            "rule_length": 10,
+            "matched_length": 10,
+            "match_coverage": 100.0,
+            "rule_relevance": 100
+          },
+          "matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
+        }
+      ],
+      "license_expressions": [
+        "mit"
+      ],
+      "percentage_of_license_text": 100.0,
+      "scan_errors": []
+    },
+    {
+      "path": "license-notice.txt",
+      "type": "file",
+      "licenses": [
+        {
+          "key": "unknown-license-reference",
+          "score": 27.0,
+          "name": "Unknown License file reference",
+          "short_name": "Unknown License reference",
+          "category": "Unstated License",
+          "is_exception": false,
+          "is_unknown": true,
+          "owner": "Unspecified",
+          "homepage_url": null,
+          "text_url": "",
+          "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference",
+          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
+          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml",
+          "spdx_license_key": "LicenseRef-scancode-unknown-license-reference",
+          "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
+          "start_line": 34,
+          "end_line": 34,
+          "matched_rule": {
+            "identifier": "unknown-license-reference_25.RULE",
+            "license_expression": "unknown-license-reference",
+            "licenses": [
+              "unknown-license-reference"
+            ],
+            "referenced_filenames": [
+              "LICENSE"
+            ],
+            "is_license_text": false,
+            "is_license_notice": false,
+            "is_license_reference": false,
+            "is_license_tag": true,
+            "is_license_intro": false,
+            "has_unknown": true,
+            "matcher": "2-aho",
+            "rule_length": 5,
+            "matched_length": 5,
+            "match_coverage": 100.0,
+            "rule_relevance": 27
+          },
+          "matched_text": "license\": \"SEE LICENSE IN LICENSE."
+        },
+        {
+          "key": "mit",
+          "score": 100.0,
+          "name": "MIT License",
+          "short_name": "MIT License",
+          "category": "Permissive",
+          "is_exception": false,
+          "is_unknown": false,
+          "owner": "MIT",
+          "homepage_url": "http://opensource.org/licenses/mit-license.php",
+          "text_url": "http://opensource.org/licenses/mit-license.php",
+          "reference_url": "https://scancode-licensedb.aboutcode.org/mit",
+          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE",
+          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.yml",
+          "spdx_license_key": "MIT",
+          "spdx_url": "https://spdx.org/licenses/MIT",
+          "start_line": 1,
+          "end_line": 1,
+          "matched_rule": {
+            "identifier": "mit_66.RULE",
+            "license_expression": "mit",
+            "licenses": [
+              "mit"
+            ],
+            "referenced_filenames": [],
+            "is_license_text": false,
+            "is_license_notice": true,
+            "is_license_reference": false,
+            "is_license_tag": false,
+            "is_license_intro": false,
+            "has_unknown": false,
+            "matcher": "1-hash",
+            "rule_length": 10,
+            "matched_length": 10,
+            "match_coverage": 100.0,
+            "rule_relevance": 100
+          },
+          "matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
+        }
+      ],
+      "license_expressions": [
+        "unknown-license-reference",
+        "mit"
+      ],
+      "percentage_of_license_text": 0.2,
+      "scan_errors": []
+    }
+  ]
+}

tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json

@@ -0,0 +1,75 @@
+{
+  "headers": [
+    {
+      "tool_name": "scancode-toolkit",
+      "options": {
+        "input": "<path>",
+        "--json": "<file>",
+        "--license": true,
+        "--license-text": true,
+        "--license-text-diagnostics": true,
+        "--strip-root": true
+      },
+      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
+      "message": null,
+      "errors": [],
+      "extra_data": {
+        "files_count": 1
+      }
+    }
+  ],
+  "files": [
+    {
+      "path": "license-notice.txt",
+      "type": "file",
+      "licenses": [
+        {
+          "key": "unknown-license-reference",
+          "score": 27.0,
+          "name": "Unknown License file reference",
+          "short_name": "Unknown License reference",
+          "category": "Unstated License",
+          "is_exception": false,
+          "is_unknown": true,
+          "owner": "Unspecified",
+          "homepage_url": null,
+          "text_url": "",
+          "reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference",
+          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
+          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml",
+          "spdx_license_key": "LicenseRef-scancode-unknown-license-reference",
+          "spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
+          "start_line": 34,
+          "end_line": 34,
+          "matched_rule": {
+            "identifier": "unknown-license-reference_25.RULE",
+            "license_expression": "unknown-license-reference",
+            "licenses": [
+              "unknown-license-reference"
+            ],
+            "referenced_filenames": [
+              "LICENSE"
+            ],
+            "is_license_text": false,
+            "is_license_notice": false,
+            "is_license_reference": false,
+            "is_license_tag": true,
+            "is_license_intro": false,
+            "has_unknown": true,
+            "matcher": "2-aho",
+            "rule_length": 5,
+            "matched_length": 5,
+            "match_coverage": 100.0,
+            "rule_relevance": 27
+          },
+          "matched_text": "license\": \"SEE LICENSE IN LICENSE."
+        }
+      ],
+      "license_expressions": [
+        "unknown-license-reference"
+      ],
+      "percentage_of_license_text": 0.2,
+      "scan_errors": []
+    }
+  ]
+}

tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref/LICENSE

@@ -0,0 +1 @@
+that is licensed under [MIT](http://opensource.org/licenses/MIT).