Add is_builtin field to Licenses and Rules and modify url output

Signed-off-by: Kevin Ji <kyji1011@gmail.com>

Author: kji22

src/licensedcode/cache.py

@@ -129,9 +129,12 @@ def load_or_build(
                         pickle.dump(additional_directories, file, protocol=PICKLE_PROTOCOL)
 
                 additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
-                validate_additional_license_data(additional_license_dirs)
+                validate_additional_license_data(additional_directories=additional_license_dirs, scancode_license_dir=licenses_data_dir)
                 combined_directories = [licenses_data_dir] + additional_license_dirs
-                licenses_db = load_licenses_from_multiple_dirs(license_directories=combined_directories)
+                licenses_db = load_licenses_from_multiple_dirs(
+                    license_directories=combined_directories,
+                    scancode_license_dir=licenses_data_dir
+                )
 
                 # create a single merged index containing license data from licenses_data_dir
                 # and data from additional directories
@@ -207,10 +210,14 @@ def build_index(
 
     # if we have additional directories, extract the rules from them
     additional_rule_dirs = get_rule_dirs(additional_dirs=additional_directories)
-    validate_ignorable_clues(additional_rule_dirs)
+    validate_ignorable_clues(rule_directories=additional_rule_dirs, is_builtin=False)
     # then combine the rules in these additional directories with the rules in the original rules directory
     combined_rule_directories = [rules_data_dir] + additional_rule_dirs
-    rules = get_rules_from_multiple_dirs(licenses_db=licenses_db, rule_directories=combined_rule_directories)
+    rules = get_rules_from_multiple_dirs(
+        licenses_db=licenses_db,
+        rule_directories=combined_rule_directories,
+        scancode_rules_dir=rules_data_dir
+    )
 
     legalese = common_license_words
     spdx_tokens = set(get_all_spdx_key_tokens(licenses_db))

src/licensedcode/models.py

@@ -189,6 +189,13 @@ class License:
             help='Free text notes.')
     )
 
+    is_builtin = attr.ib(
+        default=True,
+        repr=False,
+        metadata=dict(
+            help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.')
+    )
+
     # TODO: add the license key(s) this exception applies to
     is_exception = attr.ib(
         default=False,
@@ -349,12 +356,12 @@ class License:
     )
 
     @classmethod
-    def from_dir(cls, key, licenses_data_dir=licenses_data_dir):
+    def from_dir(cls, key, licenses_data_dir=licenses_data_dir, is_builtin=True):
         """
         Return a new License object for a license ``key`` and load its attribute
         from a data file stored in ``licenses_data_dir``.
         """
-        lic = cls(key=key)
+        lic = cls(key=key, is_builtin=is_builtin)
         data_file = lic.data_file(licenses_data_dir=licenses_data_dir)
         if exists(data_file):
             text_file = lic.text_file(licenses_data_dir=licenses_data_dir)
@@ -685,6 +692,7 @@ def load_licenses(
     licenses_data_dir=licenses_data_dir,
     with_deprecated=False,
     check_dangling=True,
+    is_builtin = True,
 ):
     """
     Return a mapping of {key: License} loaded from license data and text files
@@ -712,7 +720,7 @@ def load_licenses(
             key = file_base_name(data_file)
 
             try:
-                lic = License.from_dir(key=key, licenses_data_dir=licenses_data_dir)
+                lic = License.from_dir(key=key, licenses_data_dir=licenses_data_dir, is_builtin=is_builtin)
             except Exception as e:
                 raise Exception(f'Failed to load license: {key} from: file://{licenses_data_dir}/{key}.yml with error: {e}') from e
 
@@ -752,6 +760,7 @@ def get_rules(
     licenses_data_dir=licenses_data_dir,
     rules_data_dir=rules_data_dir,
     validate=False,
+    is_builtin=True,
 ):
     """
     Yield Rule objects loaded from a ``licenses_db`` and license files found in
@@ -764,6 +773,7 @@ def get_rules(
 
     rules = list(load_rules(
         rules_data_dir=rules_data_dir,
+        is_builtin=is_builtin,
     ))
 
     if validate:
@@ -815,6 +825,7 @@ def get_paths_to_installed_licenses_and_rules():
 
 def load_licenses_from_multiple_dirs(
     license_directories,
+    scancode_license_dir,
     with_deprecated=False,
 ):
     """
@@ -824,7 +835,8 @@ def load_licenses_from_multiple_dirs(
     combined_licenses = {}
     combined_licenses_keys = set()
     for license_dir in license_directories:
-        licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False)
+        is_builtin = scancode_license_dir == license_dir
+        licenses = load_licenses(licenses_data_dir=license_dir, with_deprecated=False, is_builtin=is_builtin)
         # check if two dictionaries have duplicate keys
         licenses_keys = set(licenses.keys())
         duplicate_keys = combined_licenses_keys.intersection(licenses_keys)
@@ -842,6 +854,7 @@ def load_licenses_from_multiple_dirs(
 def get_rules_from_multiple_dirs(
     licenses_db,
     rule_directories,
+    scancode_rules_dir,
 ):
     """
     Return a mapping of {key: License} built from a list of ``license_directories``.
@@ -851,8 +864,10 @@ def get_rules_from_multiple_dirs(
     if rule_directories:
         combined_rules = []
         for rules_dir in rule_directories:
+            is_builtin = rules_dir == scancode_rules_dir
             r = list(load_rules(
                 rules_data_dir=rules_dir,
+                is_builtin=is_builtin,
             ))
             combined_rules.append(r)
         # flatten lists of rules into a single iterable
@@ -868,12 +883,12 @@ class InvalidLicense(Exception):
     pass
 
 
-def validate_additional_license_data(additional_directories):
+def validate_additional_license_data(additional_directories, scancode_license_dir):
     """
     Raises an exception if there are any invalid licenses in the directories of
     additional licenses.
     """
-    licenses = load_licenses_from_multiple_dirs(additional_directories)
+    licenses = load_licenses_from_multiple_dirs(additional_directories, scancode_license_dir)
     errors, _, _ = License.validate(
         licenses,
         verbose=False,
@@ -907,7 +922,7 @@ def _ignorable_clue_error(rule):
         return result, expected
 
 
-def validate_ignorable_clues(rule_directories):
+def validate_ignorable_clues(rule_directories, is_builtin):
     """
     Raises an exception if any ignorable clues declared in a Rule are improperly detected
     in the rule text file.
@@ -916,6 +931,7 @@ def validate_ignorable_clues(rule_directories):
     for rules_dir in rule_directories:
         r = list(load_rules(
             rules_data_dir=rules_dir,
+            is_builtin=is_builtin,
         ))
         combined_rules.append(r)
     # flatten lists of rules into a single iterable
@@ -1012,6 +1028,7 @@ def build_rule_from_license(license_obj):
             has_stored_minimum_coverage=bool(minimum_coverage),
             minimum_coverage=minimum_coverage,
 
+            is_builtin=license_obj.is_builtin,
             is_from_license=True,
             is_license_text=True,
 
@@ -1064,7 +1081,7 @@ def get_license_tokens():
     yield 'licensed'
 
 
-def load_rules(rules_data_dir=rules_data_dir, with_checks=True):
+def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True):
     """
     Return an iterable of rules loaded from rule files in ``rules_data_dir``.
     Optionally check for consistency if ``with_checks`` is True.
@@ -1088,7 +1105,7 @@ def load_rules(rules_data_dir=rules_data_dir, with_checks=True):
             text_file = join(rules_data_dir, f'{base_name}.RULE')
 
             try:
-                yield Rule.from_files(data_file=data_file, text_file=text_file)
+                yield Rule.from_files(data_file=data_file, text_file=text_file, is_builtin=is_builtin)
             except Exception as re:
                 if with_checks:
                     model_errors.append(str(re))
@@ -1193,6 +1210,13 @@ class BasicRule:
             'string.')
     )
 
+    is_builtin = attr.ib(
+        default=True,
+        repr=False,
+        metadata=dict(
+            help='Flag set to True if this license is a built in license in the ScanCode LicenseDB.')
+    )
+
     # The is_license_xxx flags below are nn indication of what this rule
     # importance is (e.g. how important is its text when detected as a licensing
     # clue) as one of several "is_license_xxx" flags. These flags are mutually
@@ -1877,12 +1901,12 @@ def __attrs_post_init__(self, *args, **kwargs):
         self.setup()
 
     @classmethod
-    def from_files(cls, data_file, text_file):
+    def from_files(cls, data_file, text_file, is_builtin):
         """
         Return a new Rule object loaded from a data file stored at
         ``data_file`` and a companion ``text_file``.
         """
-        rule = Rule()
+        rule = Rule(is_builtin=is_builtin)
         rule.load_data(data_file=data_file, text_file=text_file)
         return rule
 

src/scancode/api.py

@@ -249,13 +249,19 @@ def _licenses_data_from_match(
         result['is_exception'] = lic.is_exception
         result['is_unknown'] = lic.is_unknown
         result['owner'] = lic.owner
-        result['homepage_url'] = lic.homepage_url
-        result['text_url'] = lic.text_urls[0] if lic.text_urls else ''
-        result['reference_url'] = license_url_template.format(lic.key)
-        # TODO: change this in the case of a private license?
-        result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key)
-        result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key)
-
+        # if the license is not builtin these should all be empty
+        if lic.is_builtin:
+            result['homepage_url'] = lic.homepage_url
+            result['text_url'] = lic.text_urls[0] if lic.text_urls else ''
+            result['reference_url'] = license_url_template.format(lic.key)
+            result['scancode_text_url'] = SCANCODE_LICENSE_TEXT_URL.format(lic.key)
+            result['scancode_data_url'] = SCANCODE_LICENSE_DATA_URL.format(lic.key)
+        else:
+            result['homepage_url'] = ''
+            result['text_url'] = ''
+            result['reference_url'] = ''
+            result['scancode_text_url'] = ''
+            result['scancode_data_url'] = ''
         spdx_key = lic.spdx_license_key
         result['spdx_license_key'] = spdx_key
 

tests/licensedcode/data/plugin_license/external_licenses/scan.expected.json

@@ -51,9 +51,9 @@
           "owner": "NexB",
           "homepage_url": null,
           "text_url": "",
-          "reference_url": "https://scancode-licensedb.aboutcode.org/example1",
-          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE",
-          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml",
+          "reference_url": "",
+          "scancode_text_url": "",
+          "scancode_data_url": "",
           "spdx_license_key": "scancode-example1",
           "spdx_url": "https://spdx.org/licenses/scancode-example1",
           "start_line": 1,

tests/licensedcode/data/plugin_license/external_licenses/scan_multiple.expected.json

@@ -52,9 +52,9 @@
           "owner": "NexB",
           "homepage_url": null,
           "text_url": "",
-          "reference_url": "https://scancode-licensedb.aboutcode.org/example1",
-          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.LICENSE",
-          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example1.yml",
+          "reference_url": "",
+          "scancode_text_url": "",
+          "scancode_data_url": "",
           "spdx_license_key": "scancode-example1",
           "spdx_url": "https://spdx.org/licenses/scancode-example1",
           "start_line": 1,

tests/licensedcode/data/plugin_license/installed_licenses/scan.expected.json

@@ -27,7 +27,6 @@
           "python_version": "3.8.10 (default, Jun 22 2022, 20:18:18) \n[GCC 9.4.0]"
         },
         "additional_directories": [
-          "/home/kevin/code/scancode-toolkit/tests/licensedcode/data/example_external_licenses/example1",
           "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1"
         ],
         "spdx_license_list_version": "3.17",
@@ -51,9 +50,9 @@
           "owner": "NexB",
           "homepage_url": null,
           "text_url": "",
-          "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1",
-          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE",
-          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml",
+          "reference_url": "",
+          "scancode_text_url": "",
+          "scancode_data_url": "",
           "spdx_license_key": "scancode-example-installed1",
           "spdx_url": "https://spdx.org/licenses/scancode-example-installed1",
           "start_line": 1,

tests/licensedcode/data/plugin_license/installed_rules/scan.expected.json

@@ -21,6 +21,9 @@
           "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
           "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
         },
+        "additional_directories": [
+          "/home/kevin/code/scancode-toolkit/venv/lib/python3.8/site-packages/licenses_to_install1"
+        ],
         "spdx_license_list_version": "3.17",
         "files_count": 1
       }
@@ -42,9 +45,9 @@
           "owner": "NexB",
           "homepage_url": "",
           "text_url": "",
-          "reference_url": "https://scancode-licensedb.aboutcode.org/example-installed-1",
-          "scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.LICENSE",
-          "scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/example-installed-1.yml",
+          "reference_url": "",
+          "scancode_text_url": "",
+          "scancode_data_url": "",
           "spdx_license_key": "scancode-example-installed1",
           "spdx_url": "https://spdx.org/licenses/scancode-example-installed1",
           "start_line": 1,