Add new license rule model attributes

To the license Rule class:

- Adds is_license_clue attribute
- Adds is_deprecated attribute

Also implements related processing.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

etc/scripts/licenses/buildrules-template.txt

@@ -222,6 +222,8 @@ def cli(licenses_file):
             base_name = "false-positive"
         elif rule.is_license_intro:
             base_name = "license-intro"
+        elif rule.is_license_clue:
+            base_name = "license-clue"
         else:
             base_name = rule.license_expression
 

etc/scripts/licenses/buildrules.py

@@ -61,6 +61,8 @@
     "is_license_tag",
     "is_license_reference",
     "is_license_intro",
+    "is_license_clue",
+    "is_deprecated",
     "has_unknown",
     "only_known_words",
     "notes",

etc/scripts/licenses/report_license_rules.py

@@ -1189,7 +1189,12 @@ def get_license_tokens():
     yield 'licensed'
 
 
-def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True):
+def load_rules(
+    rules_data_dir=rules_data_dir,
+    with_checks=True,
+    is_builtin=True,
+    ignore_deprecated=True,
+):
     """
     Return an iterable of rules loaded from rule files in ``rules_data_dir``.
     Optionally check for consistency if ``with_checks`` is True.
@@ -1211,7 +1216,12 @@ def load_rules(rules_data_dir=rules_data_dir, with_checks=True, is_builtin=True)
                 space_problems.append(rule_file)
 
             try:
-                yield Rule.from_file(rule_file=rule_file)
+                rule = Rule.from_file(rule_file=rule_file)
+                if rule.is_deprecated and ignore_deprecated:
+                    continue 
+                else:
+                    yield rule
+
             except Exception as re:
                 if with_checks:
                     model_errors.append(str(re))
@@ -1387,6 +1397,22 @@ class BasicRule:
             'after. Mutually exclusive from any other is_license_* flag')
     )
 
+    is_license_clue = attr.ib(
+        default=False,
+        repr=False,
+        metadata=dict(
+            help='True if this is rule text is a clue to a license '
+            'but cannot be considered in a proper license detection '
+            'as a license text/notice/reference/tag/intro as it is'
+            'merely a clue and does not actually point to or refer to '
+            'the actual license directly. This is still valuable information '
+            'useful in determining the license/origin of a file, but this '
+            'should not be summarized/present in the license expression for '
+            'a package or a file, nor its list of license detections. '
+            'considered in the context of the detection that it precedes. '
+            'Mutually exclusive from any other is_license_* flag')
+    )
+
     is_false_positive = attr.ib(
         default=False,
         repr=False,
@@ -1505,6 +1531,19 @@ class BasicRule:
             'built at runtime, such as an SPDX license rule.')
     )
 
+    is_deprecated = attr.ib(
+        default=False,
+        repr=False,
+        metadata=dict(
+            help='Flag set to True if this rule is deleted, '
+            'and not to be used anymore in license detection. '
+            'This happens usually when a rule is renamed/assigned '
+            'to a seperate license-expression, promoted to being a '
+            'license text or just plain retired. This is used to '
+            'preserve the link to the rule, and therefore make links '
+            'to rules as permanent.')
+    )
+
     ###########################################################################
     # lists of clues that can be ignored when detected in this license as they
     # are part of the license or rule text itself
@@ -1769,6 +1808,7 @@ def validate(self, licensing=None, thorough=False):
             self.is_license_reference,
             self.is_license_tag,
             self.is_license_intro,
+            self.is_license_clue,
         )
 
         has_license_flags = any(license_flags)
@@ -1924,6 +1964,7 @@ def to_reference(self):
         data['is_license_reference'] = self.is_license_reference
         data['is_license_tag'] = self.is_license_tag
         data['is_license_intro'] = self.is_license_intro
+        data['is_license_clue'] = self.is_license_clue
         data['is_continuous'] = self.is_continuous
         data['is_builtin'] = self.is_builtin
         data['is_from_license'] = self.is_from_license
@@ -1961,6 +2002,7 @@ def to_dict(self, include_text=False):
             'is_license_reference',
             'is_license_tag',
             'is_license_intro',
+            'is_license_clue',
             'is_continuous',
         )
 
@@ -2253,7 +2295,9 @@ def load(self, rule_file, with_checks=True):
         self.is_license_tag = data.get('is_license_tag', False)
         self.is_license_reference = data.get('is_license_reference', False)
         self.is_license_intro = data.get('is_license_intro', False)
+        self.is_license_clue = data.get('is_license_clue', False)
         self.is_continuous = data.get('is_continuous', False)
+        self.is_deprecated = data.get('is_deprecated', False)
 
         self.referenced_filenames = data.get('referenced_filenames', []) or []
 

src/licensedcode/models.py

@@ -216,7 +216,7 @@ def dget(s):
             name = i_title or am_nm or ext_nm or nm
             descriptions = [s_title, i_title, nm]
 
-    datasource_id =get_datasource_id(package_type=package_type)
+    datasource_id = get_datasource_id(package_type=package_type)
 
     descriptions = unique(descriptions)
     descriptions = [d for d in descriptions if d and d.strip() and d != name]

src/packagedcode/jar_manifest.py

@@ -250,6 +250,7 @@ class LicenseFilter(object):
     is_license_tag=LicenseFilter(min_coverage=100),
     is_license_reference=LicenseFilter(min_score=50, min_coverage=100),
     is_license_intro=LicenseFilter(min_score=100, min_coverage=100),
+    is_license_clue=LicenseFilter(min_score=100, min_coverage=100),
 )
 
 
@@ -268,6 +269,7 @@ def is_good_license(license_match_object):
             ('is_license_reference', license_match_object.rule.is_license_reference),
             ('is_license_tag', license_match_object.rule.is_license_tag),
             ('is_license_intro', license_match_object.rule.is_license_intro),
+            ('is_license_clue', license_match_object.rule.is_license_clue),
         ]
     )
     matched = False

src/summarycode/score.py

@@ -29,8 +29,8 @@ headers:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-5.15.0-75-generic-x86_64-with-glibc2.29
-        platform_version: '#82~20.04.1-Ubuntu SMP Wed Jun 7 19:37:37 UTC 2023'
+        platform: Linux-5.15.0-76-generic-x86_64-with-glibc2.29
+        platform_version: '#83~20.04.1-Ubuntu SMP Wed Jun 21 20:23:31 UTC 2023'
         python_version: "3.8.10 (default, May 26 2023, 14:05:08) \n[GCC 9.4.0]"
       spdx_license_list_version: '3.21'
       files_count: 4
@@ -1126,6 +1126,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1150,6 +1151,7 @@ license_rule_references:
     is_license_reference: yes
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1174,6 +1176,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1199,6 +1202,7 @@ license_rule_references:
     is_license_reference: yes
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1224,6 +1228,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1248,6 +1253,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1284,6 +1290,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: no
     is_license_intro: yes
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1311,6 +1318,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1335,6 +1343,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1359,6 +1368,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -1383,6 +1393,7 @@ license_rule_references:
     is_license_reference: yes
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no

tests/formattedcode/data/common/manifests-expected.yaml

@@ -29,10 +29,10 @@ headers:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-5.15.0-73-generic-x86_64-with-glibc2.29
-        platform_version: '#80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023'
+        platform: Linux-5.15.0-76-generic-x86_64-with-glibc2.29
+        platform_version: '#83~20.04.1-Ubuntu SMP Wed Jun 21 20:23:31 UTC 2023'
         python_version: "3.8.10 (default, May 26 2023, 14:05:08) \n[GCC 9.4.0]"
-      spdx_license_list_version: '3.20'
+      spdx_license_list_version: '3.21'
       files_count: 4
 summary:
   declared_license_expression: apache-2.0
@@ -369,6 +369,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: yes
@@ -596,6 +597,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: yes
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -620,6 +622,7 @@ license_rule_references:
     is_license_reference: yes
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: no
@@ -644,6 +647,7 @@ license_rule_references:
     is_license_reference: yes
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: yes
     is_builtin: yes
     is_from_license: no
@@ -668,6 +672,7 @@ license_rule_references:
     is_license_reference: no
     is_license_tag: no
     is_license_intro: no
+    is_license_clue: no
     is_continuous: no
     is_builtin: yes
     is_from_license: yes

tests/formattedcode/data/yaml/package-and-licenses-expected.yaml

@@ -196,6 +196,7 @@
       "is_license_reference": false,
       "is_license_tag": true,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": true,
       "is_from_license": false,
@@ -222,6 +223,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,
@@ -248,6 +250,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,
@@ -274,6 +277,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,
@@ -300,6 +304,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,

tests/licensedcode/data/additional_licenses/additional_license_combined_test.expected.json

@@ -85,6 +85,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,
@@ -111,6 +112,7 @@
       "is_license_reference": false,
       "is_license_tag": false,
       "is_license_intro": false,
+      "is_license_clue": false,
       "is_continuous": false,
       "is_builtin": false,
       "is_from_license": true,