Merge pull request #2592 from akugarg/improve_license_detection

Detect unknown licenses #1675

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

etc/scripts/licenses/buildrules.py

@@ -143,14 +143,22 @@ def rule_exists(text):
         return match.rule.identifier
 
 
-def all_rule_tokens():
+def all_rule_by_tokens():
     """
-    Return a set of tuples of tokens, one corresponding to every existing and
-    added rules. Used to avoid duplicates.
+    Return a mapping of {tuples of tokens: rule id}, with one item for each
+    existing and added rules. Used to avoid duplicates.
     """
-    rule_tokens = set()
+    rule_tokens = {}
     for rule in models.get_rules():
-        rule_tokens.add(tuple(rule.tokens()))
+        try:
+            rule_tokens[tuple(rule.tokens())] = rule.identifier
+        except Exception as e:
+            df=('  file://' + rule.data_file)
+            tf=('  file://' + rule.text_file)
+            raise Exception(
+                f'Failed to to get tokens from rule:: {rule.identifier}\n'
+                f'{df}\n{tf}'
+            ) from e
     return rule_tokens
 
 
@@ -185,7 +193,7 @@ def cli(licenses_file):
     """
 
     rules_data = load_data(licenses_file)
-    rules_tokens = all_rule_tokens()
+    rule_by_tokens = all_rule_by_tokens()
 
     licenses_by_key = cache.get_licenses_db()
     skinny_rules = []
@@ -205,10 +213,6 @@ def cli(licenses_file):
 
     print()
     for rule in skinny_rules:
-        existing = rule_exists(rule.text())
-        if existing:
-            print('Skipping existing rule:', existing, 'with text:\n', rule.text()[:50].strip(), '...')
-            continue
 
         if rule.is_false_positive:
             base_name = 'false-positive'
@@ -217,6 +221,21 @@ def cli(licenses_file):
         else:
             base_name = rule.license_expression
 
+        text = rule.text()
+
+        existing_rule = rule_exists(text)
+        skinny_text = ' '.join(text[:80].split())
+
+        existing_msg = (
+            f'Skipping rule for: {base_name!r}, '
+            'dupe of: {existing_rule} '
+            f'with text: {skinny_text!r}...'
+        )
+
+        if existing_rule:
+            print(existing_msg.format(**locals()))
+            continue
+
         base_loc = find_rule_base_loc(base_name)
 
         rd = rule.to_dict()
@@ -234,17 +253,20 @@ def cli(licenses_file):
 
         rule_tokens = tuple(rulerec.tokens())
 
-        if rule_tokens in rules_tokens:
-            print('Skipping already added rule with text for:', base_name)
+        existing_rule = rule_by_tokens.get(rule_tokens)
+        if existing_rule:
+            print(existing_msg.format(**locals()))
+            continue
         else:
-            print('Adding new rule:')
+            print(f'Adding new rule: {base_name}')
             print('  file://' + rulerec.data_file)
             print('  file://' + rulerec.text_file,)
-            rules_tokens.add(rule_tokens)
             rulerec.dump()
             models.update_ignorables(rulerec, verbose=False)
             rulerec.dump()
 
+            rule_by_tokens[rule_tokens] = base_name
+
 
 if __name__ == '__main__':
     cli()

src/licensedcode/data/licenses/agpl-3.0-plus.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: AGPL-3.0-or-later
 other_spdx_license_keys:
     - AGPL-3.0+
+    - LicenseRef-AGPL
 text_urls:
     - http://www.gnu.org/licenses/agpl.txt
 osi_url: http://www.opensource.org/licenses/agpl-v3.html

src/licensedcode/data/licenses/agpl-3.0.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: AGPL-3.0-only
 other_spdx_license_keys:
     - AGPL-3.0
+    - LicenseRef-AGPL-3.0
 text_urls:
     - http://www.fsf.org/licensing/licenses/agpl-3.0.html
 osi_url: http://www.opensource.org/licenses/agpl-v3.html

src/licensedcode/data/licenses/agpl-generic-additional-terms.yml

@@ -6,4 +6,5 @@ owner: Unspecified
 notes: this is a generic entry for rare one-off AGPL extra license terms. These are typically
     additional terms under section 7 of the AGPL-3.0
 is_exception: yes
+is_generic: yes
 spdx_license_key: LicenseRef-scancode-agpl-generic-additional-terms

src/licensedcode/data/licenses/apache-2.0.yml

@@ -8,6 +8,9 @@ notes: |
     Per SPDX.org, this version was released January 2004 This license is OSI
     certified
 spdx_license_key: Apache-2.0
+other_spdx_license_keys:
+    - LicenseRef-Apache
+    - LicenseRef-Apache-2.0
 osi_license_key: Apache-2.0
 text_urls:
     - http://www.apache.org/licenses/LICENSE-2.0

src/licensedcode/data/licenses/commercial-license.yml

@@ -3,4 +3,6 @@ short_name: Commercial License
 name: Commercial License
 category: Commercial
 owner: Unspecified
+is_generic: yes
+notes: this is a generic commercial license
 spdx_license_key: LicenseRef-scancode-commercial-license

src/licensedcode/data/licenses/commercial-option.yml

@@ -4,4 +4,5 @@ short_name: Commercial Option
 name: Commercial Option
 category: Commercial
 owner: Unspecified
+is_generic: yes
 notes: replaced by commercial-license

src/licensedcode/data/licenses/generic-cla.yml

@@ -1,4 +1,5 @@
 key: generic-cla
+is_generic: yes
 short_name: Generic CLA
 name: Prior Generic Contributor License Agreement
 category: Unstated License

src/licensedcode/data/licenses/generic-exception.yml

@@ -6,4 +6,5 @@ owner: Unspecified
 notes: this is a generic license exception notice. Actual terms are most commonly related to
     rare, one-off extra permission to the A/L/GPL licenses
 is_exception: yes
+is_generic: yes
 spdx_license_key: LicenseRef-scancode-generic-exception

src/licensedcode/data/licenses/generic-export-compliance.yml

@@ -5,4 +5,5 @@ category: Unstated License
 owner: Unspecified
 notes: this is a generic export compliance notice. Actual terms are most commonly related to
     cryptography
+is_generic: yes
 spdx_license_key: LicenseRef-scancode-generic-export-compliance