Detect unknown licenses #1675

etc/scripts/licenses/buildrules.py

@@ -143,14 +143,22 @@ def rule_exists(text):
         return match.rule.identifier
 
 
-def all_rule_tokens():
+def all_rule_by_tokens():
     """
-    Return a set of tuples of tokens, one corresponding to every existing and
-    added rules. Used to avoid duplicates.
+    Return a mapping of {tuples of tokens: rule id}, with one item for each
+    existing and added rules. Used to avoid duplicates.
     """
-    rule_tokens = set()
+    rule_tokens = {}
     for rule in models.get_rules():
-        rule_tokens.add(tuple(rule.tokens()))
+        try:
+            rule_tokens[tuple(rule.tokens())] = rule.identifier
+        except Exception as e:
+            df=('  file://' + rule.data_file)
+            tf=('  file://' + rule.text_file)
+            raise Exception(
+                f'Failed to to get tokens from rule:: {rule.identifier}\n'
+                f'{df}\n{tf}'
+            ) from e
     return rule_tokens
 
 
@@ -185,7 +193,7 @@ def cli(licenses_file):
     """
 
     rules_data = load_data(licenses_file)
-    rules_tokens = all_rule_tokens()
+    rule_by_tokens = all_rule_by_tokens()
 
     licenses_by_key = cache.get_licenses_db()
     skinny_rules = []
@@ -205,10 +213,6 @@ def cli(licenses_file):
 
     print()
     for rule in skinny_rules:
-        existing = rule_exists(rule.text())
-        if existing:
-            print('Skipping existing rule:', existing, 'with text:\n', rule.text()[:50].strip(), '...')
-            continue
 
         if rule.is_false_positive:
             base_name = 'false-positive'
@@ -217,6 +221,21 @@ def cli(licenses_file):
         else:
             base_name = rule.license_expression
 
+        text = rule.text()
+
+        existing_rule = rule_exists(text)
+        skinny_text = ' '.join(text[:80].split())
+
+        existing_msg = (
+            f'Skipping rule for: {base_name!r}, '
+            'dupe of: {existing_rule} '
+            f'with text: {skinny_text!r}...'
+        )
+
+        if existing_rule:
+            print(existing_msg.format(**locals()))
+            continue
+
         base_loc = find_rule_base_loc(base_name)
 
         rd = rule.to_dict()
@@ -234,17 +253,20 @@ def cli(licenses_file):
 
         rule_tokens = tuple(rulerec.tokens())
 
-        if rule_tokens in rules_tokens:
-            print('Skipping already added rule with text for:', base_name)
+        existing_rule = rule_by_tokens.get(rule_tokens)
+        if existing_rule:
+            print(existing_msg.format(**locals()))
+            continue
         else:
-            print('Adding new rule:')
+            print(f'Adding new rule: {base_name}')
             print('  file://' + rulerec.data_file)
             print('  file://' + rulerec.text_file,)
-            rules_tokens.add(rule_tokens)
             rulerec.dump()
             models.update_ignorables(rulerec, verbose=False)
             rulerec.dump()
 
+            rule_by_tokens[rule_tokens] = base_name
+
 
 if __name__ == '__main__':
     cli()

src/licensedcode/data/licenses/agpl-3.0-plus.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: AGPL-3.0-or-later
 other_spdx_license_keys:
     - AGPL-3.0+
+    - LicenseRef-AGPL
 text_urls:
     - http://www.gnu.org/licenses/agpl.txt
 osi_url: http://www.opensource.org/licenses/agpl-v3.html

src/licensedcode/data/licenses/agpl-3.0.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: AGPL-3.0-only
 other_spdx_license_keys:
     - AGPL-3.0
+    - LicenseRef-AGPL-3.0
 text_urls:
     - http://www.fsf.org/licensing/licenses/agpl-3.0.html
 osi_url: http://www.opensource.org/licenses/agpl-v3.html

src/licensedcode/data/licenses/apache-2.0.yml

@@ -8,6 +8,9 @@ notes: |
     Per SPDX.org, this version was released January 2004 This license is OSI
     certified
 spdx_license_key: Apache-2.0
+other_spdx_license_keys:
+    - LicenseRef-Apache
+    - LicenseRef-Apache-2.0
 osi_license_key: Apache-2.0
 text_urls:
     - http://www.apache.org/licenses/LICENSE-2.0

src/licensedcode/data/licenses/gpl-1.0-plus.yml

@@ -8,6 +8,7 @@ notes: Per SPDX.org, this license was released February 1989.
 spdx_license_key: GPL-1.0-or-later
 other_spdx_license_keys:
     - GPL-1.0+
+    - LicenseRef-GPL
 text_urls:
     - http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html
 other_urls:

src/licensedcode/data/licenses/gpl-2.0.yml

@@ -22,6 +22,7 @@ spdx_license_key: GPL-2.0-only
 other_spdx_license_keys:
     - GPL-2.0
     - GPL 2.0
+    - LicenseRef-GPL-2.0
 text_urls:
     - http://www.gnu.org/licenses/gpl-2.0.txt
     - http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt

src/licensedcode/data/licenses/gpl-3.0-plus.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: GPL-3.0-or-later
 other_spdx_license_keys:
     - GPL-3.0+
+    - LicenseRef-GPL-3.0-or-later
 text_urls:
     - http://www.gnu.org/licenses/gpl-3.0-standalone.html
 other_urls:

src/licensedcode/data/licenses/gpl-3.0.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: GPL-3.0-only
 other_spdx_license_keys:
     - GPL-3.0
+    - LicenseRef-gpl-3.0
 text_urls:
     - http://www.gnu.org/licenses/gpl-3.0-standalone.html
     - http://www.gnu.org/licenses/gpl-3.0.txt

src/licensedcode/data/licenses/hippocratic-3.0.yml

@@ -0,0 +1,17 @@
+key: hippocratic-3.0
+short_name: Hippocratic License v3.0
+name: Hippocratic License v3.0
+category: Free Restricted
+owner: Ethical Source
+homepage_url: https://firstdonoharm.dev/
+spdx_license_key: LicenseRef-scancode-Hippocratic-3.0
+text_urls:
+    - https://firstdonoharm.dev/version/3/0/license/license.txt
+    - https://firstdonoharm.dev/version/3/0/license/license.md
+    - https://firstdonoharm.dev/version/3/0/license/
+faq_url: https://www.un.org/en/universal-declaration-human-rights/
+ignorable_urls:
+  - https://bdsmovement.net/
+  - https://bdsmovement.net/get-involved/what-to-boycott
+  - https://wsr-network.org/what-is-wsr/statement-of-principles
+

src/licensedcode/data/licenses/lgpl-2.0-plus.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: LGPL-2.0-or-later
 other_spdx_license_keys:
     - LGPL-2.0+
+    - LicenseRef-LGPL
 text_urls:
     - http://www.gnu.org/licenses/old-licenses/lgpl-2.0-standalone.html
 other_urls:

src/licensedcode/data/licenses/lgpl-2.0.yml

@@ -10,6 +10,8 @@ notes: |
 spdx_license_key: LGPL-2.0-only
 other_spdx_license_keys:
     - LGPL-2.0
+    - LicenseRef-LGPL-2
+    - LicenseRef-LGPL-2.0
 text_urls:
     - http://www.gnu.org/licenses/lgpl-2.0.html
     - http://www.gnu.org/licenses/lgpl-2.0.txt

src/licensedcode/data/licenses/lgpl-2.1.yml

@@ -10,6 +10,7 @@ notes: |
 spdx_license_key: LGPL-2.1-only
 other_spdx_license_keys:
     - LGPL-2.1
+    - LicenseRef-LGPL-2.1
 text_urls:
     - http://www.gnu.org/licenses/lgpl-2.1.txt
 osi_url: http://opensource.org/licenses/lgpl-2.1.php

src/licensedcode/data/licenses/openjdk-exception.yml

@@ -9,6 +9,8 @@ notes: |
     terms and found only in the OpenJDK
 is_exception: yes
 spdx_license_key: LicenseRef-scancode-openjdk-exception
+other_spdx_license_keys:
+    - Assembly-exception
 text_urls:
     - http://openjdk.java.net/legal/gplv2+ce.html
     - http://openjdk.java.net/legal/assembly-exception.html

src/licensedcode/data/licenses/public-domain.yml

@@ -5,6 +5,8 @@ category: Public Domain
 owner: Unspecified
 homepage_url: http://www.linfo.org/publicdomain.html
 spdx_license_key: LicenseRef-scancode-public-domain
+other_spdx_license_keys:
+    - LicenseRef-PublicDomain
 faq_url: http://www.linfo.org/publicdomain.html
 other_urls:
     - http://creativecommons.org/licenses/publicdomain/

src/licensedcode/data/licenses/reportbug.yml

@@ -8,3 +8,4 @@ notes: found in Debian reportbug
 spdx_license_key: LicenseRef-scancode-reportbug
 text_urls:
     - https://tracker.debian.org/media/packages/r/reportbug/copyright-6.6.6
+minimum_coverage: 80

src/licensedcode/data/rules/agpl-3.0-plus_276.RULE

@@ -0,0 +1 @@
+- LicenseRef-AGPL

src/licensedcode/data/rules/agpl-3.0-plus_276.yml

@@ -0,0 +1,3 @@
+license_expression: agpl-3.0-plus
+is_license_tag: yes
+relevance: 100

src/licensedcode/data/rules/agpl-3.0-plus_with_agpl-generic-additional-terms_25.RULE

@@ -0,0 +1,10 @@
+Additional permission under {{GNU AGPL version 3}} section 7
+
+If you modify this program, or any covered work, by linking or
+combining it with the OpenSSL project's OpenSSL library (or a
+modified version of that library), containing parts covered by the
+terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
+grants you additional permission to convey the resulting work.
+Corresponding Source for a non-source form of such a combination
+shall include the source code for the parts of OpenSSL used as well
+as that of the covered work.

src/licensedcode/data/rules/agpl-3.0-plus_with_agpl-generic-additional-terms_25.yml

@@ -0,0 +1,3 @@
+license_expression: agpl-3.0-plus WITH agpl-generic-additional-terms
+is_license_notice: yes
+notes: Seen in https://github.com/ca4ti/chiaki/blob/android-decoder-ndk-input-thread/LICENSES/AGPL-3.0-only-OpenSSL.txt

src/licensedcode/data/rules/agpl-3.0-plus_with_agpl-generic-additional-terms_26.RULE

@@ -0,0 +1 @@
+SPDX-License-Identifier: LicenseRef-GPL-3.0-or-later-OpenSSL

src/licensedcode/data/rules/agpl-3.0-plus_with_agpl-generic-additional-terms_26.yml

@@ -0,0 +1,4 @@
+license_expression: agpl-3.0-plus WITH agpl-generic-additional-terms
+is_license_tag: yes
+relevance: 100
+notes: Seen in https://github.com/ca4ti/chiaki/

src/licensedcode/data/rules/agpl-3.0_370.RULE

@@ -0,0 +1 @@
+- LicenseRef-AGPL-3.0

src/licensedcode/data/rules/agpl-3.0_370.yml

@@ -0,0 +1,3 @@
+license_expression: agpl-3.0
+is_license_tag: yes
+relevance: 100