Merge pull request #2765 from nexB/omnibus-fall3-license-improvements

Add new licenses and new detection rules

Author: pombredanne

CHANGELOG.rst

@@ -5,6 +5,7 @@ Changelog
 -----------------------
 
 
+
 Important API changes:
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -36,7 +37,9 @@ Copyright detection:
 
 - The data structure in the JSON is now using consistently named attributes as
   opposed to a plain value.
-- Several copyright detection bugs have been fixed. 
+- Several copyright detection bugs have been fixed.
+- French and German copyright detection is improved.
+- Some spurious trailing dots in holders are not stripped.
 
 
 License detection:
@@ -53,6 +56,27 @@ License detection:
   `{{` and `}}`. When defined a RULE will only match when the key phrases match
   exactly.
 
+- The rule attribute "only_known_words" has been renamed to "is_continuous" and its
+  meaning has been updated and expanded. A rule tagged as "is_continuous" can only
+  be matched if there are no gaps between matched words, be they stopwords, extra
+  unknown or known words. This improves several false positive license detections. 
+
+- When scanning binary files, the detection of single word rules is filtered when
+  surrounded by gibberish or is using mixed case. For instance $#%$GpL$ is a false
+  positive and is no longer reported.
+
+- Several rules we tagged as is_license_notice incorrectly but were references
+  and have been requalified as is_license_reference. All rules made of a single
+  ord have been requalified as is_license_reference if they were not qualified
+  this way.
+
+- Matches to small license rules (with small defined as under 15 words)
+  that are scattered on too many lines are now filtered as false matches.
+
+- Small, two-words matches that overlap the previous or next match by
+  by the word "license" and assimilated are now filtered as false matches.
+
+
 Package detection:
 ~~~~~~~~~~~~~~~~~~
 

conftest.py

@@ -57,14 +57,6 @@ def pytest_addoption(parser):
     """
     group = parser.getgroup('scancode', 'Test suite options for ScanCode')
 
-    group.addoption(
-        '--force-py3',
-        dest='force_py3',
-        action='store_true',
-        default=False,
-        help='[DEPRECATED and ignored] Python 3 port is completed.',
-    )
-
     group.addoption(
         '--test-suite',
         action='store',

etc/scripts/licenses/buildrules.py

@@ -227,7 +227,7 @@ def cli(licenses_file):
         rulerec = models.Rule(**rd)
 
         # force recomputing relevance to remove junk stored relevance for long rules
-        rulerec.compute_relevance(_threshold=18.0)
+        rulerec.set_relevance()
 
         rulerec.data_file = base_loc + '.yml'
         rulerec.text_file = base_loc + '.RULE'

etc/scripts/licenses/gen_spdx_lists.py

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import click
+
+from licensedcode.cache import get_licenses_by_spdx_key
+
+import synclic
+
+"""
+A script to generate license detection rules from lists of SPDX
+licenses for their name or id/name combos.
+
+It is common to see SPDX license names and ids used for licensing documentation.
+
+Here we fetch the latest SPDX licenses list and generate rules for each
+license id/name, name and a few other related combinations.
+"""
+
+TRACE = False
+
+template = '''----------------------------------------
+license_expression: {key}
+relevance: 100
+{is_license}: yes
+minimum_coverage: 100
+is_continuous: yes
+notes: Rule based on an SPDX license identifier and name
+---
+{text}
+'''
+
+
+@click.command()
+@click.argument(
+    # 'A buildrules-formatted file used to generate new licenses rules.')
+    'output', type=click.Path(), metavar='FILE')
+
+@click.help_option('-h', '--help')
+def cli(output):
+    """
+    Generate ScanCode license detection rules from a list of SPDX
+    license. Save these in FILE for use with buildrules.
+
+    The `spdx` directory is used as a temp store for fetched SPDX licenses.
+    """
+
+    licenses_by_spdx_key = get_licenses_by_spdx_key(
+        licenses=None,
+        include_deprecated=False,
+        lowercase_keys=False,
+        include_other_spdx_license_keys=True,
+    )
+
+    spdx_source = synclic.SpdxSource(external_base_dir=None)
+    spdx_data = list(spdx_source.fetch_spdx_licenses())
+
+    messages = []
+    with open(output, 'w') as o:
+        for spdx in spdx_data:
+            is_exception = 'licenseExceptionId' in spdx
+            spdx_key = spdx.get('licenseId') or spdx.get('licenseExceptionId')
+            name = spdx['name']
+            lic = licenses_by_spdx_key.get(spdx_key)
+            if not lic:
+                print('--> Skipping SPDX license unknown in ScanCode:', spdx_key,)
+                continue
+            for rule in build_rules(lic.key, spdx_key, name, is_exception):
+                o.write(rule)
+
+        o.write('----------------------------------------\n')
+
+    for msg in messages:
+        print(*msg)
+
+
+def build_rules(key, spdx_key, name, is_exception=False):
+    yield template.format(
+        key=key,
+        is_license='is_license_reference',
+        text=name,
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_reference',
+        text=f'name: {name}',
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_reference',
+        text=f'{spdx_key} {name}',
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_reference',
+        text=f'{name} {spdx_key}',
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_tag',
+        text=f'{spdx_key} {name}',
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_tag',
+        text=f'license: {spdx_key}',
+    )
+
+    yield template.format(
+        key=key,
+        is_license='is_license_tag',
+        text=f'license: {name}',
+    )
+
+    if is_exception:
+        yield template.format(
+            key=key,
+            is_license='is_license_tag',
+            text=f'licenseExceptionId: {spdx_key}',
+        )
+    else:
+        yield template.format(
+            key=key,
+            is_license='is_license_tag',
+            text=f'licenseId: {spdx_key}',
+        )
+
+
+if __name__ == '__main__':
+    cli()

etc/scripts/licenses/gen_spdx_lists_fp.py

@@ -64,7 +64,7 @@ def cli(source, replacement):
         rulerec = models.Rule(**rd)
 
         # force recomputing relevance to remove junk stored relevance for long rules
-        rulerec.compute_relevance(_threshold=18.0)
+        rulerec.set_relevance()
 
         rulerec.data_file = base_loc + '.yml'
         rulerec.text_file = base_loc + '.RULE'