Improve license detection accuracy

CHANGELOG.rst

@@ -7,53 +7,69 @@ v21.x.x (next, future)
 Important API changes:
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
- - The data structure of the JSON output is now versioned and the next version
-   is available with a new command line option. We are also documenting a new
-   and clear API policy and backward compatibility policy.
-
- - The data structure of the JSON output has changed for copyrights, authors
-   and holders: we now use proper name for attributes and not a generic "value".
-
- - The data structure of the JSON output has changed for licenses: we now
-   return match details once for each matched license expression rather than
-   once for each license in a matched expression. There is a new top-level
-   "licenses" attributes that contains the data details for each detected
-   licenses only once. This data can contain the reference license text
-   as an option.
-
- - The data structure of the JSON output has changed for packages: we now
-   return "package_manifests" package information at the manifest file-level
-   rather than "packages". There is a a new top-level "packages" attribute
-   that contains each package instance that can be aggregating data from
-   multiple manifests for a single package instance.
-
- - The data structure for HTML output has been changed to include emails and urls under the 
-   "infos" object. Now HTML template will output holders, authors, emails, and 
-   urls into separate tables like "licenses" and "copyrights".
+- The data structure of the JSON output is now versioned and the next version
+  is available with a new command line option. We are also documenting a new
+  and clear API policy and backward compatibility policy.
+
+- The data structure of the JSON output has changed for copyrights, authors
+  and holders: we now use proper name for attributes and not a generic "value".
+
+- The data structure of the JSON output has changed for licenses: we now
+  return match details once for each matched license expression rather than
+  once for each license in a matched expression. There is a new top-level
+  "licenses" attributes that contains the data details for each detected
+  licenses only once. This data can contain the reference license text
+  as an option.
+
+- The data structure of the JSON output has changed for packages: we now
+  return "package_manifests" package information at the manifest file-level
+  rather than "packages". There is a a new top-level "packages" attribute
+  that contains each package instance that can be aggregating data from
+  multiple manifests for a single package instance.
+
+- The data structure for HTML output has been changed to include emails and urls under the 
+  "infos" object. Now HTML template will output holders, authors, emails, and 
+  urls into separate tables like "licenses" and "copyrights".
 
 Copyright detection:
 ~~~~~~~~~~~~~~~~~~~~
 
- - The data structure in the JSON is now using consistently named attributes as
-   opposed to a plain value.
+- The data structure in the JSON is now using consistently named attributes as
+  opposed to a plain value.
+- Several copyright detection bugs have been fixed. 
 
 
 Package detection:
 ~~~~~~~~~~~~~~~~~~
 
- - Add support for OpenWRT packages.
- - Add support for Yocto/BitBake .bb recipes.
- - Add support to track installed files for each Package type.
+- Add support for OpenWRT packages.
+- Add support for Yocto/BitBake .bb recipes.
+- Add support to track installed files for each Package type.
+- Debian copyright license detection has been significantly improved with new
+  license detection rules.
 
 
 License detection:
 ~~~~~~~~~~~~~~~~~~~
 
+- There have been XXX new licenses added, YYY new license detection rules added
+  and ZZZ updated license or rules.
+
+- Several license detection bugs have fixed.
+
+- The SPDX license list 3.14 is now supported. We also include the version
+  of the SPDX license list in the ScanCode JSON and SPDX outputs, as well as
+  display it with the --version command line option.
+
 - Unknown licenses have a new flag "is_unknown" to identify them
   beyond just the naming convention of having "unknown" as part of their name.
 
 - Rules that match at least one unknown license have a flag "has_unknown" set
   in the returned match results.
+
+- There is a new experimental command line option "--unknown-licenses" to
+  detect unknown licenses and follow license references such as "See license in
+  file COPYING". The actual data structure for this new option is evolving.
 
 
 Many thanks to every contributors that made this possible and in particular:
@@ -64,7 +80,6 @@ Many thanks to every contributors that made this possible and in particular:
 - Philippe Ombredanne @pombredanne
 
 
-
 v21.8.4
 ---------
 

etc/scripts/licenses/buildrules.py

@@ -226,6 +226,9 @@ def cli(licenses_file):
 
         rulerec = models.Rule(**rd)
 
+        # force recomputing relevance to remove junk stored relevance for long rules
+        rulerec.compute_relevance(_threshold=18.0)
+
         rulerec.data_file = base_loc + '.yml'
         rulerec.text_file = base_loc + '.RULE'
 
@@ -234,10 +237,13 @@ def cli(licenses_file):
         if rule_tokens in rules_tokens:
             print('Skipping already added rule with text for:', base_name)
         else:
+            print('Adding new rule:')
+            print('  file://' + rulerec.data_file)
+            print('  file://' + rulerec.text_file,)
             rules_tokens.add(rule_tokens)
+            rulerec.dump()
             models.update_ignorables(rulerec, verbose=False)
             rulerec.dump()
-            print('Rule added:', 'file://' + rulerec.data_file, '\n', 'file://' + rulerec.text_file,)
 
 
 if __name__ == '__main__':

etc/scripts/licenses/genrulevariants.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import click
+
+from licensedcode import models
+
+"""
+A script to generate license detection rules from existing license rules by
+replacing strings.
+"""
+
+from buildrules import find_rule_base_loc
+from buildrules import rule_exists
+
+
+def get_rules(source, replacement):
+    """
+    Yield tuple of (rule, new text) for non-false positive existing Rules with a
+    text that contains source.
+    """
+    for rule in models.load_rules():
+        if rule.is_false_positive:
+            continue
+        text = rule.text()
+        if source in text:
+            yield rule, text.replace(source, replacement)
+
+
+@click.command()
+@click.option('--source', metavar='SOURCE', type=str, help='The source, old string to replace.')
+@click.option('--replacement', metavar='REPLACEMENT', type=str, help='The replacement string to use.')
+@click.help_option('-h', '--help')
+def cli(source, replacement):
+    """
+    Create new license detection rules from existing rules by replacing a SOURCE
+    string by a REPLACEMENT string in any rule text that contains this SOURCE string.
+    """
+
+    for rule, new_text in get_rules(source, replacement):
+        existing = rule_exists(new_text)
+        if existing:
+            continue
+
+        if rule.is_license_intro:
+            base_name = 'license-intro'
+        else:
+            base_name = rule.license_expression
+
+        base_loc = find_rule_base_loc(base_name)
+
+        rd = rule.to_dict()
+        rd['stored_text'] = new_text
+        rd['has_stored_relevance'] = rule.has_stored_relevance
+        rd['has_stored_minimum_coverage'] = rule.has_stored_minimum_coverage
+
+        rulerec = models.Rule(**rd)
+
+        # force recomputing relevance to remove junk stored relevance for long rules
+        rulerec.compute_relevance(_threshold=18.0)
+
+        rulerec.data_file = base_loc + '.yml'
+        rulerec.text_file = base_loc + '.RULE'
+
+        print('Adding new rule:')
+        print('  file://' + rulerec.data_file)
+        print('  file://' + rulerec.text_file,)
+        rulerec.dump()
+        models.update_ignorables(rulerec, verbose=False)
+        rulerec.dump()
+
+
+if __name__ == '__main__':
+    cli()

etc/scripts/licenses/synclic.py

@@ -406,16 +406,25 @@ def build_license(self, mapping, skip_oddities=True, scancode_licenses=None):
             return
 
         # these keys have a complicated history
-        if skip_oddities and key in set([
-            'gpl-1.0', 'gpl-2.0', 'gpl-3.0',
-            'lgpl-2.0', 'lgpl-2.1', 'lgpl-3.0',
-            'agpl-1.0', 'agpl-2.0', 'agpl-3.0',
-            'gfdl-1.1', 'gfdl-1.2', 'gfdl-1.3',
+        spdx_keys_with_complicated_past = set([
+            'gpl-1.0',
+            'gpl-2.0',
+            'gpl-3.0',
+            'lgpl-2.0',
+            'lgpl-2.1',
+            'lgpl-3.0',
+            'agpl-1.0',
+            'agpl-2.0',
+            'agpl-3.0',
+            'gfdl-1.1',
+            'gfdl-1.2',
+            'gfdl-1.3',
             'nokia-qt-exception-1.1',
             'bzip2-1.0.5',
             'bsd-2-clause-freebsd',
             'bsd-2-clause-netbsd',
-        ]):
+        ])
+        if skip_oddities and key in spdx_keys_with_complicated_past:
             return
 
         deprecated = mapping.get('isDeprecatedLicenseId', False)
@@ -505,8 +514,9 @@ def __init__(self, external_base_dir, api_base_url=None, api_key=None):
         self.api_base_url = api_base_url or os.getenv('DEJACODE_API_URL')
         self.api_key = api_key or os.getenv('DEJACODE_API_KEY')
         assert (self.api_key and self.api_base_url), (
-            'You must set the DEJACODE_API_URL and DEJACODE_API_KEY ' +
-            'environment variables before running this script.')
+            'You must set the DEJACODE_API_URL and DEJACODE_API_KEY '
+            'environment variables before running this script.'
+        )
 
         super(DejaSource, self).__init__(external_base_dir)
 
@@ -546,14 +556,30 @@ def build_license(self, mapping, scancode_licenses):
             return
 
         # these licenses are combos of many others and are ignored: we detect
-        # instead each part of the combo
+        # instead each part of the combos separately
         dejacode_special_composites = set([
-              'intel-bsd-special',
-              # 'newlib-subdirectory',
-            ])
-        is_component_license = mapping.get('is_component_license') or False
-
-        is_combo = is_component_license or key in dejacode_special_composites
+            'net-snmp',
+            'aes-128-3.0',
+            'agpl-3.0-bacula',
+            'bacula-exception',
+            'componentace-jcraft',
+            'nvidia-cuda-supplement-2020',
+            'dejacode',
+            'ibm-icu',
+            'unicode-icu-58',
+            'info-zip-1997-10',
+            'info-zip-2001-01',
+            'info-zip-2002-02',
+            'info-zip-2003-05',
+            'info-zip-2004-05',
+            'info-zip-2005-02',
+            'info-zip-2007-03',
+            'info-zip-2009-01',
+            'intel-bsd-special',
+            'lgpl-3.0-plus-openssl',
+            'newlib-subdirectory',
+        ])
+        is_combo = key in dejacode_special_composites
         if is_combo:
             if TRACE: print('Skipping DejaCode combo/component license', key)
             return

requirements.txt

@@ -12,7 +12,7 @@ colorama==0.4.4
 commoncode==21.8.31
 construct==2.10.67
 cryptography==3.4.7
-debian-inspector==21.5.25
+debian-inspector==30.0.0
 dparse==0.5.1
 extractcode==21.7.23
 extractcode-7z==16.5.210531

setup-mini.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit-mini
-version = 21.8.4
+version = 30.0.0
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts. scancode-toolkit-mini is a special build that does not come with pre-built binary dependencies by default. These are instead installed separately or with the extra_requires scancode-toolkit-mini[full]
@@ -59,8 +59,8 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 21.8.27
-    debian-inspector >= 21.5.25
+    commoncode >= 21.8.31
+    debian-inspector >= 30.0.0
     dparse >= 0.5.1
     fasteners
     fingerprints >= 0.6.0
@@ -197,6 +197,7 @@ scancode_output =
     jsonlines = formattedcode.output_jsonlines:JsonLinesOutput
     template = formattedcode.output_html:CustomTemplateOutput
     debian = formattedcode.output_debian:DebianCopyrightOutput
+    yaml = formattedcode.output_yaml:YamlOutput
 
 
 [tool:pytest]

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit
-version = 21.8.4
+version = 30.0.0
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts.
@@ -60,7 +60,7 @@ install_requires =
     click >= 6.7, !=7.0
     colorama >= 0.3.9
     commoncode >= 21.8.31
-    debian-inspector >= 21.5.25
+    debian-inspector >= 30.0.0
     dparse >= 0.5.1
     fasteners
     fingerprints >= 0.6.0