Add new flag in License Data Model definition

CHANGELOG.rst

@@ -43,6 +43,25 @@ Package detection:
  - Add support to track installed files for each Package type.
 
 
+License detection:
+~~~~~~~~~~~~~~~~~~~
+
+- Unknown licenses have a new flag "is_unknown" to identify them
+  beyond just the naming convention of having "unknown" as part of their name.
+
+- Rules that match at least one unknown license have a flag "has_unknown" set
+  in the returned match results.
+
+
+Many thanks to every contributors that made this possible and in particular:
+
+- Akanksha Garg @akugarg
+- Ayan Sinha Mahapatra @AyanSinhaMahapatra
+- Jono Yang @JonoYang
+- Philippe Ombredanne @pombredanne
+
+
+
 v21.8.4
 ---------
 

docs/source/cli-reference/output-format.rst

@@ -177,6 +177,7 @@ following options.
               "short_name": "MIT Old Style",
               "category": "Permissive",
               "is_exception": false,
+              "is_unknown": false,
               "owner": "MIT",
               "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
               "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",

docs/source/cli-reference/synopsis.rst

@@ -228,6 +228,7 @@ A sample JSON output for an individual file will look like::
           "short_name": "MIT Old Style",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "MIT",
           "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
           "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",

src/licensedcode/data/licenses/free-unknown.yml

@@ -3,4 +3,6 @@ short_name: Free unknown
 name: Free unknown license detected but not recognized
 category: Unstated License
 owner: Unspecified
+is_unknown: yes
 spdx_license_key: LicenseRef-scancode-free-unknown
+

src/licensedcode/data/licenses/license-file-reference.yml

@@ -4,4 +4,5 @@ short_name: license-file-reference
 name: license-file-reference
 category: Unstated License
 owner: Unspecified
+is_unknown: yes
 notes: this was known before as "see-license" and is now unknown-license-reference

src/licensedcode/data/licenses/see-license.yml

@@ -4,4 +4,5 @@ name: See License mention
 category: Unstated License
 owner: Unspecified
 is_deprecated: yes
+is_unknown: yes
 notes: replaced by unknown-license-reference

src/licensedcode/data/licenses/unknown-license-reference.yml

@@ -3,6 +3,7 @@ short_name: Unknown License reference
 name: Unknown License file reference
 category: Unstated License
 owner: Unspecified
+is_unknown: yes
 notes: This is reference to a license file with no clear license. this was known before as "see-license"
     and "license-file-reference"
 spdx_license_key: LicenseRef-scancode-unknown-license-reference

src/licensedcode/data/licenses/unknown-spdx.yml

@@ -3,6 +3,7 @@ short_name: unknown SPDX
 name: Unknown SPDX license detected but not recognized
 category: Unstated License
 owner: Unspecified
+is_unknown: yes
 spdx_license_key: LicenseRef-scancode-unknown-spdx
 notes: This is something that clearly ressembles a license in an SPDX license
  expression but is not conclusively an SPDX license ID.

src/licensedcode/data/licenses/unknown.yml

@@ -3,5 +3,6 @@ short_name: unknown
 name: Unknown license detected but not recognized
 category: Unstated License
 owner: Unspecified
+is_unknown: yes
 spdx_license_key: LicenseRef-scancode-unknown
 notes: This is something that clearly ressembles a license but is not conclusive.

src/licensedcode/data/rules/unknown_german_1.yml

@@ -1,3 +1,3 @@
 license_expression: unknown
 is_license_reference: yes
-notes: in german
+notes: in german

src/licensedcode/models.py

@@ -108,6 +108,8 @@ class License(object):
     # if this is a license exception, the license key this exception applies to
     is_exception = __attrib(default=False)
 
+    # if the license falls in unknwon category then this flag should be set to true
+    is_unknown = __attrib(default=False)
     # SPDX key for SPDX licenses
     spdx_license_key = __attrib(default=None)
     # list of other keys, such as deprecated ones
@@ -342,6 +344,10 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
             if not lic.owner:
                 error('No owner')
 
+            if lic.is_unknown:
+                if not "unknown" in lic.key:
+                    error('is_unknown should not be true')
+
             # URLS dedupe and consistency
             if no_dupe_urls:
                 if lic.text_urls and not all(lic.text_urls):
@@ -863,6 +869,15 @@ def setup(self):
             self.license_expression = expression.render()
             self.license_expression_object = expression
 
+    @property
+    def has_unknown(self):
+        """
+        Return True if any of this rule licenses is an unknown license.
+        """
+        # TODO: consider using the license_expression_object and the is_unknown
+        # license flag instead
+        return self.license_expression and 'unknown' in self.license_expression
+
     def validate(self, licensing=None):
         """
         Validate this rule using the provided ``licensing`` Licensing and yield

src/scancode/api.py

@@ -239,6 +239,7 @@ def _licenses_data_from_match(
         result['short_name'] = lic.short_name
         result['category'] = lic.category
         result['is_exception'] = lic.is_exception
+        result['is_unknown'] = lic.is_unknown
         result['owner'] = lic.owner
         result['homepage_url'] = lic.homepage_url
         result['text_url'] = lic.text_urls[0] if lic.text_urls else ''
@@ -270,6 +271,7 @@ def _licenses_data_from_match(
         matched_rule['is_license_reference'] = match.rule.is_license_reference
         matched_rule['is_license_tag'] = match.rule.is_license_tag
         matched_rule['is_license_intro'] = match.rule.is_license_intro
+        matched_rule['has_unknown'] = match.rule.has_unknown
         matched_rule['matcher'] = match.matcher
         matched_rule['rule_length'] = match.rule.length
         matched_rule['matched_length'] = match.len()

tests/cluecode/data/plugin_filter_clues/filtered-expected.json

@@ -48,6 +48,7 @@
           "short_name": "Apache 1.1",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "Apache Software Foundation",
           "homepage_url": "http://www.apache.org/licenses/",
           "text_url": "http://apache.org/licenses/LICENSE-1.1",
@@ -69,6 +70,7 @@
             "is_license_reference": false,
             "is_license_tag": false,
             "is_license_intro": false,
+            "has_unknown": false,
             "matcher": "3-seq",
             "rule_length": 367,
             "matched_length": 367,

tests/cluecode/data/plugin_filter_clues/filtered-expected2.json

@@ -48,6 +48,7 @@
           "short_name": "PyGres License 2.2",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "Unspecified",
           "homepage_url": null,
           "text_url": "http://shell.vex.net/viewvc.cgi/pygresql/trunk/module/pgmodule.c?view=markup&pathrev=431",
@@ -69,6 +70,7 @@
             "is_license_reference": false,
             "is_license_tag": false,
             "is_license_intro": false,
+            "has_unknown": false,
             "matcher": "2-aho",
             "rule_length": 145,
             "matched_length": 145,

tests/cluecode/data/plugin_filter_clues/filtered-expected3.json

@@ -48,6 +48,7 @@
           "short_name": "PCRE License",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "University of Cambridge",
           "homepage_url": "http://www.pcre.org/licence.txt",
           "text_url": "http://www.pcre.org/licence.txt",
@@ -69,6 +70,7 @@
             "is_license_reference": false,
             "is_license_tag": false,
             "is_license_intro": false,
+            "has_unknown": false,
             "matcher": "1-hash",
             "rule_length": 303,
             "matched_length": 303,