Improve licenses detection accuracy of unknowns using ngrams

Signed-off-by: akugarg <akanksha.garg2k@gmail.com>

Author: akugarg

src/licensedcode/index.py

@@ -19,14 +19,15 @@
 
 from intbitset import intbitset
 
-from licensedcode import SMALL_RULE
+from licensedcode import SMALL_RULE, match_unknown
 from licensedcode.legalese import common_license_words
 from licensedcode import match
 from licensedcode import match_aho
 from licensedcode import match_hash
 from licensedcode import match_seq
 from licensedcode import match_set
 from licensedcode import match_spdx_lid
+from licensedcode import match_unknown
 from licensedcode.dmp import match_blocks as match_blocks_dmp
 from licensedcode.seq import match_blocks as match_blocks_seq
 from licensedcode import query
@@ -128,6 +129,7 @@ class LicenseIndex(object):
         'rules_automaton',
         'fragments_automaton',
         'starts_automaton',
+        'unknown_ngrams',
 
         'regular_rids',
         'false_positive_rids',
@@ -136,7 +138,7 @@ class LicenseIndex(object):
         'optimized',
     )
 
-    def __init__(self, rules=None, _legalese=common_license_words, _spdx_tokens=frozenset()):
+    def __init__(self, rules=None, _legalese=common_license_words, _spdx_tokens=frozenset(), _unknown_ngram_length=7):
         """
         Initialize the index with an iterable of Rule objects.
         `_legalese` is a set of common license-specific words aka. legalese
@@ -185,6 +187,7 @@ def __init__(self, rules=None, _legalese=common_license_words, _spdx_tokens=froz
         self.rules_automaton = match_aho.get_automaton()
         self.fragments_automaton = USE_AHO_FRAGMENTS and match_aho.get_automaton()
         self.starts_automaton = USE_RULE_STARTS and match_aho.get_automaton()
+        self.unknown_ngrams = match_aho.get_automaton()
 
         # disjunctive sets of rule ids: regular and false positive
 
@@ -206,7 +209,7 @@ def __init__(self, rules=None, _legalese=common_license_words, _spdx_tokens=froz
                 logger_debug('LicenseIndex: building index.')
             # index all and optimize
             self._add_rules(
-                rules, _legalese=_legalese, _spdx_tokens=_spdx_tokens)
+                rules, _legalese=_legalese, _spdx_tokens=_spdx_tokens, , _unknown_ngram_length=_unknown_ngram_length)
 
             if TRACE_TOKEN_DOC_FREQ:
                 logger_debug('LicenseIndex: token, frequency')
@@ -222,7 +225,7 @@ def __init__(self, rules=None, _legalese=common_license_words, _spdx_tokens=froz
                       '%(duration)f seconds.' % locals())
                 self._print_index_stats()
 
-    def _add_rules(self, rules, _legalese=common_license_words, _spdx_tokens=frozenset()):
+    def _add_rules(self, rules, _legalese=common_license_words, _spdx_tokens=frozenset(), _unknown_ngram_length=7):
         """
         Add a list of Rule objects to the index and constructs optimized and
         immutable index structures.
@@ -358,6 +361,12 @@ def _add_rules(self, rules, _legalese=common_license_words, _spdx_tokens=frozens
             rid_by_hash[rule_hash] = rid
             regular_rids_add(rid)
 
+            match_unknown.add_ngrams(
+                automaton=self.unknown_ngrams,
+                tids=rule_token_ids,
+                rule_length=rule.length,
+                unknown_ngram_length=_unknown_ngram_length,
+            )
             # Some rules cannot be matched as a sequence are "weak" rules
             if not is_weak:
                 approx_matchable_rids_add(rid)

src/licensedcode/match_unknown.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from licensedcode import tokenize
+"""
+Matching strategy for unknown matching using ngrams.
+"""
+
+# Set to False to enable debug tracing
+TRACE = False
+
+if TRACE:
+    import logging
+    import sys
+
+    logger = logging.getLogger(__name__)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
+
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+else:
+
+    def logger_debug(*args):
+        pass
+
+MATCH_UNKNOWN = '6-unknown'
+
+
+def add_ngrams(automaton, tids, rule_length, unknown_ngram_length=7):
+    """
+    Add the `tids` sequence of token ids to an unknown ngram automaton.
+    """
+    if rule_length < unknown_ngram_length:
+        return
+
+    rule_ngrams = tokenize.ngrams(tids, ngram_length=unknown_ngram_length)
+
+    for ngram in rule_ngrams:
+        ngram = tuple(ngram)
+        automaton.add_word(ngram, ngram)
+
+
+def unknown_match(idx, query_run, automaton, unknown_ngram_length=7, **kwargs):
+    """
+    Return a list of unknown LicenseMatch by matching the `query_run` against
+    the `automaton` and `idx` index.
+    """
+    matches = list(get_matches(
+        qtokens=query_run.tokens,
+        qbegin=query_run.start,
+        automaton=automaton,
+        unknown_ngram_length=unknown_ngram_length,
+    ))
+    return matches
+
+
+def get_matches(qtokens, qbegin, automaton, unknown_ngram_length=7):
+    """
+    Yield tuples of automaton matches positions as (match start, match end, match value) from
+    matching `qtokens` sequence of query token ids starting at the `qbegin` absolute
+    query start position position using the `automaton`.
+    """
+    # iterate over matched strings: the matched value is matching ngram
+    qtokens = tuple(qtokens)
+    for qend, matched_ngram in automaton.iter(qtokens):
+        qend = qbegin + qend + 1
+        qstart = qend - unknown_ngram_length
+        yield qstart, qend, matched_ngram