Improve licenses detection accuracy of unknowns using ngrams

Signed-off-by: akugarg <akanksha.garg2k@gmail.com>

Author: akugarg

src/licensedcode/index.py

@@ -19,7 +19,8 @@
 
 from intbitset import intbitset
 
-from licensedcode import SMALL_RULE, match_unknown
+from licensedcode import SMALL_RULE
+from licensedcode import match_unknown
 from licensedcode.legalese import common_license_words
 from licensedcode import match
 from licensedcode import match_aho
@@ -32,6 +33,7 @@
 from licensedcode.seq import match_blocks as match_blocks_seq
 from licensedcode import query
 from licensedcode import tokenize
+from licensedcode.spans import Span
 
 """
 Main license index construction, query processing and matching entry points for
@@ -883,6 +885,25 @@ def match(
             # break if deadline has passed
             if time() > deadline:
                 break
+        
+        # refining matches without filtering false positives
+        matches, _discarded = match.refine_matches(
+            matches=matches,
+            idx=self,
+            query=qry,
+            min_score=min_score,
+            filter_false_positive=False,
+            merge=True,
+        )
+
+        original_qspan = Span(0, len(qry.tokens)-1)
+        matched_qspans = [m.qspan for m in matches]
+        matched_qspan = Span()
+        matched_qspan.union(*matched_qspans)
+        unmatched_qspan = original_qspan.difference(matched_qspan)
+        
+        for subspan in unmatched_qspan.subspans():
+            query_run = query.QueryRun(query=qry, start=subspan.start, end=subspan.end)
 
         if not matches:
             return []

src/licensedcode/match_unknown.py

@@ -8,6 +8,9 @@
 #
 
 from licensedcode import tokenize
+from licensedcode.match import LicenseMatch
+from licensedcode.models import UnknownRule
+from licensedcode.spans import Span
 """
 Matching strategy for unknown matching using ngrams.
 """
@@ -49,17 +52,23 @@ def add_ngrams(automaton, tids, rule_length, unknown_ngram_length=7):
         automaton.add_word(ngram, ngram)
 
 
-def unknown_match(idx, query_run, automaton, unknown_ngram_length=7, **kwargs):
+def match_unknowns(idx, query_run, automaton, unknown_ngram_length=7, **kwargs):
     """
     Return a list of unknown LicenseMatch by matching the `query_run` against
     the `automaton` and `idx` index.
     """
-    matches = list(get_matches(
+    matches = get_matches(
         qtokens=query_run.tokens,
         qbegin=query_run.start,
         automaton=automaton,
         unknown_ngram_length=unknown_ngram_length,
-    ))
+    )
+
+    qspans = (Span(qstart, qend) for qstart, qend, matched_ngram in matches)
+    qspan = Span().union(*qspans)
+    ispan = Span(0, len(qspan))
+    rule = UnknownRule()
+
     return matches
 
 

src/licensedcode/models.py

@@ -1473,6 +1473,36 @@ def dump(self):
         raise NotImplementedError
 
 
+@attr.s(slots=True, repr=False)
+class UnknownRule(Rule):
+    """
+    A specialized rule object that is used for the special case of unknown license
+    detection.
+    Since we may have an infinite possible number of unknown licenses and these
+    are not backed by a traditional rule text file, we use this class to handle
+    the specifics of these how rules are built at matching time: one rule
+    is created for each detected unknown license.
+    """
+
+    def __attrs_post_init__(self, *args, **kwargs):
+        self.identifier = f'unknown-license-identifier: ' 
+        self.license_expression = 'unknown-license'
+        expression = self.licensing.parse(self.license_expression)
+
+        self.is_unknown = True
+        self.license_expression_object = expression
+        self.is_license_notice = True
+        self.is_small = False
+        self.relevance = 100
+        self.has_stored_relevance = True
+
+    def load(self):
+        raise NotImplementedError
+
+    def dump(self):
+        raise NotImplementedError
+
+
 def _print_rule_stats():
     """
     Print rules statistics.