Improve licenses detection accuracy of unknowns using ngrams

Signed-off-by: akugarg <akanksha.garg2k@gmail.com>

Author: akugarg

src/licensedcode/index.py

@@ -19,7 +19,8 @@
 
 from intbitset import intbitset
 
-from licensedcode import SMALL_RULE, match_unknown
+from licensedcode import SMALL_RULE
+from licensedcode import match_unknown
 from licensedcode.legalese import common_license_words
 from licensedcode import match
 from licensedcode import match_aho
@@ -32,6 +33,7 @@
 from licensedcode.seq import match_blocks as match_blocks_seq
 from licensedcode import query
 from licensedcode import tokenize
+from licensedcode.spans import Span
 
 """
 Main license index construction, query processing and matching entry points for
@@ -883,6 +885,25 @@ def match(
             # break if deadline has passed
             if time() > deadline:
                 break
+        
+        # refining matches without filtering false positives
+        matches, _discarded = match.refine_matches(
+            matches=matches,
+            idx=self,
+            query=qry,
+            min_score=min_score,
+            filter_false_positive=False,
+            merge=True,
+        )
+
+        original_qspan = Span(0, len(qry.tokens)-1)
+        matched_qspans = [m.qspan for m in matches]
+        matched_qspan = Span()
+        matched_qspan.union(*matched_qspans)
+        unmatched_qspan = original_qspan.difference(matched_qspan)
+        
+        for subspan in unmatched_qspan.subspans():
+            query_run = query.QueryRun(query=qry, start=subspan.start, end=subspan.end)
 
         if not matches:
             return []

src/licensedcode/match_unknown.py

@@ -49,7 +49,7 @@ def add_ngrams(automaton, tids, rule_length, unknown_ngram_length=7):
         automaton.add_word(ngram, ngram)
 
 
-def unknown_match(idx, query_run, automaton, unknown_ngram_length=7, **kwargs):
+def match_unknowns(idx, query_run, automaton, unknown_ngram_length=7, **kwargs):
     """
     Return a list of unknown LicenseMatch by matching the `query_run` against
     the `automaton` and `idx` index.