Fix unknown license detection #3343

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/licensedcode/match_unknown.py

@@ -23,6 +23,7 @@
 TRACE = False
 
 if TRACE:
+    use_print = True
     import logging
     import sys
 
@@ -31,6 +32,9 @@
     def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
+    if use_print:
+        logger_debug = print
+
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
@@ -142,22 +146,25 @@ def match_unknowns(
         unknown_ngram_length=unknown_ngram_length,
     )
 
+    # build match from merged matched ngrams
+    qspans = (Span(qstart, qend) for qstart, qend in matched_ngrams)
+    qspan = Span().union(*qspans)
+
     if TRACE:
         tokens_by_tid = idx.tokens_by_tid
 
         def get_tokens(_toks):
             return (' '.join(tokens_by_tid[t] for t in _toks))
 
         print('match_unknowns: matched_ngrams')
-        for qstart, qend, matched_toks in matched_ngrams:
+
+        for qstart, qend in matched_ngrams:
+            _span = Span(qstart, qend)
+            _tokens = [query_tokens[qpos] for qpos in _span]
             print(
                 '   ', 'qstart', qstart,
                 'qend', qend,
-                'matched_toks', get_tokens(matched_toks))
-
-    # build match from merged matched ngrams
-    qspans = (Span(qstart, qend) for qstart, qend in matched_ngrams)
-    qspan = Span().union(*qspans)
+                'matched_toks', get_tokens(_tokens))
 
     if not qspan:
         return
@@ -169,7 +176,8 @@ def get_tokens(_toks):
     match_len = len(qspan)
 
     if TRACE:
-        print('match_unknowns: matched_span:', get_tokens(matched_tokens))
+        #print('match_unknowns: matched_span:', get_tokens(matched_tokens))
+        print('match_unknowns: qspan, match_len, matched_span:', qspan, match_len, matched_tokens)
 
     # we use the query side to build the ispans
     ispan = Span(0, match_len)
@@ -180,9 +188,8 @@ def get_tokens(_toks):
     try:
         match_start_line = line_by_pos[qspan.start]
         match_end_line = line_by_pos[qspan.end]
-    except:
-        print('empty span:', qspan)
-        raise
+    except Exception as e:
+        raise Exception('empty span:', qspan) from e
 
     text = ''.join(get_full_qspan_matched_text(
         match_qspan=qspan,

src/licensedcode/models.py

@@ -2542,14 +2542,12 @@ def __attrs_post_init__(self, *args, **kwargs):
         self.identifier = f'license-detection-unknown-{self._unique_id}'
 
         self.license_expression = UNKNOWN_LICENSE_KEY
-        # note that this could be shared across rules as an optimization
+        #TODO: that this could be shared across rules as an optimization
         self.license_expression_object = self.licensing.parse(UNKNOWN_LICENSE_KEY)
         self.is_license_notice = True
         self.notes = 'Unknown license based on a composite of license words.'
         self.is_synthetic = True
         self.setup()
-        # called only for it's side effects
-        self.tokens()
 
 
 @attr.s(slots=True, repr=False)