Add unit tests to key_phrase_tokenizer

Signed-off-by: Mike Rombout <mike.rombout@elastisys.com>

Author: mrombout

src/licensedcode/tokenize.py

@@ -69,13 +69,17 @@ def query_lines(location=None, query_string=None, strip=True, start_line=1):
 KEY_PHRASE_CLOSE = "}}"
 
 def key_phrase_tokenizer(text, stopwords=STOPWORDS):
+    """
+    Return an iterable of tokens from a unicode query test. It must behave identically as the `index_tokenizer` with the
+    exception that it returns KEY_PHRASE_OPEN and KEY_PHRASE_CLOSE as separate tokens so that they can be used to parse
+    key phrases.
+    """
     if not text:
         return []
     words = key_phrase_splitter(text.lower())
 
     new_words = []
     for word in words:
-        # TODO: Optimize for readability
         if word.startswith(KEY_PHRASE_OPEN):
             new_words.append(KEY_PHRASE_OPEN)
 

tests/licensedcode/data/tokenize/htmlish.html

@@ -93,7 +93,7 @@
 	</div>
 <div class="main-cont">
 	<div class="main-text-cont">
-		<h1>Third party services</h1>
+		<h1>{{Third party services}}</h1>
 		<p> 
 			Of course we are all for responsible disclosure, which you might have seen if you have read our blog. So here we have listed all the third party services we use in different ways in alphabetical order and an added a short comment on why or how we use them.
 		</p>