Add simple tokenizer and improve error checks #29

Also update boolean.py to 3.6

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

setup.py

@@ -30,7 +30,7 @@
     include_package_data=True,
     zip_safe=False,
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
         'Intended Audience :: Developers',
         'Operating System :: OS Independent',
@@ -48,6 +48,6 @@
         'licence'
     ],
     install_requires=[
-        'boolean.py >= 3.5, < 4.0.0',
+        'boolean.py >= 3.6, < 4.0.0',
     ]
 )

src/license_expression/__init__.py

@@ -52,9 +52,11 @@
 from boolean.boolean import PARSE_ERRORS
 from boolean.boolean import PARSE_INVALID_EXPRESSION
 from boolean.boolean import PARSE_INVALID_NESTING
+from boolean.boolean import PARSE_INVALID_OPERATOR_SEQUENCE
 from boolean.boolean import PARSE_INVALID_SYMBOL_SEQUENCE
 from boolean.boolean import PARSE_UNBALANCED_CLOSING_PARENS
 from boolean.boolean import PARSE_UNKNOWN_TOKEN
+
 from boolean.boolean import ParseError
 from boolean.boolean import TOKEN_SYMBOL
 from boolean.boolean import TOKEN_AND
@@ -239,10 +241,10 @@ def __init__(self, symbols=tuple(), quiet=True):
                 raise ValueError('\n'.join(warns + errors))
 
         # mapping of known symbol key to symbol for reference
-        self.known_symbols_by_key = {symbol.key: symbol for symbol in symbols}
+        self.known_symbols = {symbol.key: symbol for symbol in symbols}
 
         # mapping of known symbol lowercase key to symbol for reference
-        self.known_symbols_by_keylow = {symbol.key.lower(): symbol for symbol in symbols}
+        self.known_symbols_lowercase = {symbol.key.lower(): symbol for symbol in symbols}
 
         # Aho-Corasick automaton-based Advanced Tokenizer
         self.advanced_tokenizer = None
@@ -375,7 +377,7 @@ def unknown_license_symbols(self, expression, unique=True, **kwargs):
         Extra kwargs are passed down to the parse() function.
         """
         return [ls for ls in self.license_symbols(expression, unique=unique, decompose=True, **kwargs)
-                if not ls.key in self.known_symbols_by_key]
+                if not ls.key in self.known_symbols]
 
     def unknown_license_keys(self, expression, unique=True, **kwargs):
         """
@@ -447,7 +449,7 @@ def parse(self, expression, validate=False, strict=False, simple=False, **kwargs
             return
         try:
             # this will raise a ParseError on errors
-            tokens = list(self.tokenize(expression, strict=strict))
+            tokens = list(self.tokenize(expression, strict=strict, simple=simple))
             expression = super(Licensing, self).parse(tokens)
         except TypeError as e:
             msg = 'Invalid expression syntax: ' + repr(e)
@@ -557,9 +559,9 @@ def get_advanced_tokenizer(self):
         for keyword in KEYWORDS:
             add_item(keyword.value, keyword)
 
-        # self.known_symbols_by_key has been created at Licensing initialization time and is
+        # self.known_symbols has been created at Licensing initialization time and is
         # already validated and trusted here
-        for key, symbol in self.known_symbols_by_key.items():
+        for key, symbol in self.known_symbols.items():
             # always use the key even if there are no aliases.
             add_item(key, symbol)
             aliases = getattr(symbol, 'aliases', [])
@@ -594,7 +596,7 @@ def simple_tokenizer(self, expression):
         tokenizing expressions.
         """
 
-        symbols = self.known_symbols_by_keylow or {}
+        symbols = self.known_symbols_lowercase or {}
 
         for match in _simple_tokenizer(expression):
             if not match:

tests/test_license_expression.py

@@ -20,6 +20,7 @@
 from collections import namedtuple
 from collections import OrderedDict
 from unittest import TestCase
+from unittest.case import expectedFailure
 import sys
 
 from boolean.boolean import PARSE_UNBALANCED_CLOSING_PARENS
@@ -29,6 +30,7 @@
 from license_expression import PARSE_INVALID_NESTING
 from license_expression import PARSE_INVALID_EXCEPTION
 from license_expression import PARSE_INVALID_SYMBOL_AS_EXCEPTION
+from license_expression import PARSE_INVALID_OPERATOR_SEQUENCE
 
 from license_expression import ExpressionError
 from license_expression import Keyword
@@ -424,8 +426,15 @@ def test_parse_invalid_expression_raise_exception6(self):
         try:
             licensing.parse(expr)
             self.fail("Exception not raised when validating '%s'" % expr)
-        except ExpressionError as ee:
-            assert 'OR requires two or more licenses as in: MIT OR BSD' == str(ee)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'OR',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_not_invalid_expression_raise_no_exception2(self):
         licensing = Licensing()
@@ -466,18 +475,28 @@ def test_parse_errors_catch_invalid_expression_with_bare_and(self):
         try:
             licensing.parse('and')
             self.fail('Exception not raised')
-        except ExpressionError as pe:
-            expected = 'AND requires two or more licenses as in: MIT AND BSD'
-            assert expected == str(pe)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'and',
+                'token_type': TOKEN_AND
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_errors_catch_invalid_expression_with_or_and_no_other(self):
         licensing = Licensing()
         try:
             licensing.parse('or that')
             self.fail('Exception not raised')
-        except ExpressionError as pe:
-            expected = 'OR requires two or more licenses as in: MIT OR BSD'
-            assert expected == str(pe)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_errors_catch_invalid_expression_with_empty_parens(self):
         licensing = Licensing()
@@ -828,6 +847,78 @@ def test_Licensing_can_parse_valid_expressions_with_symbols_that_contain_spaces(
         expected = 'GPL-2.0 OR (mit AND LGPL 2.1) OR bsd OR GPL-2.0 OR (mit AND LGPL 2.1)'
         assert expected == str(parsed)
 
+    def test_parse_invalid_expression_with_trailing_or(self):
+        licensing = Licensing()
+        expr = 'mit or'
+        try:
+            licensing.parse(expr)
+            self.fail("Exception not raised when validating '%s'" % expr)
+        except ExpressionError as ee:
+            assert 'OR requires two or more licenses as in: MIT OR BSD' == str(ee)
+
+    def test_parse_invalid_expression_with_trailing_or_and_valid_start_does_not_raise_exception(self):
+        licensing = Licensing()
+        expression = ' mit or mit or '
+        parsed = licensing.parse(expression)
+        # ExpressionError: OR requires two or more licenses as in: MIT OR BSD
+        expected = 'mit OR mit'
+        assert expected == str(parsed)
+
+    def test_parse_invalid_expression_with_repeated_trailing_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit or or'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 14,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
+    @expectedFailure
+    def test_parse_invalid_expression_with_single_trailing_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit or'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 14,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
+    def test_parse_invalid_expression_with_single_trailing_and_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit and'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ExpressionError as ee:
+            assert 'AND requires two or more licenses as in: MIT AND BSD' == str(ee)
+
+    def test_parse_invalid_expression_with_single_leading_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'or mit or mit'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
 
 class LicensingParseWithSymbolsSimpleTest(TestCase):
 
@@ -1950,17 +2041,29 @@ def test_and_and_or_is_invalid(self):
         try:
             licensing.parse(expression)
             self.fail('Exception not raised')
-        except ExpressionError as e:
-            assert 'AND requires two or more licenses as in: MIT AND BSD' == str(e)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 27,
+                'token_string': 'and',
+                'token_type': TOKEN_AND}
+            assert expected == _parse_error_as_dict(pe)
 
-    def test_or_or_is_not_invalid(self):
+    def test_or_or_is_invalid(self):
         expression = 'gpl-2.0 with classpath or or or or gpl-2.0-plus'
         licensing = Licensing()
-        result = str(licensing.parse(expression))
-        assert 'gpl-2.0 WITH classpath OR gpl-2.0-plus' == result
+        try:
+            licensing.parse(expression)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 26,
+                'token_string': 'or',
+                'token_type': TOKEN_OR}
+            assert expected == _parse_error_as_dict(pe)
 
     def test_tokenize_or_or(self):
-        expression = 'gpl-2.0 with classpath or or gpl-2.0-plus'
+        expression = 'gpl-2.0 with classpath or or or gpl-2.0-plus'
         licensing = Licensing()
         results = list(licensing.tokenize(expression))
         expected = [
@@ -1969,7 +2072,8 @@ def test_tokenize_or_or(self):
                 exception_symbol=LicenseSymbol(u'classpath')), 'gpl-2.0 with classpath', 0),
             (2, 'or', 23),
             (2, 'or', 26),
-            (LicenseSymbol(u'gpl-2.0-plus'), 'gpl-2.0-plus', 29)
+            (2, 'or', 29),
+            (LicenseSymbol(u'gpl-2.0-plus'), 'gpl-2.0-plus', 32)
         ]
 
         assert expected == results

thirdparty/prod/boolean.py-3.5-py2.py3-none-any.whl

@@ -1,5 +1,5 @@
-about_resource: boolean.py-3.5-py2.py3-none-any.whl
-version: 3.5
+about_resource: boolean.py-3.6-py2.py3-none-any.whl
+version: 3.6
 download_url: https://pypi.python.org/packages/80/f3/0508ae7ba76b02f7fd666b705766edc1863fc8ef29d0519b4c95d60ab1bb/boolean.py-3.5-py2.py3-none-any.whl#md5=cf90b0c0530663bbf71a53fb58f6fa72
 
 name: boolean.py