Add simple tokenizer and improve error checks #29

Also update boolean.py to 3.6

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

setup.py

@@ -30,7 +30,7 @@
     include_package_data=True,
     zip_safe=False,
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
         'Intended Audience :: Developers',
         'Operating System :: OS Independent',
@@ -48,6 +48,6 @@
         'licence'
     ],
     install_requires=[
-        'boolean.py >= 3.5, < 4.0.0',
+        'boolean.py >= 3.6, < 4.0.0',
     ]
 )

src/license_expression/__init__.py

@@ -39,8 +39,6 @@
 from copy import deepcopy
 from functools import total_ordering
 import itertools
-import logging
-from pprint import pprint
 import re
 import string
 
@@ -52,9 +50,11 @@
 from boolean.boolean import PARSE_ERRORS
 from boolean.boolean import PARSE_INVALID_EXPRESSION
 from boolean.boolean import PARSE_INVALID_NESTING
+from boolean.boolean import PARSE_INVALID_OPERATOR_SEQUENCE
 from boolean.boolean import PARSE_INVALID_SYMBOL_SEQUENCE
 from boolean.boolean import PARSE_UNBALANCED_CLOSING_PARENS
 from boolean.boolean import PARSE_UNKNOWN_TOKEN
+
 from boolean.boolean import ParseError
 from boolean.boolean import TOKEN_SYMBOL
 from boolean.boolean import TOKEN_AND
@@ -65,6 +65,7 @@
 from license_expression._pyahocorasick import Trie as AdvancedTokenizer
 from license_expression._pyahocorasick import Token
 
+
 # Python 2 and 3 support
 try:
     # Python 2
@@ -74,23 +75,6 @@
     # Python 3
     unicode = str  # NOQA
 
-TRACE = False
-
-logger = logging.getLogger(__name__)
-
-
-def logger_debug(*args):
-    pass
-
-
-if TRACE:
-
-    def logger_debug(*args):
-        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
-
-    import sys
-    logging.basicConfig(stream=sys.stdout)
-    logger.setLevel(logging.DEBUG)
 
 # append new error codes to PARSE_ERRORS by monkey patching
 PARSE_EXPRESSION_NOT_UNICODE = 100
@@ -239,10 +223,10 @@ def __init__(self, symbols=tuple(), quiet=True):
                 raise ValueError('\n'.join(warns + errors))
 
         # mapping of known symbol key to symbol for reference
-        self.known_symbols_by_key = {symbol.key: symbol for symbol in symbols}
+        self.known_symbols = {symbol.key: symbol for symbol in symbols}
 
         # mapping of known symbol lowercase key to symbol for reference
-        self.known_symbols_by_keylow = {symbol.key.lower(): symbol for symbol in symbols}
+        self.known_symbols_lowercase = {symbol.key.lower(): symbol for symbol in symbols}
 
         # Aho-Corasick automaton-based Advanced Tokenizer
         self.advanced_tokenizer = None
@@ -375,7 +359,7 @@ def unknown_license_symbols(self, expression, unique=True, **kwargs):
         Extra kwargs are passed down to the parse() function.
         """
         return [ls for ls in self.license_symbols(expression, unique=unique, decompose=True, **kwargs)
-                if not ls.key in self.known_symbols_by_key]
+                if not ls.key in self.known_symbols]
 
     def unknown_license_keys(self, expression, unique=True, **kwargs):
         """
@@ -447,7 +431,7 @@ def parse(self, expression, validate=False, strict=False, simple=False, **kwargs
             return
         try:
             # this will raise a ParseError on errors
-            tokens = list(self.tokenize(expression, strict=strict))
+            tokens = list(self.tokenize(expression, strict=strict, simple=simple))
             expression = super(Licensing, self).parse(tokens)
         except TypeError as e:
             msg = 'Invalid expression syntax: ' + repr(e)
@@ -488,40 +472,20 @@ def tokenize(self, expression, strict=False, simple=False):
             raise ParseError(error_code=PARSE_EXPRESSION_NOT_UNICODE)
 
         if simple:
-            if TRACE: logger_debug('using simple tokenizer')
             tokens = self.simple_tokenizer(expression)
         else:
-            if TRACE: logger_debug('using advanced tokenizer')
             advanced_tokenizer = self.get_advanced_tokenizer()
             tokens = advanced_tokenizer.tokenize(expression)
 
-        if TRACE:
-            tokens = list(tokens)
-            logger_debug('tokenize: tokens')
-            pprint(tokens)
-
         # Assign symbol for unknown tokens
         tokens = build_symbols_from_unknown_tokens(tokens)
-        if TRACE:
-            tokens = list(tokens)
-            logger_debug('tokenize: token with symbols')
-            pprint(tokens)
 
         # skip whitespace-only tokens
         tokens = (t for t in tokens if t.string and t.string.strip())
-        if TRACE:
-            tokens = list(tokens)
-            logger_debug('tokenize: token NO spaces')
-            pprint(tokens)
 
         # create atomic LicenseWithExceptionSymbol from WITH subexpressions
         tokens = replace_with_subexpression_by_license_symbol(tokens, strict)
 
-        if TRACE:
-            tokens = list(tokens)
-            logger_debug('tokenize: LicenseWithExceptionSymbol replaced')
-            pprint(tokens)
-
         # finally yield the actual args expected by the boolean parser
         for token in tokens:
             pos = token.start
@@ -557,9 +521,9 @@ def get_advanced_tokenizer(self):
         for keyword in KEYWORDS:
             add_item(keyword.value, keyword)
 
-        # self.known_symbols_by_key has been created at Licensing initialization time and is
+        # self.known_symbols has been created at Licensing initialization time and is
         # already validated and trusted here
-        for key, symbol in self.known_symbols_by_key.items():
+        for key, symbol in self.known_symbols.items():
             # always use the key even if there are no aliases.
             add_item(key, symbol)
             aliases = getattr(symbol, 'aliases', [])
@@ -594,7 +558,7 @@ def simple_tokenizer(self, expression):
         tokenizing expressions.
         """
 
-        symbols = self.known_symbols_by_keylow or {}
+        symbols = self.known_symbols_lowercase or {}
 
         for match in _simple_tokenizer(expression):
             if not match:

tests/test_license_expression.py

@@ -20,6 +20,7 @@
 from collections import namedtuple
 from collections import OrderedDict
 from unittest import TestCase
+from unittest.case import expectedFailure
 import sys
 
 from boolean.boolean import PARSE_UNBALANCED_CLOSING_PARENS
@@ -29,6 +30,7 @@
 from license_expression import PARSE_INVALID_NESTING
 from license_expression import PARSE_INVALID_EXCEPTION
 from license_expression import PARSE_INVALID_SYMBOL_AS_EXCEPTION
+from license_expression import PARSE_INVALID_OPERATOR_SEQUENCE
 
 from license_expression import ExpressionError
 from license_expression import Keyword
@@ -424,8 +426,15 @@ def test_parse_invalid_expression_raise_exception6(self):
         try:
             licensing.parse(expr)
             self.fail("Exception not raised when validating '%s'" % expr)
-        except ExpressionError as ee:
-            assert 'OR requires two or more licenses as in: MIT OR BSD' == str(ee)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'OR',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_not_invalid_expression_raise_no_exception2(self):
         licensing = Licensing()
@@ -466,18 +475,28 @@ def test_parse_errors_catch_invalid_expression_with_bare_and(self):
         try:
             licensing.parse('and')
             self.fail('Exception not raised')
-        except ExpressionError as pe:
-            expected = 'AND requires two or more licenses as in: MIT AND BSD'
-            assert expected == str(pe)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'and',
+                'token_type': TOKEN_AND
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_errors_catch_invalid_expression_with_or_and_no_other(self):
         licensing = Licensing()
         try:
             licensing.parse('or that')
             self.fail('Exception not raised')
-        except ExpressionError as pe:
-            expected = 'OR requires two or more licenses as in: MIT OR BSD'
-            assert expected == str(pe)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
 
     def test_parse_errors_catch_invalid_expression_with_empty_parens(self):
         licensing = Licensing()
@@ -828,6 +847,78 @@ def test_Licensing_can_parse_valid_expressions_with_symbols_that_contain_spaces(
         expected = 'GPL-2.0 OR (mit AND LGPL 2.1) OR bsd OR GPL-2.0 OR (mit AND LGPL 2.1)'
         assert expected == str(parsed)
 
+    def test_parse_invalid_expression_with_trailing_or(self):
+        licensing = Licensing()
+        expr = 'mit or'
+        try:
+            licensing.parse(expr)
+            self.fail("Exception not raised when validating '%s'" % expr)
+        except ExpressionError as ee:
+            assert 'OR requires two or more licenses as in: MIT OR BSD' == str(ee)
+
+    def test_parse_invalid_expression_with_trailing_or_and_valid_start_does_not_raise_exception(self):
+        licensing = Licensing()
+        expression = ' mit or mit or '
+        parsed = licensing.parse(expression)
+        # ExpressionError: OR requires two or more licenses as in: MIT OR BSD
+        expected = 'mit OR mit'
+        assert expected == str(parsed)
+
+    def test_parse_invalid_expression_with_repeated_trailing_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit or or'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 14,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
+    @expectedFailure
+    def test_parse_invalid_expression_with_single_trailing_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit or'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 14,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
+    def test_parse_invalid_expression_with_single_trailing_and_raise_exception(self):
+        licensing = Licensing()
+        expression = 'mit or mit and'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ExpressionError as ee:
+            assert 'AND requires two or more licenses as in: MIT AND BSD' == str(ee)
+
+    def test_parse_invalid_expression_with_single_leading_or_raise_exception(self):
+        licensing = Licensing()
+        expression = 'or mit or mit'
+        try:
+            licensing.parse(expression, simple=False)
+            self.fail('Exception not raised')
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 0,
+                'token_string': 'or',
+                'token_type': TOKEN_OR
+            }
+            assert expected == _parse_error_as_dict(pe)
+
 
 class LicensingParseWithSymbolsSimpleTest(TestCase):
 
@@ -1950,17 +2041,29 @@ def test_and_and_or_is_invalid(self):
         try:
             licensing.parse(expression)
             self.fail('Exception not raised')
-        except ExpressionError as e:
-            assert 'AND requires two or more licenses as in: MIT AND BSD' == str(e)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 27,
+                'token_string': 'and',
+                'token_type': TOKEN_AND}
+            assert expected == _parse_error_as_dict(pe)
 
-    def test_or_or_is_not_invalid(self):
+    def test_or_or_is_invalid(self):
         expression = 'gpl-2.0 with classpath or or or or gpl-2.0-plus'
         licensing = Licensing()
-        result = str(licensing.parse(expression))
-        assert 'gpl-2.0 WITH classpath OR gpl-2.0-plus' == result
+        try:
+            licensing.parse(expression)
+        except ParseError as pe:
+            expected = {
+                'error_code': PARSE_INVALID_OPERATOR_SEQUENCE,
+                'position': 26,
+                'token_string': 'or',
+                'token_type': TOKEN_OR}
+            assert expected == _parse_error_as_dict(pe)
 
     def test_tokenize_or_or(self):
-        expression = 'gpl-2.0 with classpath or or gpl-2.0-plus'
+        expression = 'gpl-2.0 with classpath or or or gpl-2.0-plus'
         licensing = Licensing()
         results = list(licensing.tokenize(expression))
         expected = [
@@ -1969,7 +2072,8 @@ def test_tokenize_or_or(self):
                 exception_symbol=LicenseSymbol(u'classpath')), 'gpl-2.0 with classpath', 0),
             (2, 'or', 23),
             (2, 'or', 26),
-            (LicenseSymbol(u'gpl-2.0-plus'), 'gpl-2.0-plus', 29)
+            (2, 'or', 29),
+            (LicenseSymbol(u'gpl-2.0-plus'), 'gpl-2.0-plus', 32)
         ]
 
         assert expected == results

thirdparty/prod/boolean.py-3.5-py2.py3-none-any.whl

@@ -0,0 +1,18 @@
+about_resource: boolean.py-3.6-py2.py3-none-any.whl
+attribute: true
+checksum_md5: da39999eb131b589e84ad935dc4ca642
+checksum_sha1: d31b55e7ad2ee917232b3213afe3ae9678156a9f
+copyright: Copyright (c) 2009-2016 Sebastian Kraemer, basti.kr@gmail.com and others
+description: Implements boolean algebra in one module.
+download_url: https://files.pythonhosted.org/packages/9b/27/d22062a221010e17935237ba4b574cd828238ea02e0765337c238466a512/boolean.py-3.6-py2.py3-none-any.whl
+homepage_url: https://github.com/bastikr/boolean.py
+license_expression: bsd-simplified
+licenses:
+-   file: bsd-simplified.LICENSE
+    key: bsd-simplified
+    name: BSD-2-Clause
+name: boolean.py
+notice_file: boolean.py-3.6-py2.py3-none-any.whl.NOTICE
+notice_url: https://github.com/bastikr/boolean.py/blob/master/LICENSE.txt
+owner: Sebastian Kraemer
+version: '3.6'