Allow colon ":" in license symbols #33

Otherwise some valid SPDX references are not parsed, such as:
DocumentRef-James-1.0:LicenseRef-Eric-2.0

Reported-by: @papadeltasierra
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/license_expression/__init__.py

@@ -846,7 +846,7 @@ def __contains__(self, other):
 
 
 # validate license keys
-is_valid_license_key = re.compile(r'^[-\w\s\.\+]+$', re.UNICODE).match
+is_valid_license_key = re.compile(r'^[-:\w\s\.\+]+$', re.UNICODE).match
 
 
 # TODO: we need to implement comparison by hand instead
@@ -882,14 +882,15 @@ def __init__(self, key, aliases=tuple(), is_exception=False, *args, **kwargs):
         if not is_valid_license_key(key):
             raise ExpressionError(
                 'Invalid license key: the valid characters are: letters and numbers, '
-                'underscore, dot or hyphen signs and spaces: "%(key)s"' % locals())
+                'underscore, dot, colon or hyphen signs and spaces: "%(key)s"' % locals())
 
         # normalize for spaces
         key = ' '.join(key.split())
 
         if key.lower() in KEYWORDS_STRINGS:
             raise ExpressionError(
-                'Invalid license key: a key cannot be a reserved keyword: "or", "and" or "with: "%(key)s"' % locals())
+                'Invalid license key: a key cannot be a reserved keyword: "or",'
+                ' "and" or "with: "%(key)s"' % locals())
 
         self.key = key
 

tests/test_license_expression.py

@@ -919,6 +919,14 @@ def test_parse_invalid_expression_with_single_leading_or_raise_exception(self):
             }
             assert expected == _parse_error_as_dict(pe)
 
+    def test_Licensing_can_parse_expressions_with_symbols_that_contain_a_colon(self):
+        licensing = Licensing()
+        expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0'
+
+        result = licensing.parse(expression)
+        expected = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0'
+        assert expected == result.render('{symbol.key}')
+
 
 class LicensingParseWithSymbolsSimpleTest(TestCase):
 
@@ -1752,6 +1760,52 @@ def test_simple_tokenizer(self):
         ]
         assert expected == results
 
+    def test_tokenize_can_handle_expressions_with_symbols_that_contain_a_colon(self):
+        licensing = Licensing()
+        expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0'
+
+        result = list(licensing.tokenize(expression))
+        expected = [
+            (LicenseSymbol(u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', is_exception=False),
+             u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', 0)
+        ]
+
+        assert expected == result
+
+    def test_tokenize_simple_can_handle_expressions_with_symbols_that_contain_a_colon(self):
+        licensing = Licensing()
+        expression = 'DocumentRef-James-1.0:LicenseRef-Eric-2.0'
+
+        result = list(licensing.tokenize(expression, simple=True))
+        expected = [
+            (LicenseSymbol(u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', is_exception=False),
+             u'DocumentRef-James-1.0:LicenseRef-Eric-2.0', 0)
+        ]
+
+        assert expected == result
+
+    def test_tokenize_can_handle_expressions_with_tabs_and_new_lines(self):
+        licensing = Licensing()
+        expression = 'this\t \tis \n\n an expression'
+        result = list(licensing.tokenize(expression, simple=False))
+        expected = [
+            (LicenseSymbol(u'this is an expression', is_exception=False),
+            u'this is an expression', 0)
+        ]
+        assert expected == result
+
+    def test_tokenize_simple_can_handle_expressions_with_tabs_and_new_lines(self):
+        licensing = Licensing()
+        expression = 'this\t \tis \n\n an expression'
+        result = list(licensing.tokenize(expression, simple=True))
+        expected = [
+            (LicenseSymbol(u'this', is_exception=False), u'this', 0),
+            (LicenseSymbol(u'is', is_exception=False), u'is', 7),
+            (LicenseSymbol(u'an', is_exception=False), u'an', 13),
+            (LicenseSymbol(u'expression', is_exception=False), u'expression', 16)
+        ]
+        assert expected == result
+
     def test_tokenize_step_by_step_does_not_munge_trailing_symbols(self):
         gpl2 = LicenseSymbol(key='GPL-2.0')
         gpl2plus = LicenseSymbol(key='GPL-2.0 or LATER')