Detect copyright from damaged PDF

Signed-off-by: Philippe Ombredanne <pombredanne@aboutcode.org>

Author: pombredanne

src/cluecode/copyrights.py

@@ -61,7 +61,6 @@ def logger_debug(*args):
 if TRACE_DEEP:
     logger_debug = print
 
-
 """
 Detect and collect copyright statements.
 
@@ -427,9 +426,9 @@ def get_tokens(numbered_lines, splitter=re.compile(r'[\t =;]+').split):
 
         if TRACE_TOK:
             logger_debug('  get_tokens: preped line: ' + repr(line))
-        
+
         # when there is a an openning parens in the middle of a word, add a
-        # space before in some cases: exclude (digit , (s , (c , (- 
+        # space before in some cases: exclude (digit , (s , (c , (-
         line = re.sub(pattern=r'(\([^rsc\-\d])', repl=' \g<1>', string=line, flags=re.IGNORECASE)
         for tok in splitter(line):
             # strip trailing quotes+comma
@@ -733,8 +732,8 @@ def build_detection_from_node(
     (r'^[Ss][Pp][Dd][Xx]-[Ff]ile[Cc]ontributor', 'SPDX-CONTRIB'),
 
     # damaged PDF to text conversion with (cid:13) and rarer (cid:2)
-    (r'^[Cc]?\(cid:13\)$','COPY'),
-    (r'^[Cc]?\(cid:2\)$','COPY'),
+    (r'^[Cc]?\(cid:13\)$', 'COPY'),
+    (r'^[Cc]?\(cid:2\)$', 'COPY'),
 
     ############################################################################
     # ALL Rights Reserved.
@@ -821,6 +820,9 @@ def build_detection_from_node(
     (r'^W3C\(r\)$', 'COMP'),
     (r'^TeX$', 'NNP'),
 
+    # Copyright (c) 1989 ITAA, formerly ADAPSO
+    (r'^formerly', 'NNP'),
+
     # Three or more AsCamelCase GetQueueReference, with some exceptions
     (r'^(?:OpenStreetMap|AliasDotCom|AllThingsTalk).?$', 'NAME'),
 
@@ -1240,6 +1242,9 @@ def build_detection_from_node(
     # owlocationNameEntitieship.
     (r"^([a-z]{2,}[A-Z]){2,}[a-z]+[\.,]?", 'JUNK'),
 
+    # lower case with (s)
+    # but not owner/author(s).
+    (r"^owner/author\(s\)\.?$", 'NNP'),
     (r"^[a-z].+\(s\)[\.,]?$", 'JUNK'),
 
     # parens in the middle: for(var
@@ -1292,7 +1297,7 @@ def build_detection_from_node(
     (r'^[Bb]oth$', 'JUNK'),
     (r'^[Cc]aller$', 'JUNK'),
 
-    #'!(r)'
+    # '!(r)'
     (r'^!\(r\)$', 'JUNK'),
 
     # vars with curly braces
@@ -2248,6 +2253,7 @@ def build_detection_from_node(
     (r"^[a-z]'[A-Z]?[a-z]+[,\.]?$", 'NNP'),
 
     # exceptions to all CAPS words
+    (r'^ISBN$', 'NN'),
     (r'^[A-Z]{3,4}[0-9]{4},?$', 'NN'),
 
     # exceptions to CAPS used in obfuscated emails like in joe AT foo DOT com
@@ -3091,8 +3097,13 @@ def build_detection_from_node(
     # Author: Jeff LaBundy <jeff@labundy.com>
     COPYRIGHT: {<COPY>  <COPY>  <YR-RANGE>  <AUTH>  <NAME-EMAIL>} #2280-3
 
+    # Common for ACM notices
+    # Copyright is held by the owner/author(s).
+    # (c) 2016 Copyright held by the owner/author(s).
+    COPYRIGHT2: {<COPY>+ <YR-RANGE>+ <COPY><IS>?<HELD><BY><NN>?<NNP>}        #2280-9
 
     COPYRIGHT2: {<COPY>+ <NN|CAPS>? <YR-RANGE>+ <PN>*}        #2280
+    COPYRIGHT2: {<COPY><IS>?<HELD><BY><NN>?<NNP>}        #2280-12
 
     COPYRIGHT: {<COPYRIGHT2>  <BY>  <NAME-YEAR|NAME-EMAIL> <BY>?  <NAME-YEAR|NAME-EMAIL>? } #2280-4
 
@@ -3122,7 +3133,8 @@ def build_detection_from_node(
     # Rare form Copyright (c) 2008 All rights reserved by Amalasoft Corporation.
     COPYRIGHT: {<COPYRIGHT2> <ALLRIGHTRESERVED> <BY> <COMPANY>}        #2861
 
-    # Copyright (c) 1996 Adrian Rodriguez (adrian@franklins-tower.rutgers.edu) Laboratory for Computer Science Research Computing Facility
+    # Copyright (c) 1996 Adrian Rodriguez (adrian@franklins-tower.rutgers.edu) L
+    # aboratory for Computer Science Research Computing Facility
     COPYRIGHT: {<COPYRIGHT> <NAME>} #2400
 
     # copyrights in the style of Scilab/INRIA
@@ -4029,10 +4041,6 @@ def remove_dupe_copyright_words(c):
     c = c.replace('and later', ' ')
     c = c.replace('build.year', ' ')
 
-    # PDF to text damages
-    c = c.replace('(cid:13)', '(c)')
-    c = c.replace('(cid:2)', '(c)')
-    
     return c
 
 
@@ -4565,6 +4573,10 @@ def prepare_text_line(line):
         .replace('\xc2', '')
         .replace('\\xc2', '')
 
+        # PDF to text damages
+        .replace('c(cid:13)', '(c) ')
+        .replace('c(cid:2)', '(c) ')
+
         # not really a dash: an emdash
         .replace('–', '-')
 

tests/cluecode/data/copyrights/copyrights_with_parens/cop-kisa.txt.yml

@@ -1,3 +1,7 @@
 what:
   - copyrights
   - holders
+copyrights:
+  - Copyright (c) 2007 KISA (Korea Information Security Agency)
+holders:
+  - KISA (Korea Information Security Agency)

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-1.txt.yml

@@ -1,4 +1,7 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - Copyright (c) 1989 ITAA, formerly ADAPSO, and 1991, 1993
+holders:
+  - ITAA, formerly ADAPSO,

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-10.txt

@@ -1 +1,10 @@
-c(cid:13) 2016 Copyright held by the owner/author(s).
+c(cid:13) 2016 Copyright held by the owner/author(s).
+
+
+(c) 2016 Copyright held by the owner/author(s).
+
+
+Copyright held by the owner/author(s).
+
+
+Copyright is held by the owner/author(s)

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-10.txt.yml

@@ -1,4 +1,13 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - (c) 2016 Copyright held by the owner/author(s)
+  - (c) 2016 Copyright held by the owner/author(s)
+  - Copyright held by the owner/author(s)
+  - Copyright is held by the owner/author(s)
+holders:
+  - the owner/author(s)
+  - the owner/author(s)
+  - the owner/author(s)
+  - the owner/author(s)

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-11.txt.yml

@@ -1,4 +1,9 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - (c) BANDAI NAMCO Entertainment Inc.
+  - Copyright (c) 2017 Live2D Inc.
+holders:
+  - BANDAI NAMCO Entertainment Inc.
+  - Live2D Inc.

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-12.txt.yml

@@ -1,4 +1,7 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - Copyright (c) 2006, American Association for Articial Intelli- gence (www.aaai.org)
+holders:
+  - American Association for Articial Intelli- gence

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-13.txt.yml

@@ -1,4 +1,7 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - (c) 2016 ACM.
+holders:
+  - ACM

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-14.txt.yml

@@ -1,4 +1,5 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - Copyright (c) 2006

tests/cluecode/data/copyrights/copyrights_with_parens/copy-with-cid13-15.txt.yml

@@ -1,4 +1,7 @@
 what:
   - copyrights
   - holders
-  - holders_summary
+copyrights:
+  - Crown copyright 2017
+holders:
+  - Crown