Skip to content

Commit b55d003

Browse files
pombredannepombredanne
committed
Improve copyright detection accuracy
 * handle better some trailing words and complex university names * handle "Portions Copyright ..." Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
1 parent 27f2f1f commit b55d003

5 files changed

Lines changed: 170 additions & 526 deletions

File tree

‎src/cluecode/copyrights.py‎

Lines changed: 77 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
logger = logging.getLogger(__name__)
4040
if os.environ.get('SCANCODE_COPYRIGHT_DEBUG'):
4141
import sys
42-
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
42+
logging.basicConfig(stream=sys.stdout)
4343
logger.setLevel(logging.DEBUG)
4444

4545

@@ -120,6 +120,14 @@ def detect(location):
120120
(r'^(Company:|For|File|Last|[Rr]eleased?|[Cc]opyrighting)$', 'JUNK'),
121121
(r'^Authori.*$', 'JUNK'),
122122
(r'^[Bb]uild$', 'JUNK'),
123+
(r'^[Ss]tring$', 'JUNK'),
124+
(r'^Implementation-Vendor$', 'JUNK'),
125+
(r'^(dnl|rem|REM)$', 'JUNK'),
126+
(r'^Implementation-Vendor$', 'JUNK'),
127+
(r'^Supports$', 'JUNK'),
128+
(r'^\.byte$', 'JUNK'),
129+
(r'^[Cc]ontributed?$', 'JUNK'),
130+
(r'^[Ff]unctions?$', 'JUNK'),
123131

124132
# various trailing words that are junk
125133
(r'^(?:Copyleft|LegalCopyright|AssemblyCopyright|Distributed|Report|Available|true|false|node|jshint|node\':true|node:true)$', 'JUNK'),
@@ -161,26 +169,30 @@ def detect(location):
161169
# (dutch and belgian) company suffix
162170
(r'^[Bb]\.?[Vv]\.?|BVBA$', 'COMP'),
163171
# university
164-
(r'^[Uu]niv([.]|ersit(y|e|at?|ad?))$', 'UNI'),
172+
(r'^\(?[Uu]niv(?:[.]|ersit(?:y|e|at?|ad?))\)?$', 'UNI'),
173+
# university
165174
# institutes
166175
(r'^[Ii]nstitut(s|o|os|e|es|et|a|at|as|u|i)?$', 'NNP'),
167176
# "holders" is considered as a common noun
168177
(r'^([Hh]olders?|HOLDERS?|[Rr]espective)$', 'NN'),
169178

170179
# (r'^[Cc]ontributors?\.?', 'NN'),
171180
# "authors" or "contributors" is interesting, and so a tag of its own
172-
(r'^[Aa]uthors?$', 'AUTH'),
173-
(r'^[Aa]uthor\(s\)$', 'AUTH'),
181+
(r'^[Aa]uthors?\.?$', 'AUTH'),
182+
(r'^[Aa]uthor\(s\)\.?$', 'AUTH'),
174183
(r'^[Cc]ontribut(ors?|ing)\.?$', 'AUTH'),
175184

176185
# commiters is interesting, and so a tag of its own
177-
(r'[Cc]ommitters?', 'COMMIT'),
178-
# same for maintainer, developed, etc...
186+
(r'[Cc]ommitters\.??', 'COMMIT'),
187+
# same for maintainers.
188+
(r'^([Mm]aintainers?\.?|[Dd]evelopers?\.?)$', 'MAINT'),
189+
190+
# same for developed, etc...
179191
(r'^(([Rr]e)?[Cc]oded|[Mm]odified|[Mm]ai?nt[ea]ine(d|r)|[Ww]ritten|[Dd]eveloped)$', 'AUTH2'),
180192
# author
181193
(r'@author', 'AUTH'),
182194
# of
183-
(r'^[Oo][Ff]|[Dd][Ee]$', 'OF'),
195+
(r'^[Oo][Ff]|[Dd][Eei]$', 'OF'),
184196
# in
185197
(r'^in$', 'IN'),
186198
# by
@@ -199,6 +211,9 @@ def detect(location):
199211
# explicitly ignoring these words: FIXME: WHY?
200212
(r'^([Tt]his|THIS|[Pp]ermissions?|PERMISSIONS?|All)$', 'NN'),
201213

214+
# Portions copyright .... are worth keeping
215+
(r'[Pp]ortions?', 'PORTIONS'),
216+
202217
# in dutch/german names, like Marco van Basten, or Klemens von Metternich
203218
# and Spanish/French Da Siva and De Gaulle
204219
(r'^(([Vv][ao]n)|[Dd][aeu])$', 'VAN'),
@@ -257,20 +272,22 @@ def detect(location):
257272
(r'^AT&T$', 'ATT'),
258273
# comma as a conjunction
259274
(r'^,$', 'CC'),
260-
# .\ is not a noun
261-
(r'^\.\\$', 'JUNK'),
275+
# .\" is not a noun
276+
(r'^\.\\\?"?$', 'JUNK'),
262277

263278
# nouns (default)
264279
(r'.+', 'NN'),
265280
]
266281

267282
# Comments in the Grammar are lines that start with #
268-
grammar = """
283+
grammar = """
269284
COPY: {<COPY>}
270-
YR-RANGE: {<YR>+ <CC> <YR>}
285+
YR-RANGE: {<YR>+ <CC>+ <YR>}
271286
YR-RANGE: {<YR> <DASH>* <YR|CD>+}
272287
YR-RANGE: {<CD>? <YR>+}
273288
YR-RANGE: {<YR>+ }
289+
YR-AND: {<CC>? <YR>+ <CC>+ <YR>}
290+
YR-RANGE: {<YR-AND>+}
274291
275292
NAME: {<NNP> <VAN|OF> <NN*> <NNP>}
276293
NAME: {<NNP> <PN> <VAN> <NNP>}
@@ -294,6 +311,7 @@ def detect(location):
294311
COMPANY: {<NNP|CAPS> <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <COMP> <COMP>?}
295312
COMPANY: {<UNI|NNP> <VAN|OF> <NNP>+ <UNI>?}
296313
COMPANY: {<NNP>+ <UNI>}
314+
COMPANY: {<UNI> <OF> <NN|NNP>}
297315
COMPANY: {<COMPANY> <CC> <COMPANY>}
298316
COMPANY: {<ATT> <COMP>?}
299317
COMPANY: {<COMPANY> <CC> <NNP>}
@@ -308,6 +326,10 @@ def detect(location):
308326
NAME: {<NNP> <PN|VAN>? <PN|VAN>? <NNP>}
309327
NAME: {<NNP> <NN> <NNP>}
310328
NAME: {<NNP> <COMMIT>}
329+
# the LGPL VGABios developers Team
330+
NAME: {<NN>? <NNP> <MAINT> <COMP>}
331+
# Debian Qt/KDE Maintainers
332+
NAME: {<NNP> <NN>? <MAINT>}
311333
NAME: {<NN> <NNP> <ANDCO>}
312334
NAME: {<NN>? <NNP> <CC> <NAME>}
313335
NAME: {<NN>? <NNP> <OF> <NN>? <NNP> <NNP>?}
@@ -329,6 +351,8 @@ def detect(location):
329351
330352
NAME: {<NNP|CAPS>+ <AUTH>}
331353
354+
NAME: {<VAN|OF> <NAME>}
355+
NAME: {<NAME3> <COMP>}
332356
333357
# Companies
334358
COMPANY: {<NAME|NAME2|NAME3|NNP>+ <OF> <NN>? <COMPANY|COMP>}
@@ -339,6 +363,9 @@ def detect(location):
339363
COMPANY: {<NN> <COMP>+}
340364
COMPANY: {<URL>}
341365
366+
# The Regents of the University of California
367+
NAME: {<NN> <NNP> <OF> <NN> <COMPANY>}
368+
342369
# Trailing Authors
343370
COMPANY: {<NAME|NAME2|NAME3|NNP>+ <AUTH>}
344371
@@ -350,44 +377,47 @@ def detect(location):
350377
351378
NAME: {<BY> <NN> <AUTH>}
352379
380+
# NetGroup, Politecnico di Torino (Italy)
381+
COMPANY: {<NNP> <COMPANY> <NN>}
382+
383+
# Arizona Board of Regents (University of Arizona)
384+
NAME: {<COMPANY> <OF> <NN|NNP>}
385+
386+
# The Regents of the University of California
387+
NAME: {<NAME> <COMPANY>}
388+
389+
# John Doe and Myriam Doe
390+
NAME: {<NAME|NNP> <CC> <NNP|NAME>}
391+
353392
# Various forms of copyright statements
354393
COPYRIGHT: {<COPY> <NAME> <COPY> <YR-RANGE>}
355394
356-
COPYRIGHT: {<COPY> <COPY> <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
357-
COPYRIGHT: {<COPY> <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
395+
COPYRIGHT: {<COPY> <COPY>? <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
358396
359-
COPYRIGHT: {<COPY> <COPY> <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
360-
COPYRIGHT: {<COPY> <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
397+
COPYRIGHT: {<COPY> <COPY>? <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
361398
362-
COPYRIGHT: {<COPY> <COPY> <NAME|NAME2|NAME3>+ <YR-RANGE>*}
363-
COPYRIGHT: {<COPY> <NAME|NAME2|NAME3>+ <YR-RANGE>*}
399+
COPYRIGHT: {<COPY> <COPY>? <NAME|NAME2|NAME3>+ <YR-RANGE>*}
364400
365-
COPYRIGHT: {<COPY> <COPY> <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
366-
COPYRIGHT: {<COPY> <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
401+
COPYRIGHT: {<COPY> <COPY>? <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
367402
368-
COPYRIGHT: {<COPY> <COPY> <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
369-
COPYRIGHT: {<COPY> <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
403+
COPYRIGHT: {<COPY> <COPY>? <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
370404
371-
COPYRIGHT: {<NNP>? <COPY> <COPY> (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
372-
COPYRIGHT: {<NNP>? <COPY> (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
405+
COPYRIGHT: {<NNP>? <COPY> <COPY>? (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
373406
374-
COPYRIGHT: {<COPY> <COPY> <NN> <NAME> <YR-RANGE>}
375-
COPYRIGHT: {<COPY> <NN> <NAME> <YR-RANGE>}
407+
COPYRIGHT: {<COPY> <COPY>? <NN> <NAME> <YR-RANGE>}
376408
377409
COPYRIGHT: {<COPY>+ <BY> <NAME|NAME2|NAME3>+}
378410
379411
COPYRIGHT: {<COPY> <COPY> <COMP>+}
380412
381413
COPYRIGHT: {<COPY> <COPY> <NN>+ <COMPANY|NAME|NAME2>+}
382414
383-
COPYRIGHT: {<COPY> <COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
384-
COPYRIGHT: {<COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
415+
COPYRIGHT: {<COPY> <COPY>? <NN> <NN>? <COMP> <YR-RANGE>?}
385416
386-
COPYRIGHT: {<COPY> <COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
417+
COPYRIGHT: {<COPY> <COPY>? <NN> <NN>? <COMP> <YR-RANGE>?}
387418
COPYRIGHT: {<COPY> <NN> <NN>? <COMPANY> <YR-RANGE>?}
388419
389-
COPYRIGHT: {<COPY> <COPY> <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
390-
COPYRIGHT: {<COPY> <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
420+
COPYRIGHT: {<COPY> <COPY>? <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
391421
392422
COPYRIGHT: {<COPY> <COPY> <NNP>+}
393423
@@ -402,43 +432,40 @@ def detect(location):
402432
COPYRIGHT2: {<COPY> <COPY><NN>? <COPY> <YR-RANGE> <BY> <NN>}
403433
COPYRIGHT2: {<COPY> <NN>? <COPY> <YR-RANGE> <BY> <NN>}
404434
405-
COPYRIGHT2: {<COPY> <COPY><NN> <YR-RANGE> <BY> <NAME>}
406-
COPYRIGHT2: {<COPY> <NN> <YR-RANGE> <BY> <NAME>}
435+
COPYRIGHT2: {<COPY> <COPY>? <NN> <YR-RANGE> <BY> <NAME>}
407436
408-
COPYRIGHT2: {<COPY> <COPY><YR-RANGE> <DASH> <NAME2|NAME>}
409-
COPYRIGHT2: {<COPY> <YR-RANGE> <DASH> <NAME2|NAME>}
437+
COPYRIGHT2: {<COPY> <COPY>? <YR-RANGE> <DASH> <BY>? <NAME2|NAME>}
410438
411-
COPYRIGHT2: {<COPY> <COPY> <YR-RANGE> <NNP> <NAME>}
412-
COPYRIGHT2: {<COPY> <YR-RANGE> <NNP> <NAME>}
439+
COPYRIGHT2: {<COPY> <COPY>? <YR-RANGE> <NNP> <NAME>}
413440
414441
COPYRIGHT2: {<NAME> <COPY> <YR-RANGE>}
415442
416-
COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
417-
COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
443+
COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
418444
419-
COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
420-
COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
445+
COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
421446
422-
COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
423-
COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
447+
COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
424448
425449
COPYRIGHT2: {<NNP|NAME|COMPANY> <COPYRIGHT2>}
426450
427451
COPYRIGHT: {<COPYRIGHT> <NN> <COMPANY>}
428452
429-
COPYRIGHT: {<COPY> <COPY> <BY>? <NN> <COMPANY>}
430-
COPYRIGHT: {<COPY> <BY>? <NN> <COMPANY>}
453+
COPYRIGHT: {<COPY> <COPY>? <BY>? <NN> <COMPANY>}
431454
432455
COPYRIGHT: {<COMPANY> <NN> <NAME> <COPYRIGHT2>}
433456
COPYRIGHT: {<COPYRIGHT2> <COMP> <COMPANY>}
434457
COPYRIGHT: {<COMPANY> <NN> <COPYRIGHT2>}
435458
COPYRIGHT: {<COPYRIGHT2> <NNP> <CC> <COMPANY>}
436459
460+
#COPYRIGHT2: {<COPYRIGHT2> <JUNK>* <COMPANY>}
437461
438462
# copyrights in the style of Scilab/INRIA
439463
COPYRIGHT: {<NNP> <NN> <COPY> <NNP>}
440464
COPYRIGHT: {<NNP> <COPY> <NNP>}
441465
466+
# portions copyright
467+
COPYRIGHT: {<PORTIONS> <COPYRIGHT|COPYRIGHT2>}
468+
442469
# Authors
443470
AUTH: {<AUTH2>+ <BY>}
444471
AUTHOR: {<AUTH>+ <NN>? <COMPANY|NAME|YR-RANGE>* <BY>? <EMAIL>+}
@@ -714,11 +741,11 @@ def detect(self, numbered_lines):
714741

715742
# first, POS tag each token using token regexes
716743
tagged_text = self.tagger.tag(tokens)
717-
# logger.debug('CopyrightDetector:tagged_text: ' + str(tagged_text))
744+
logger.debug('CopyrightDetector:tagged_text: ' + str(tagged_text))
718745

719746
# then build a parse tree based on tagged tokens
720747
tree = self.chunker.parse(tagged_text)
721-
# logger.debug('CopyrightDetector:parse tree: ' + str(tree))
748+
logger.debug('CopyrightDetector:parse tree: ' + str(tree))
722749

723750
# OPTIMIZED
724751
nltk_tree_Tree = nltk.tree.Tree
@@ -991,12 +1018,12 @@ def prepare_text_line(line):
9911018
# common comment characters
9921019
line = line.strip('\\/*#%;')
9931020
# un common comment line prefix in dos
994-
line = re_sub('^rem ', ' ', line)
995-
line = re_sub('^\@rem ', ' ', line)
1021+
line = re_sub('^rem\s+', ' ', line)
1022+
line = re_sub('^\@rem\s+', ' ', line)
9961023
# un common comment line prefix in autotools am/in
997-
line = re_sub('^dnl ', ' ', line)
1024+
line = re_sub('^dnl\s+', ' ', line)
9981025
# un common comment line prefix in man pages
999-
line = re_sub('^\.\\"', ' ', line)
1026+
line = re_sub('^\.\\\\"', ' ', line)
10001027
# un common pipe chars in some ascii art
10011028
line = line.replace('|', ' ')
10021029

0 commit comments

Comments
 (0)