Merge pull request #353 from nexB/99-license-in-binary

#99 license detection in binaries fails sometime

Author: pombredanne

apache-2.0.LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -174,3 +175,28 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

src/cluecode/copyrights.py

@@ -39,7 +39,7 @@
 logger = logging.getLogger(__name__)
 if os.environ.get('SCANCODE_COPYRIGHT_DEBUG'):
     import sys
-    logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
+    logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
 
@@ -120,6 +120,14 @@ def detect(location):
     (r'^(Company:|For|File|Last|[Rr]eleased?|[Cc]opyrighting)$', 'JUNK'),
     (r'^Authori.*$', 'JUNK'),
     (r'^[Bb]uild$', 'JUNK'),
+    (r'^[Ss]tring$', 'JUNK'),
+    (r'^Implementation-Vendor$', 'JUNK'),
+    (r'^(dnl|rem|REM)$', 'JUNK'),
+    (r'^Implementation-Vendor$', 'JUNK'),
+    (r'^Supports$', 'JUNK'),
+    (r'^\.byte$', 'JUNK'),
+    (r'^[Cc]ontributed?$', 'JUNK'),
+    (r'^[Ff]unctions?$', 'JUNK'),
 
     # various trailing words that are junk
     (r'^(?:Copyleft|LegalCopyright|AssemblyCopyright|Distributed|Report|Available|true|false|node|jshint|node\':true|node:true)$', 'JUNK'),
@@ -161,26 +169,30 @@ def detect(location):
     # (dutch and belgian) company suffix
     (r'^[Bb]\.?[Vv]\.?|BVBA$', 'COMP'),
     # university
-    (r'^[Uu]niv([.]|ersit(y|e|at?|ad?))$', 'UNI'),
+    (r'^\(?[Uu]niv(?:[.]|ersit(?:y|e|at?|ad?))\)?$', 'UNI'),
+    # university
     # institutes
     (r'^[Ii]nstitut(s|o|os|e|es|et|a|at|as|u|i)?$', 'NNP'),
     # "holders" is considered as a common noun
     (r'^([Hh]olders?|HOLDERS?|[Rr]espective)$', 'NN'),
 
     # (r'^[Cc]ontributors?\.?', 'NN'),
     # "authors" or "contributors" is interesting, and so a tag of its own
-    (r'^[Aa]uthors?$', 'AUTH'),
-    (r'^[Aa]uthor\(s\)$', 'AUTH'),
+    (r'^[Aa]uthors?\.?$', 'AUTH'),
+    (r'^[Aa]uthor\(s\)\.?$', 'AUTH'),
     (r'^[Cc]ontribut(ors?|ing)\.?$', 'AUTH'),
 
     # commiters is interesting, and so a tag of its own
-    (r'[Cc]ommitters?', 'COMMIT'),
-    # same for maintainer, developed, etc...
+    (r'[Cc]ommitters\.??', 'COMMIT'),
+    # same for maintainers.
+    (r'^([Mm]aintainers?\.?|[Dd]evelopers?\.?)$', 'MAINT'),
+    
+    # same for developed, etc...
     (r'^(([Rr]e)?[Cc]oded|[Mm]odified|[Mm]ai?nt[ea]ine(d|r)|[Ww]ritten|[Dd]eveloped)$', 'AUTH2'),
     # author
     (r'@author', 'AUTH'),
     # of
-    (r'^[Oo][Ff]|[Dd][Ee]$', 'OF'),
+    (r'^[Oo][Ff]|[Dd][Eei]$', 'OF'),
     # in
     (r'^in$', 'IN'),
     # by
@@ -199,6 +211,9 @@ def detect(location):
     # explicitly ignoring these words: FIXME: WHY?
     (r'^([Tt]his|THIS|[Pp]ermissions?|PERMISSIONS?|All)$', 'NN'),
 
+    # Portions copyright .... are worth keeping
+    (r'[Pp]ortions?', 'PORTIONS'),
+    
     # in dutch/german names, like Marco van Basten, or Klemens von Metternich
     # and Spanish/French Da Siva and De Gaulle
     (r'^(([Vv][ao]n)|[Dd][aeu])$', 'VAN'),
@@ -257,20 +272,22 @@ def detect(location):
     (r'^AT&T$', 'ATT'),
     # comma as a conjunction
     (r'^,$', 'CC'),
-    # .\ is not a noun
-    (r'^\.\\$', 'JUNK'),
+    # .\" is not a noun
+    (r'^\.\\\?"?$', 'JUNK'),
 
     # nouns (default)
     (r'.+', 'NN'),
 ]
 
 # Comments in the Grammar are lines that start with #
-grammar = """
+grammar = """   
     COPY: {<COPY>}
-    YR-RANGE: {<YR>+ <CC> <YR>}
+    YR-RANGE: {<YR>+ <CC>+ <YR>}
     YR-RANGE: {<YR> <DASH>* <YR|CD>+}
     YR-RANGE: {<CD>? <YR>+}
     YR-RANGE: {<YR>+ }
+    YR-AND: {<CC>? <YR>+ <CC>+ <YR>}
+    YR-RANGE: {<YR-AND>+}
 
     NAME: {<NNP> <VAN|OF> <NN*> <NNP>}
     NAME: {<NNP> <PN> <VAN> <NNP>}
@@ -294,6 +311,7 @@ def detect(location):
     COMPANY: {<NNP|CAPS> <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <NNP|CAPS>? <COMP> <COMP>?}
     COMPANY: {<UNI|NNP> <VAN|OF> <NNP>+ <UNI>?}
     COMPANY: {<NNP>+ <UNI>}
+    COMPANY: {<UNI> <OF> <NN|NNP>}
     COMPANY: {<COMPANY> <CC> <COMPANY>}
     COMPANY: {<ATT> <COMP>?}
     COMPANY: {<COMPANY> <CC> <NNP>}
@@ -308,6 +326,10 @@ def detect(location):
     NAME: {<NNP> <PN|VAN>? <PN|VAN>? <NNP>}
     NAME: {<NNP> <NN> <NNP>}
     NAME: {<NNP> <COMMIT>}
+    # the LGPL VGABios developers Team
+    NAME: {<NN>? <NNP> <MAINT> <COMP>}
+    # Debian Qt/KDE Maintainers
+    NAME: {<NNP> <NN>? <MAINT>}
     NAME: {<NN> <NNP> <ANDCO>}
     NAME: {<NN>? <NNP> <CC> <NAME>}
     NAME: {<NN>? <NNP> <OF> <NN>? <NNP> <NNP>?}
@@ -329,6 +351,8 @@ def detect(location):
 
     NAME: {<NNP|CAPS>+ <AUTH>}
 
+    NAME: {<VAN|OF> <NAME>}
+    NAME: {<NAME3> <COMP>}
 
 # Companies
     COMPANY: {<NAME|NAME2|NAME3|NNP>+ <OF> <NN>? <COMPANY|COMP>}
@@ -339,6 +363,9 @@ def detect(location):
     COMPANY: {<NN> <COMP>+}
     COMPANY: {<URL>}
 
+# The Regents of the University of California
+    NAME: {<NN> <NNP> <OF> <NN> <COMPANY>}
+
 # Trailing Authors
     COMPANY: {<NAME|NAME2|NAME3|NNP>+ <AUTH>}
 
@@ -350,44 +377,47 @@ def detect(location):
 
     NAME: {<BY> <NN> <AUTH>}
 
+# NetGroup, Politecnico di Torino (Italy)
+    COMPANY: {<NNP> <COMPANY> <NN>}
+
+# Arizona Board of Regents (University of Arizona)
+    NAME: {<COMPANY> <OF> <NN|NNP>}
+
+# The Regents of the University of California
+    NAME: {<NAME> <COMPANY>}
+
+# John Doe and Myriam Doe
+    NAME: {<NAME|NNP> <CC> <NNP|NAME>}
+
 # Various forms of copyright statements
     COPYRIGHT: {<COPY> <NAME> <COPY> <YR-RANGE>}
 
-    COPYRIGHT: {<COPY> <COPY> <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
-    COPYRIGHT: {<COPY> <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
+    COPYRIGHT: {<COPY> <COPY>? <BY>? <COMPANY|NAME*|YR-RANGE>* <BY>? <EMAIL>+}
 
-    COPYRIGHT: {<COPY> <COPY> <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
-    COPYRIGHT: {<COPY> <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
+    COPYRIGHT: {<COPY> <COPY>? <NAME|NAME2|NAME3> <CAPS> <YR-RANGE>}
 
-    COPYRIGHT: {<COPY> <COPY> <NAME|NAME2|NAME3>+ <YR-RANGE>*}
-    COPYRIGHT: {<COPY> <NAME|NAME2|NAME3>+ <YR-RANGE>*}
+    COPYRIGHT: {<COPY> <COPY>? <NAME|NAME2|NAME3>+ <YR-RANGE>*}
 
-    COPYRIGHT: {<COPY> <COPY> <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
-    COPYRIGHT: {<COPY> <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
+    COPYRIGHT: {<COPY> <COPY>? <CAPS|NNP>+ <CC> <NN> <COPY> <YR-RANGE>?}
 
-    COPYRIGHT: {<COPY> <COPY> <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
-    COPYRIGHT: {<COPY> <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
+    COPYRIGHT: {<COPY> <COPY>? <BY>? <COMPANY|NAME*>+ <YR-RANGE>*}
 
-    COPYRIGHT: {<NNP>? <COPY> <COPY> (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
-    COPYRIGHT: {<NNP>? <COPY> (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
+    COPYRIGHT: {<NNP>? <COPY> <COPY>? (<YR-RANGE>+ <BY>? <NN>? <COMPANY|NAME|NAME2>+ <EMAIL>?)+}
 
-    COPYRIGHT: {<COPY> <COPY> <NN> <NAME> <YR-RANGE>}
-    COPYRIGHT: {<COPY> <NN> <NAME> <YR-RANGE>}
+    COPYRIGHT: {<COPY> <COPY>? <NN> <NAME> <YR-RANGE>}
 
     COPYRIGHT: {<COPY>+ <BY> <NAME|NAME2|NAME3>+}
 
     COPYRIGHT: {<COPY> <COPY> <COMP>+}
 
     COPYRIGHT: {<COPY> <COPY> <NN>+ <COMPANY|NAME|NAME2>+}
 
-    COPYRIGHT: {<COPY> <COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
-    COPYRIGHT: {<COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
+    COPYRIGHT: {<COPY> <COPY>? <NN> <NN>? <COMP> <YR-RANGE>?}
 
-    COPYRIGHT: {<COPY> <COPY> <NN> <NN>? <COMP> <YR-RANGE>?}
+    COPYRIGHT: {<COPY> <COPY>? <NN> <NN>? <COMP> <YR-RANGE>?}
     COPYRIGHT: {<COPY> <NN> <NN>? <COMPANY> <YR-RANGE>?}
 
-    COPYRIGHT: {<COPY> <COPY> <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
-    COPYRIGHT: {<COPY> <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
+    COPYRIGHT: {<COPY> <COPY>? <YR-RANGE|NNP> <CAPS|BY>? <NNP|YR-RANGE|NAME>+}
 
     COPYRIGHT: {<COPY> <COPY> <NNP>+}
 
@@ -402,43 +432,40 @@ def detect(location):
     COPYRIGHT2: {<COPY> <COPY><NN>? <COPY> <YR-RANGE> <BY> <NN>}
     COPYRIGHT2: {<COPY> <NN>? <COPY> <YR-RANGE> <BY> <NN>}
 
-    COPYRIGHT2: {<COPY> <COPY><NN> <YR-RANGE> <BY> <NAME>}
-    COPYRIGHT2: {<COPY> <NN> <YR-RANGE> <BY> <NAME>}
+    COPYRIGHT2: {<COPY> <COPY>? <NN> <YR-RANGE> <BY> <NAME>}
 
-    COPYRIGHT2: {<COPY> <COPY><YR-RANGE> <DASH> <NAME2|NAME>}
-    COPYRIGHT2: {<COPY> <YR-RANGE> <DASH> <NAME2|NAME>}
+    COPYRIGHT2: {<COPY> <COPY>? <YR-RANGE> <DASH> <BY>? <NAME2|NAME>}
 
-    COPYRIGHT2: {<COPY> <COPY> <YR-RANGE> <NNP> <NAME>}
-    COPYRIGHT2: {<COPY> <YR-RANGE> <NNP> <NAME>}
+    COPYRIGHT2: {<COPY> <COPY>? <YR-RANGE> <NNP> <NAME>}
 
     COPYRIGHT2: {<NAME> <COPY> <YR-RANGE>}
 
-    COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
-    COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
+    COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>*}
 
-    COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
-    COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
+    COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <COMPANY>}
 
-    COPYRIGHT2: {<COPY> <COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
-    COPYRIGHT2: {<COPY> <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
+    COPYRIGHT2: {<COPY> <COPY>? <NN|CAPS>? <YR-RANGE>+ <NN|CAPS>* <DASH> <COMPANY>}
 
     COPYRIGHT2: {<NNP|NAME|COMPANY> <COPYRIGHT2>}
 
     COPYRIGHT: {<COPYRIGHT> <NN> <COMPANY>}
 
-    COPYRIGHT: {<COPY> <COPY> <BY>? <NN> <COMPANY>}
-    COPYRIGHT: {<COPY> <BY>? <NN> <COMPANY>}
+    COPYRIGHT: {<COPY> <COPY>? <BY>? <NN> <COMPANY>}
 
     COPYRIGHT: {<COMPANY> <NN> <NAME> <COPYRIGHT2>}
     COPYRIGHT: {<COPYRIGHT2> <COMP> <COMPANY>}
     COPYRIGHT: {<COMPANY> <NN> <COPYRIGHT2>}
     COPYRIGHT: {<COPYRIGHT2> <NNP> <CC> <COMPANY>}
 
+    #COPYRIGHT2: {<COPYRIGHT2> <JUNK>* <COMPANY>}
 
 # copyrights in the style of Scilab/INRIA
     COPYRIGHT: {<NNP> <NN> <COPY> <NNP>}
     COPYRIGHT: {<NNP> <COPY> <NNP>}
 
+# portions copyright
+    COPYRIGHT: {<PORTIONS> <COPYRIGHT|COPYRIGHT2>}
+
 # Authors
     AUTH: {<AUTH2>+ <BY>}
     AUTHOR: {<AUTH>+ <NN>? <COMPANY|NAME|YR-RANGE>* <BY>? <EMAIL>+}
@@ -714,11 +741,11 @@ def detect(self, numbered_lines):
 
         # first, POS tag each token using token regexes
         tagged_text = self.tagger.tag(tokens)
-        # logger.debug('CopyrightDetector:tagged_text: ' + str(tagged_text))
+        logger.debug('CopyrightDetector:tagged_text: ' + str(tagged_text))
 
         # then build a parse tree based on tagged tokens
         tree = self.chunker.parse(tagged_text)
-        # logger.debug('CopyrightDetector:parse tree: ' + str(tree))
+        logger.debug('CopyrightDetector:parse tree: ' + str(tree))
 
         # OPTIMIZED
         nltk_tree_Tree = nltk.tree.Tree
@@ -991,12 +1018,12 @@ def prepare_text_line(line):
     # common comment characters
     line = line.strip('\\/*#%;')
     # un common comment line prefix in dos
-    line = re_sub('^rem ', ' ', line)
-    line = re_sub('^\@rem ', ' ', line)
+    line = re_sub('^rem\s+', ' ', line)
+    line = re_sub('^\@rem\s+', ' ', line)
     # un common comment line prefix in autotools am/in
-    line = re_sub('^dnl ', ' ', line)
+    line = re_sub('^dnl\s+', ' ', line)
     # un common comment line prefix in man pages
-    line = re_sub('^\.\\"', ' ', line)
+    line = re_sub('^\.\\\\"', ' ', line)
     # un common pipe chars in some ascii art
     line = line.replace('|', ' ')
 

src/licensedcode/data/licenses/3pp-proprietary-03.LICENSE

@@ -1,5 +1,3 @@
-*  Copyright {Date} {Company}
-*
 *  {Company or ProductName} Registered in U.S. Patent and Trademark Office.
 *  Confidential computer software. Valid license from {CompanyName} or authorized
 *  sublicensor required for possession, use or copying. Consistent with

src/licensedcode/data/licenses/adobe-scl.LICENSE

@@ -1,29 +1,27 @@
-Please read this Source Code License Agreement carefully before using
-the source code.
+Please read this Source Code License Agreement carefully before using the source
+code.
 
-Adobe Systems Incorporated grants to you a perpetual, worldwide, non-exclusive,
-no-charge, royalty-free, irrevocable copyright license, to reproduce,
-prepare derivative works of, publicly display, publicly perform, and
-distribute this source code and such derivative works in source or
-object code form without any attribution requirements.
+Adobe Systems Incorporated grants to you a perpetual, worldwide, non-exclusive, no-
+charge, royalty-free, irrevocable copyright license, to reproduce, prepare derivative
+works of, publicly display, publicly perform, and distribute this source code and
+such derivative works in source or object code form without any attribution
+requirements.
 
 The name "Adobe Systems Incorporated" must not be used to endorse or promote products
 derived from the source code without prior written permission.
 
 You agree to indemnify, hold harmless and defend Adobe Systems Incorporated from and
-against any loss, damage, claims or lawsuits, including attorney's
-fees that arise or result from your use or distribution of the source
-code.
+against any loss, damage, claims or lawsuits, including attorney's fees that arise or
+result from your use or distribution of the source code.
 
-THIS SOURCE CODE IS PROVIDED "AS IS" AND "WITH ALL FAULTS", WITHOUT
-ANY TECHNICAL SUPPORT OR ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING,
-BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE ARE DISCLAIMED. ALSO, THERE IS NO WARRANTY OF
-NON-INFRINGEMENT, TITLE OR QUIET ENJOYMENT. IN NO EVENT SHALL MACROMEDIA
-OR ITS SUPPLIERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOURCE CODE, EVEN IF
-ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THIS SOURCE CODE IS PROVIDED "AS IS" AND "WITH ALL FAULTS", WITHOUT ANY TECHNICAL
+SUPPORT OR ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. ALSO, THERE IS NO WARRANTY OF NON-INFRINGEMENT, TITLE OR QUIET ENJOYMENT.
+IN NO EVENT SHALL MACROMEDIA OR ITS SUPPLIERS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOURCE CODE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.