Merge commit 'cc137eb13987010bd140f08d5581cc579f695d1c' into develop

Author: pombredanne

src/packagedcode/maven.py

@@ -37,19 +37,20 @@
 
 import attr
 import javaproperties
+from license_expression import Licensing
 from lxml import etree
 from packageurl import PackageURL
-from pymaven import pom
 from pymaven import artifact
+from pymaven import pom
 
 from commoncode import filetype
 from commoncode import fileutils
 from packagedcode import models
+from packagedcode.models import Package
 from packagedcode.utils import normalize_vcs_url
 from packagedcode.utils import VCS_URLS
 from textcode import analysis
 from typecode import contenttype
-from packagedcode.models import Package
 
 
 TRACE = False
@@ -97,7 +98,7 @@ def get_package_root(cls, manifest_resource, codebase):
                     if ancestor.name == 'META-INF':
                         jar_root_dir = ancestor.parent(codebase)
                         return jar_root_dir
-    
+
             return manifest_resource.parent(codebase)
 
         elif manifest_resource.path.endswith('META-INF/MANIFEST.MF'):
@@ -145,6 +146,85 @@ def api_data_url(self, baseurl=default_api_baseurl):
             filename=filename,
             baseurl=baseurl)
 
+    def compute_normalized_license(self):
+        return compute_normalized_license(self.declared_license)
+
+
+def compute_normalized_license(declared_license):
+    """
+    Return a detected license expression from a declared license mapping.
+    """
+    if not declared_license:
+        return
+
+    licensing = Licensing()
+
+    detected_licenses = []
+
+    for license_declaration in declared_license:
+        # 1. try detection on the value of name if not empty and keep this
+        name = license_declaration.get('name')
+        via_name = models.compute_normalized_license(name)
+
+        # 2. try detection on the value of url  if not empty and keep this
+        url = license_declaration.get('url')
+        via_url = models.compute_normalized_license(url)
+
+        # 3. try detection on the value of comment  if not empty and keep this
+        comments = license_declaration.get('comments')
+        via_comments = models.compute_normalized_license(comments)
+
+
+        if via_name:
+            # The name should have precedence and any unknowns
+            # in url and comment should be ignored.
+            if via_url == 'unknown':
+                via_url = None
+            if via_comments == 'unknown':
+                via_comments = None
+
+        # Check the three detections to decide which license to keep
+        name_and_url = via_name == via_url
+        name_and_comment = via_name == via_comments
+        all_same = name_and_url and name_and_comment
+
+        if via_name:
+            if all_same:
+                detected_licenses.append(via_name)
+
+            # name and (url or comment) are same
+            elif name_and_url and not via_comments:
+                detected_licenses.append(via_name)
+            elif name_and_comment and not via_url:
+                detected_licenses.append(via_name)
+
+            else:
+                # we have some non-unknown license detected in url or comment
+                detections = via_name, via_url, via_comments
+                detections = [l for l in detections if l]
+                if detections:
+                    if len(detections) == 1:
+                        combined_expression = detections[0]
+                    else:
+                        expressions = [
+                            licensing.parse(le, simple=True) for le in detections]
+                        combined_expression = str(licensing.AND(*expressions))
+                    detected_licenses.append(combined_expression)
+
+        elif via_url:
+            detected_licenses.append(via_url)
+        elif via_comments:
+            detected_licenses.append(via_comments)
+
+    if len(detected_licenses) == 1:
+        return detected_licenses[0]
+
+    if detected_licenses:
+        # Combine if pom contains more than one licenses declarations.
+        expressions = [licensing.parse(le, simple=True) for le in detected_licenses]
+        combined_expression = licensing.AND(*expressions)
+        return str(combined_expression)
+
 
 def build_url(group_id, artifact_id, version, filename, baseurl='http://repo1.maven.org/maven2'):
     """
@@ -973,12 +1053,7 @@ def parse(location=None, text=None, check_is_pom=True, extra_properties=None):
             # complex defeinition in Maven
             qualifiers['type'] = extension
 
-    # TODO: we join all data in a single text: this may not be right
-    declared_license = []
-    for lic in pom.licenses:
-        lt = (l for l in [lic['name'], lic['url'], lic['comments']] if l)
-        declared_license.extend(lt)
-    declared_license = '\n'.join(declared_license)
+    declared_license = pom.licenses
 
     source_packages = []
     # TODO: what does this mean????

tests/packagedcode/data/m2/aopalliance/aopalliance/1.0/aopalliance-1.0.pom.package.json

@@ -21,8 +21,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "Public Domain", 
+  "license_expression": "public-domain", 
+  "declared_license": [
+    {
+      "name": "Public Domain", 
+      "url": null, 
+      "comments": null, 
+      "distribution": null
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [], 

tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.3/aspectjrt-1.5.3.pom.package.json

@@ -21,8 +21,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "Eclipse Public License - v 1.0\nhttp://www.eclipse.org/legal/epl-v10.html", 
+  "license_expression": "epl-1.0", 
+  "declared_license": [
+    {
+      "name": "Eclipse Public License - v 1.0", 
+      "url": "http://www.eclipse.org/legal/epl-v10.html", 
+      "comments": null, 
+      "distribution": "repo"
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [], 

tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.203/bndlib-0.0.203.pom.package.json

@@ -29,8 +29,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "This material is licensed under the Apache Software License, Version 2.0\nhttp://www.apache.org/licenses/LICENSE-2.0", 
+  "license_expression": "(apache-2.0 AND unknown) AND apache-2.0", 
+  "declared_license": [
+    {
+      "name": "This material is licensed under the Apache Software License, Version 2.0", 
+      "url": "http://www.apache.org/licenses/LICENSE-2.0", 
+      "comments": null, 
+      "distribution": "repo"
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [], 

tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.238/bndlib-0.0.238.pom.package.json

@@ -29,8 +29,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "This material is licensed under the Apache Software License, Version 2.0\nhttp://www.apache.org/licenses/LICENSE-2.0", 
+  "license_expression": "(apache-2.0 AND unknown) AND apache-2.0", 
+  "declared_license": [
+    {
+      "name": "This material is licensed under the Apache Software License, Version 2.0", 
+      "url": "http://www.apache.org/licenses/LICENSE-2.0", 
+      "comments": null, 
+      "distribution": "repo"
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [], 

tests/packagedcode/data/m2/bytebuddy/pom.xml.package.json

@@ -29,8 +29,15 @@
   "code_view_url": "https://github.com/raphw/byte-buddy.git", 
   "vcs_url": "git+https://github.com/raphw/byte-buddy.git", 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "The Apache Software License, Version 2.0\nhttp://www.apache.org/licenses/LICENSE-2.0.txt\nA business-friendly OSS license", 
+  "license_expression": "apache-2.0", 
+  "declared_license": [
+    {
+      "name": "The Apache Software License, Version 2.0", 
+      "url": "http://www.apache.org/licenses/LICENSE-2.0.txt", 
+      "comments": "A business-friendly OSS license", 
+      "distribution": "repo"
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [

tests/packagedcode/data/m2/c3p0/c3p0/0.9.0.4/c3p0-0.9.0.4.pom.package.json

@@ -21,8 +21,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "GNU LESSER GENERAL PUBLIC LICENSE\nhttp://www.gnu.org/licenses/lgpl.txt", 
+  "license_expression": "lgpl-2.0-plus AND ((gpl-1.0-plus AND lgpl-2.1-plus) AND unknown)", 
+  "declared_license": [
+    {
+      "name": "GNU LESSER GENERAL PUBLIC LICENSE", 
+      "url": "http://www.gnu.org/licenses/lgpl.txt", 
+      "comments": null, 
+      "distribution": null
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [], 

tests/packagedcode/data/m2/codec/commons-codec/1.3/commons-codec-1.3.pom.package.json

@@ -148,8 +148,15 @@
   "code_view_url": "http://cvs.apache.org/viewcvs/jakarta-commons/codec/", 
   "vcs_url": "cvs+pserver:anoncvs@cvs.apache.org:/home/cvspublic:jakarta-commons/codec", 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "The Apache Software License, Version 2.0\n/LICENSE.txt", 
+  "license_expression": "apache-2.0", 
+  "declared_license": [
+    {
+      "name": "The Apache Software License, Version 2.0", 
+      "url": "/LICENSE.txt", 
+      "comments": null, 
+      "distribution": null
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [

tests/packagedcode/data/m2/collections/commons-collections/3.2/commons-collections-3.2.pom.package.json

@@ -687,8 +687,15 @@
   "code_view_url": "http://svn.apache.org/repos/asf/jakarta/commons/proper/collections/trunk", 
   "vcs_url": "svn+http://svn.apache.org/repos/asf/jakarta/commons/proper/collections/trunk", 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "The Apache Software License, Version 2.0\n/LICENSE.txt", 
+  "license_expression": "apache-2.0", 
+  "declared_license": [
+    {
+      "name": "The Apache Software License, Version 2.0", 
+      "url": "/LICENSE.txt", 
+      "comments": null, 
+      "distribution": null
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [

tests/packagedcode/data/m2/com/jcraft/jsch/0.1.23/jsch-0.1.23.pom.package.json

@@ -29,8 +29,15 @@
   "code_view_url": null, 
   "vcs_url": null, 
   "copyright": null, 
-  "license_expression": null, 
-  "declared_license": "BSD\nhttp://www.jcraft.com/jsch/LICENSE.txt\nLicense information from http://www.jcraft.com/jsch", 
+  "license_expression": "unknown AND bsd-new", 
+  "declared_license": [
+    {
+      "name": "BSD", 
+      "url": "http://www.jcraft.com/jsch/LICENSE.txt", 
+      "comments": "License information from http://www.jcraft.com/jsch", 
+      "distribution": null
+    }
+  ], 
   "notice_text": null, 
   "manifest_path": null, 
   "dependencies": [],