Merge pull request #1526 from nexB/1403-declared-licenses-storage-improve

Improve declared_license storage #1403

Author: pombredanne

src/packagedcode/cargo.py

@@ -79,9 +79,6 @@ def repository_download_url(self, baseurl=default_download_baseurl):
     def api_data_url(self, baseurl=default_api_baseurl):
         return '{}/crates/{}'.format(baseurl, self.name)
 
-    def compute_normalized_license(self):
-        return models.compute_normalized_license(self.declared_license)
-
 
 def is_cargo_toml(location):
     return (filetype.is_file(location) and fileutils.file_name(location).lower() == 'cargo.toml')
@@ -112,12 +109,15 @@ def build_package(package_data):
 
     authors = core_package_data.get('authors')
     parties = list(party_mapper(authors, party_role='author'))
+    
+    declared_license = core_package_data.get('license')
 
     package = RustCargoCrate(
         name=name,
         version=version,
         description=description,
         parties=parties,
+        declared_license = declared_license 
     )
 
     return package

src/packagedcode/freebsd.py

@@ -37,6 +37,7 @@
 from commoncode import fileutils
 from commoncode import saneyaml
 from packagedcode import models
+from packagedcode.utils import combine_expressions
 
 """
 Handle FreeBSD ports
@@ -66,6 +67,37 @@ def recognize(cls, location):
     def get_package_root(cls, manifest_resource, codebase):
         return manifest_resource.parent(codebase)
 
+    def compute_normalized_license(self):
+        return compute_normalized_license(self.declared_license)
+
+
+def compute_normalized_license(declared_license):
+    """
+    Return a normalized license expression string detected from a list of
+    declared license items or an ordered dict.
+    """
+    if not declared_license:
+        return
+
+    licenses = declared_license.get('licenses')
+    if not licenses:
+        return
+
+    license_logic = declared_license.get('licenselogic')
+    relation= 'AND'
+    if license_logic:
+        if license_logic == 'or' or license_logic == 'dual':
+            relation = 'OR'
+    
+    detected_licenses = []
+    for declared in licenses:
+        detected_license = models.compute_normalized_license(declared)
+        if detected_license:
+            detected_licenses.append(detected_license)
+    
+    if detected_licenses:
+        return combine_expressions(detected_licenses, relation)
+
 
 def is_freebsd_manifest(location):
     return (filetype.is_file(location)
@@ -148,20 +180,13 @@ def license_mapper(package_data, package):
     if not licenses:
         return
 
-    # licenselogic is found as 'or' in some cases in the wild
-    if license_logic == 'or' or license_logic == 'dual':
-        lics = [l.strip() for l in licenses if l and l.strip()]
-        lics = ' OR '.join(lics)
-    # licenselogic is found as 'and' in some cases in the wild
-    elif license_logic == 'and' or license_logic == 'multi':
-        lics = [l.strip() for l in licenses if l and l.strip()]
-        lics = ' AND '.join(lics)
-    # 'single' or default licenselogic value
-    else:
-        lics = [l.strip() for l in licenses if l and l.strip()]
-        lics = ', '.join(lics)
-
-    package.declared_license = lics or None
+    declared_license = OrderedDict()
+    lics = [l.strip() for l in licenses if l and l.strip()]
+    declared_license['licenses'] = lics
+    if license_logic:
+        declared_license['licenselogic'] = license_logic
+        
+    package.declared_license = declared_license
     return package
 
 

src/packagedcode/phpcomposer.py

@@ -39,6 +39,8 @@
 from commoncode import filetype
 from commoncode import fileutils
 from packagedcode import models
+from packagedcode.utils import combine_expressions
+
 
 """
 Parse PHP composer package manifests, see https://getcomposer.org/ and
@@ -94,7 +96,39 @@ def compute_normalized_license(self):
         """
         Per https://getcomposer.org/doc/04-schema.md#license this is an expression
         """
-        return models.Package.compute_normalized_license(self)
+        return compute_normalized_license(self.declared_license)
+
+
+def compute_normalized_license(declared_license):
+    """
+    Return a normalized license expression string detected from a list of
+    declared license items or string type.
+    """
+    if not declared_license:
+        return
+
+    detected_licenses = []
+
+    if isinstance(declared_license, string_types):
+        if declared_license == 'proprietary':
+            return declared_license
+        if '(' in declared_license and ')' in declared_license and ' or ' in declared_license:
+            declared_license = declared_license.strip().rstrip(')').lstrip('(')
+            declared_license = declared_license.split(' or ')
+        else:
+            return models.compute_normalized_license(declared_license)
+
+    if isinstance(declared_license, list):
+        for declared in declared_license:
+            detected_license = models.compute_normalized_license(declared)
+            detected_licenses.append(detected_license)
+    else:
+        declared_license = repr(declared_license)
+        detected_license = models.compute_normalized_license(declared_license)
+
+    if detected_licenses:
+        # build a proper license expression: the defaultfor composer is OR
+        return combine_expressions(detected_licenses, 'OR')
 
 
 def is_phpcomposer_json(location):
@@ -200,25 +234,11 @@ def licensing_mapper(licenses, package, is_private=False):
     license and the `is_private` Fkag is True, we return a "proprietary-license"
     license.
     """
-    if not licenses:
+    if not licenses and is_private:
+        package.declared_license = 'proprietary-license'
         return package
 
-    if isinstance(licenses, list):
-        # For a package, when there is a choice between licenses
-        # ("disjunctive license"), multiple can be specified as array.
-        # build a proper license expression: the defaultfor composer is OR
-        lics = [l.strip() for l in licenses if l and l.strip()]
-        lics = ' OR '.join(lics)
-
-    elif not isinstance(licenses, string_types):
-        lics = repr(licenses)
-    else:
-        lics = licenses
-
-    if not lics and is_private:
-        lics ='proprietary-license'
-
-    package.declared_license = lics or None
+    package.declared_license = licenses
     return package
 
 

src/packagedcode/utils.py

@@ -150,9 +150,9 @@ def build_description(summary, description):
     return description
 
 
-def combine_expressions(expressions, licensing=Licensing()):
+def combine_expressions(expressions, relation='AND', licensing=Licensing()):
     """
-    Return a combined license expression string with AND, given a list of
+    Return a combined license expression string with relation, given a list of
     license expressions strings.
 
     For example:
@@ -181,4 +181,7 @@ def combine_expressions(expressions, licensing=Licensing()):
         return expressions[0]
 
     expressions = [licensing.parse(le, simple=True) for le in expressions]
-    return str(licensing.AND(*expressions))
+    if relation == 'OR':
+        return str(licensing.OR(*expressions))
+    else:
+        return str(licensing.AND(*expressions))

tests/packagedcode/data/cargo/clap/Cargo.toml.expected

@@ -1,43 +1,43 @@
 {
- "type": "cargo",
- "namespace": null,
- "name": "clap",
- "version": "2.32.0",
- "qualifiers": null,
- "subpath": null,
- "primary_language": "Rust",
- "description": "A simple to use, efficient, and full featured  Command Line Argument Parser",
- "release_date": null,
- "parties": [
-   {
-     "type": "person",
-     "role": "author",
-     "name": "Kevin K.",
-     "email": "kbknapp@gmail.com",
-     "url": null
-   }
- ],
- "keywords": [],
- "homepage_url": null,
- "download_url": null,
- "size": null,
- "sha1": null,
- "md5": null,
- "sha256": null,
- "sha512": null,
- "bug_tracking_url": null,
- "code_view_url": null,
- "vcs_url": null,
- "copyright": null,
- "license_expression": null,
- "declared_license": null,
- "notice_text": null,
- "manifest_path": null,
- "dependencies": [],
- "contains_source_code": null,
- "source_packages": [],
- "purl": "pkg:cargo/clap@2.32.0",
- "repository_homepage_url": "https://crates.io/crates/clap",
- "repository_download_url": "https://crates.io/api/v1/crates/clap/2.32.0/download",
- "api_data_url": "https://crates.io/api/v1/crates/clap"
-}
+  "type": "cargo",
+  "namespace": null,
+  "name": "clap",
+  "version": "2.32.0",
+  "qualifiers": null,
+  "subpath": null,
+  "primary_language": "Rust",
+  "description": "A simple to use, efficient, and full featured  Command Line Argument Parser",
+  "release_date": null,
+  "parties": [
+    {
+      "type": "person",
+      "role": "author",
+      "name": "Kevin K.",
+      "email": "kbknapp@gmail.com",
+      "url": null
+    }
+  ],
+  "keywords": [],
+  "homepage_url": null,
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": "mit",
+  "declared_license": "MIT",
+  "notice_text": null,
+  "manifest_path": null,
+  "dependencies": [],
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": "pkg:cargo/clap@2.32.0",
+  "repository_homepage_url": "https://crates.io/crates/clap",
+  "repository_download_url": "https://crates.io/api/v1/crates/clap/2.32.0/download",
+  "api_data_url": "https://crates.io/api/v1/crates/clap"
+}