Format extracted_license_statement as YAML

CHANGELOG.rst

@@ -135,7 +135,9 @@ License detection:
     These expressions are parallel to detections.
 
   - The ``declared_license`` attribute is renamed ``extracted_license_statement``
-    and is now a YAML-encoded string.
+    and is now a YAML-encoded string, which can be parsed to recreate the
+    original extracted license statement. Previously this used to be nested
+    python objects lists/dicts/string, but now this is always a YAML string.
 
     See `license updates documentation <https://scancode-toolkit.readthedocs.io/en/latest/explanations/license-detection-reference.html#change-in-license-data-format-package>`_
     for examples and details.

src/packagedcode/models.py

@@ -16,6 +16,7 @@
 import attr
 from packageurl import normalize_qualifiers
 from packageurl import PackageURL
+import saneyaml
 
 from commoncode import filetype
 from commoncode.datautils import choices
@@ -780,7 +781,7 @@ def populate_license_fields(self):
                 )
 
         if self.extracted_license_statement and not isinstance(self.extracted_license_statement, str):
-            self.extracted_license_statement = repr(self.extracted_license_statement)
+            self.extracted_license_statement = saneyaml.dump(self.extracted_license_statement)
 
     def to_dict(self, with_details=True, **kwargs):
         mapping = super().to_dict(with_details=with_details, **kwargs)

tests/formattedcode/data/csv/packages/expected-no-root.csv

@@ -1,3 +1,4 @@
 path,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__holder,package__declared_license_expression,package__declared_license_expression_spdx,package__license_detections,package__other_license_expression,package__other_license_expression_spdx,package__other_license_detections,package__extracted_license_statement,package__notice_text,package__file_references,package__extra_data,package__repository_homepage_url,package__repository_download_url,package__api_data_url,package__datasource_id,package__purl
 package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,,artistic-2.0,Artistic-2.0,"[{'license_expression': 'artistic-2.0', 'matches': [{'score': 50.0, 'start_line': 1, 'end_line': 1, 'matched_length': 3, 'match_coverage': 100.0, 'matcher': '1-hash', 'license_expression': 'artistic-2.0', 'rule_identifier': 'spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'rule_relevance': 50, 'rule_url': 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'matched_text': 'Artistic-2.0'}], 'identifier': 'artistic_2_0-2ca48deb-7b37-fc93-edd9-b349ad073774'}]",,,,['Artistic-2.0'],,,,https://www.npmjs.com/package/npm,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,https://registry.npmjs.org/npm/2.13.5,npm_package_json,pkg:npm/npm@2.13.5
+package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,,artistic-2.0,Artistic-2.0,"[{'license_expression': 'artistic-2.0', 'matches': [{'score': 50.0, 'start_line': 1, 'end_line': 1, 'matched_length': 3, 'match_coverage': 100.0, 'matcher': '1-hash', 'license_expression': 'artistic-2.0', 'rule_identifier': 'spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'rule_relevance': 50, 'rule_url': 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'matched_text': 'Artistic-2.0'}], 'identifier': 'artistic_2_0-2ca48deb-7b37-fc93-edd9-b349ad073774'}]",,,,"- Artistic-2.0
+",,,,https://www.npmjs.com/package/npm,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,https://registry.npmjs.org/npm/2.13.5,npm_package_json,pkg:npm/npm@2.13.5

tests/formattedcode/data/csv/packages/expected.csv

@@ -1,4 +1,5 @@
 path,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__holder,package__declared_license_expression,package__declared_license_expression_spdx,package__license_detections,package__other_license_expression,package__other_license_expression_spdx,package__other_license_detections,package__extracted_license_statement,package__notice_text,package__file_references,package__extra_data,package__repository_homepage_url,package__repository_download_url,package__api_data_url,package__datasource_id,package__purl
 scan/,directory,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 scan/package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-scan/package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,,artistic-2.0,Artistic-2.0,"[{'license_expression': 'artistic-2.0', 'matches': [{'score': 50.0, 'start_line': 1, 'end_line': 1, 'matched_length': 3, 'match_coverage': 100.0, 'matcher': '1-hash', 'license_expression': 'artistic-2.0', 'rule_identifier': 'spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'rule_relevance': 50, 'rule_url': 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'matched_text': 'Artistic-2.0'}], 'identifier': 'artistic_2_0-2ca48deb-7b37-fc93-edd9-b349ad073774'}]",,,,['Artistic-2.0'],,,,https://www.npmjs.com/package/npm,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,https://registry.npmjs.org/npm/2.13.5,npm_package_json,pkg:npm/npm@2.13.5
+scan/package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,,artistic-2.0,Artistic-2.0,"[{'license_expression': 'artistic-2.0', 'matches': [{'score': 50.0, 'start_line': 1, 'end_line': 1, 'matched_length': 3, 'match_coverage': 100.0, 'matcher': '1-hash', 'license_expression': 'artistic-2.0', 'rule_identifier': 'spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'rule_relevance': 50, 'rule_url': 'https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_artistic-2.0_for_artistic-2.0.RULE', 'matched_text': 'Artistic-2.0'}], 'identifier': 'artistic_2_0-2ca48deb-7b37-fc93-edd9-b349ad073774'}]",,,,"- Artistic-2.0
+",,,,https://www.npmjs.com/package/npm,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,https://registry.npmjs.org/npm/2.13.5,npm_package_json,pkg:npm/npm@2.13.5

tests/licensedcode/data/licenses_reference_reporting/scan-matched-text-with-reference.expected.json

@@ -63,7 +63,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "['Artistic-2.0 OR MIT']",
+      "extracted_license_statement": "- Artistic-2.0 OR MIT\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {},
@@ -514,7 +514,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "['Artistic-2.0 OR MIT']",
+          "extracted_license_statement": "- Artistic-2.0 OR MIT\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/licensedcode/data/licenses_reference_reporting/scan-with-reference.expected.json

@@ -63,7 +63,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "['Artistic-2.0 OR MIT']",
+      "extracted_license_statement": "- Artistic-2.0 OR MIT\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {},
@@ -508,7 +508,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "['Artistic-2.0 OR MIT']",
+          "extracted_license_statement": "- Artistic-2.0 OR MIT\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/licensedcode/data/plugin_license/package/package.expected.json

@@ -63,7 +63,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "[{'type': 'MIT', 'url': 'http://github.com/mscdex/busboy/raw/master/LICENSE'}]",
+      "extracted_license_statement": "- type: MIT\n  url: http://github.com/mscdex/busboy/raw/master/LICENSE\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {},
@@ -190,7 +190,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "[{'type': 'MIT', 'url': 'http://github.com/mscdex/busboy/raw/master/LICENSE'}]",
+          "extracted_license_statement": "- type: MIT\n  url: http://github.com/mscdex/busboy/raw/master/LICENSE\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/bower/author-objects/expected.json

@@ -106,7 +106,7 @@
     "other_license_expression": null,
     "other_license_expression_spdx": null,
     "other_license_detections": [],
-    "extracted_license_statement": "['MIT', 'Apache 2.0', 'BSD-3-Clause']",
+    "extracted_license_statement": "- MIT\n- Apache 2.0\n- BSD-3-Clause\n",
     "notice_text": null,
     "source_packages": [],
     "file_references": [],

tests/packagedcode/data/bower/list-of-licenses/expected.json

@@ -99,7 +99,7 @@
     "other_license_expression": null,
     "other_license_expression_spdx": null,
     "other_license_detections": [],
-    "extracted_license_statement": "['MIT', 'Apache 2.0', 'BSD-3-Clause']",
+    "extracted_license_statement": "- MIT\n- Apache 2.0\n- BSD-3-Clause\n",
     "notice_text": null,
     "source_packages": [],
     "file_references": [],

tests/packagedcode/data/bower/scan-expected.json

@@ -107,7 +107,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "['MIT', 'Apache 2.0', 'BSD-3-Clause']",
+      "extracted_license_statement": "- MIT\n- Apache 2.0\n- BSD-3-Clause\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {},
@@ -287,7 +287,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "['MIT', 'Apache 2.0', 'BSD-3-Clause']",
+          "extracted_license_statement": "- MIT\n- Apache 2.0\n- BSD-3-Clause\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/build/buck/end2end-expected.json

@@ -96,7 +96,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "['LICENSE']",
+      "extracted_license_statement": "- LICENSE\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {},

tests/packagedcode/data/debian/basic-rootfs-expected.json

@@ -341,7 +341,7 @@
           "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
         }
       ],
-      "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+      "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
       "notice_text": null,
       "source_packages": [
         "pkg:deb/libndp"
@@ -711,7 +711,7 @@
               "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
             }
           ],
-          "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+          "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/debian/debian-container-layer.tar.xz.get-installed-expected.json

@@ -562,7 +562,7 @@
         "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
       }
     ],
-    "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+    "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
     "notice_text": null,
     "source_packages": [
       "pkg:deb/libndp"
@@ -757,7 +757,7 @@
                 "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
               }
             ],
-            "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+            "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
             "notice_text": null,
             "source_packages": [],
             "file_references": [],

tests/packagedcode/data/debian/debian-container-layer.tar.xz.scan-expected.json

@@ -341,7 +341,7 @@
           "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
         }
       ],
-      "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+      "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
       "notice_text": null,
       "source_packages": [
         "pkg:deb/libndp"
@@ -725,7 +725,7 @@
               "identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e"
             }
           ],
-          "extracted_license_statement": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
+          "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json

@@ -68,7 +68,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: MIT License']}",
+      "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: MIT License'\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
@@ -179,7 +179,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: MIT License']}",
+          "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: MIT License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/license_detection/reference-to-package/base.expected.json

@@ -70,7 +70,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+      "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
@@ -179,7 +179,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+          "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/license_detection/reference-to-package/google_appengine_sdk.expected.json

@@ -70,7 +70,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+      "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
@@ -162,7 +162,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+      "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
@@ -783,7 +783,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+          "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
@@ -1031,7 +1031,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+          "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
@@ -1443,7 +1443,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'classifiers': ['License :: OSI Approved :: BSD License']}",
+          "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: BSD License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json

@@ -83,7 +83,7 @@
       "other_license_expression": null,
       "other_license_expression_spdx": null,
       "other_license_detections": [],
-      "extracted_license_statement": "{'license': 'Apache 2.0', 'classifiers': ['License :: OSI Approved :: Apache Software License']}",
+      "extracted_license_statement": "license: Apache 2.0\nclassifiers:\n  - 'License :: OSI Approved :: Apache Software License'\n",
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
@@ -1386,7 +1386,7 @@
           "other_license_expression": null,
           "other_license_expression_spdx": null,
           "other_license_detections": [],
-          "extracted_license_statement": "{'license': 'Apache 2.0', 'classifiers': ['License :: OSI Approved :: Apache Software License']}",
+          "extracted_license_statement": "license: Apache 2.0\nclassifiers:\n  - 'License :: OSI Approved :: Apache Software License'\n",
           "notice_text": null,
           "source_packages": [],
           "file_references": [],

tests/packagedcode/data/m2/aopalliance/aopalliance/1.0/aopalliance-1.0.pom.package.json

@@ -48,7 +48,7 @@
   "other_license_expression": null,
   "other_license_expression_spdx": null,
   "other_license_detections": [],
-  "extracted_license_statement": "[{'name': 'Public Domain', 'url': None, 'comments': None, 'distribution': None}]",
+  "extracted_license_statement": "- name: Public Domain\n  url:\n  comments:\n  distribution:\n",
   "notice_text": null,
   "source_packages": [
     "pkg:maven/aopalliance/aopalliance@1.0?classifier=sources"

tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.3/aspectjrt-1.5.3.pom.package.json

@@ -67,7 +67,7 @@
   "other_license_expression": null,
   "other_license_expression_spdx": null,
   "other_license_detections": [],
-  "extracted_license_statement": "[{'name': 'Eclipse Public License - v 1.0', 'url': 'http://www.eclipse.org/legal/epl-v10.html', 'comments': None, 'distribution': 'repo'}]",
+  "extracted_license_statement": "- name: Eclipse Public License - v 1.0\n  url: http://www.eclipse.org/legal/epl-v10.html\n  comments:\n  distribution: repo\n",
   "notice_text": null,
   "source_packages": [
     "pkg:maven/aspectj/aspectjrt@1.5.3?classifier=sources"