Merge pull request #1415 from nexB/1398-json2csv

Accept non-standard package data in CSV #1398

Author: pombredanne

src/formattedcode/output_csv.py

@@ -37,6 +37,37 @@
 from scancode import FileOptionType
 from scancode import OUTPUT_GROUP
 
+# Python 2 and 3 support
+try:
+    # Python 2
+    unicode
+    str = unicode  # NOQA
+except NameError:
+    # Python 3
+    unicode = str  # NOQA
+    basestring = str  # NOQA
+
+
+# Tracing flags
+TRACE = False
+
+
+def logger_debug(*args):
+    pass
+
+
+if TRACE:
+    import sys
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, unicode)
+                                     and a or repr(a) for a in args))
+
 
 @output_impl
 class CsvOutput(OutputPlugin):
@@ -199,114 +230,98 @@ def collect_keys(mapping, key_group):
             yield flat
 
 
-def flatten_package(_package, path, prefix='package__'):
+def get_package_columns(_columns=set()):
+    """
+    Return (and cache in_columns) a set of package column names included in the
+    CSV output.
+    Some columsn are excluded for now such as lists of mappings: these do not
+    serialize well to CSV
+    """
+    if _columns:
+        return _columns
+
+    from packagedcode.models import Package
 
     # exclude some columns for now that contain list of items
-    excluded_package_columns = {
+    excluded_columns = {
         # list of strings
-        'download_checksums',
         'keywords',
         # list of dicts
         'parties',
         'dependencies',
+        'source_packages',
+    }
 
-        # comming from a match
+    # some extra columns for components
+    extra_columns = [
+        'components',
+        'owner_name',
+        'reference_notes',
+        'description',
+        'notice_filename',
+        'notice_url',
+    ]
 
-        # we have license_expression we do not need more
-        'licenses_summary',
-        'licenses',
-        'license_choices_expression',
-        'license_choices',
+    fields = Package.fields() + extra_columns
+    _columns = set(f for f in fields if f not in excluded_columns)
+    return _columns
 
-        'api_url',
-        'uuid',
-        'sha1',
-        'md5',
-        'owner',
 
-        'reference_notes',
-        'project',
-        'codescan_identifier',
-        'is_license_notice',
-        'is_copyright_notice',
-        'is_notice_in_codebase',
-        # 'notice_filename',
-        # 'notice_url',
-        'website_terms_of_use',
-        'is_active',
-        'curation_level',
-        'completion_level',
-        'guidance',
-        'admin_notes',
-        'ip_sensitivity_approved',
-        'affiliate_obligations',
-        'affiliate_obligation_triggers',
-        'concluded_license',
-        'legal_comments',
-        'legal_reviewed',
-        'approval_reference',
-        'distribution_formats_allowed',
-        'acceptable_linkage',
-        'export_restrictions',
-        'approved_download_location',
-        'approved_community_interaction',
-        'urn',
-        'created_date',
-        'last_modified_date',
-        'dataspace',
-        'external_references',
-        'display_name',
-        'notes',
-        'origin_date',
-        'sublicense_allowed',
-    }
+def flatten_package(_package, path, prefix='package__'):
+
+    # known package columns
+
+    package_columns = get_package_columns()
 
     pack = OrderedDict(Resource=path)
     for k, val in _package.items():
-        # FIXME: we only keep for now some of the value collections
-        if k in excluded_package_columns:
+        if k not in package_columns:
             continue
+
         # prefix columns with "package__"
         nk = prefix + k
 
         if k == 'version':
-            if val:
+            val = val or ''
+            if val and not val.lower().startswith('v'):
                 # prefix versions with a v to avoid spreadsheet tools to mistake
-                # a version for a number or date.
+                # a version for a number or date when reading CSVs (common with
+                # Excel and LibreOffice).
                 val = 'v ' + val
-                pack[nk] = val
-            else:
-                pack[nk] = ''
+            pack[nk] = val
             continue
 
-        # these may come from a match
-
+        # these may come from a component matched
         if k == 'components' and val and isinstance(val, list):
-            for compo in val:
-                for compo_key, compo_val in compo.items():
-                    if compo_key in excluded_package_columns:
+            for component in val:
+                for component_key, component_val in component.items():
+                    if component_key not in package_columns:
                         continue
 
-                    compo_nk = nk + '__' + compo_key
+                    component_new_key = nk + '__' + component_key
 
-                    if compo_val is None:
-                        pack[compo_nk] = ''
+                    if component_val is None:
+                        pack[component_new_key] = ''
                         continue
 
-                    if not isinstance(compo_val, basestring):
-                        compo_val = repr(compo_val)
-                    existing = pack.get(compo_nk) or []
-                    pack[compo_nk] = ' \n'.join(existing + [compo_val])
-            continue
+                    if isinstance(component_val, list):
+                        component_val = '\n'.join(component_val)
+
+                    if not isinstance(component_val, str):
+                        component_val = repr(component_val)
+
+                    existing = pack.get(component_new_key) or []
+                    if not isinstance(existing, list):
+                        existing = [existing]
+
+                    if TRACE:
+                        logger_debug('component_new_key:', component_new_key, 'existing:', type(existing), repr(existing))
+                        logger_debug('component_key:', component_key, 'component_val:', type(component_val), repr(component_val))
 
-        if k == 'match':
-            for mk, mval in val.items():
-                match_nk = nk + '__' + mk
-                pack[match_nk] = mval
+                    pack[component_new_key] = ' \n'.join(existing + [component_val])
             continue
 
         # everything else
-        # collect all the keys
 
         pack[nk] = ''
 

src/packagedcode/models.py

@@ -125,6 +125,13 @@ def create(cls, ignore_unknown=True, **kwargs):
             kwargs = {k: v for k, v in kwargs.items() if k in known_attr}
         return cls(**kwargs)
 
+    @classmethod
+    def fields(cls):
+        """
+        Return a list of field names defined on this model.
+        """
+        return [a.name for a in attr.fields(cls)]
+
 
 party_person = 'person'
 # often loosely defined

tests/formattedcode/data/csv/flatten_scan/full.json-expected

@@ -147,15 +147,10 @@
         "package__download_url": "",
         "package__bug_tracking_url": "",
         "package__code_view_url": "",
-        "package__vcs_tool": "",
-        "package__vcs_repository": "",
-        "package__vcs_revision": "",
         "package__copyright": "",
         "package__license_expression": "",
-        "package__declared_licensing": "",
         "package__notice_text": "",
-        "package__contains_source_code": "",
-        "package__source_packages": ""
+        "package__contains_source_code": ""
     },
     {
         "Resource": "/samples/JGroups/",
@@ -1418,15 +1413,10 @@
         "package__download_url": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz",
         "package__bug_tracking_url": "https://github.com/visionmedia/node-cookie-signature/issues",
         "package__code_view_url": "",
-        "package__vcs_tool": "git",
-        "package__vcs_repository": "https://github.com/visionmedia/node-cookie-signature.git",
-        "package__vcs_revision": "",
         "package__copyright": "",
         "package__license_expression": "",
-        "package__declared_licensing": "",
         "package__notice_text": "",
-        "package__contains_source_code": "",
-        "package__source_packages": ""
+        "package__contains_source_code": ""
     },
     {
         "Resource": "/samples/srp/scan/json2csv.rb",

tests/formattedcode/data/csv/flatten_scan/package_license_value_null.json-expected

@@ -39,14 +39,9 @@
         "package__download_url": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz",
         "package__bug_tracking_url": "https://github.com/visionmedia/node-cookie-signature/issues",
         "package__code_view_url": "",
-        "package__vcs_tool": "git",
-        "package__vcs_repository": "https://github.com/visionmedia/node-cookie-signature.git",
-        "package__vcs_revision": "",
         "package__copyright": "",
         "package__license_expression": "",
-        "package__declared_licensing": "",
         "package__notice_text": "",
-        "package__contains_source_code": "",
-        "package__source_packages": ""
+        "package__contains_source_code": ""
     }
 ]

tests/formattedcode/data/csv/livescan/expected.csv

@@ -1,21 +1,21 @@
-Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,files_count,dirs_count,size_count,scan_errors,license_expression,license__key,license__score,license__name,license__short_name,license__category,license__is_exception,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_expression,matched_rule__licenses,matched_rule__is_license_text,matched_rule__is_license_notice,matched_rule__is_license_reference,matched_rule__is_license_tag,copyright,copyright_holder,email,url,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__manifest_path,package__contains_source_code,package__source_packages,package__purl,package__repository_homepage_url,package__repository_download_url,package__api_data_url
-/json2csv.rb,file,json2csv.rb,json2csv,.rb,912,2019-01-07,1236469a06a2bacbdd8e172ad718482af5b0a936,1307c281e0b153202e291b217eab85d5,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,apache-2.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,100.00,Apache License 2.0,Apache 2.0,Permissive,False,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,13,apache-2.0_7.RULE,apache-2.0,[u'apache-2.0'],False,True,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,,,,Copyright (c) 2017 nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,,,,,nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,,,,,http://nexb.com/,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,,,,,https://github.com/nexB/scancode-toolkit/,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,8,,,,,,,,,,,http://www.apache.org/licenses/LICENSE-2.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/license,file,license,license,,679,2018-04-11,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/license,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/license,,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GNU General Public License 2.0 or later,GPL 2.0 or later,Copyleft,False,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0-or-later,https://spdx.org/licenses/GPL-2.0-or-later,1,12,gpl-2.0-plus.LICENSE,gpl-2.0-plus,[u'gpl-2.0-plus'],True,False,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,file,package.json,package,.json,2200,2018-04-11,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",,False,True,False,False,False,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,mit,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,99.40,MIT License,MIT License,Permissive,False,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_31.RULE,mit,[u'mit'],True,False,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,24,,,,,,,,Copyright (c) 2012 LearnBoost <tj@learnboost.com>,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,24,,,,,,,,,LearnBoost,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12,,,,,,,,,,tj@learnboost.com,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,16,,,,,,,,,,,https://github.com/visionmedia/node-cookie-signature.git,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,27,,,,,,,,,,,https://github.com/visionmedia/node-cookie-signature/issues,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,npm,,cookie-signature,v 1.0.3,,,JavaScript,Sign and unsign cookies,,,https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz,,,,https://github.com/visionmedia/node-cookie-signature/issues,,git+https://github.com/visionmedia/node-cookie-signature.git,,,,,,,,pkg:npm/cookie-signature@1.0.3,https://www.npmjs.com/package/cookie-signature,https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz,https://registry.npmjs.org/cookie-signature/1.0.3
+Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,files_count,dirs_count,size_count,scan_errors,license_expression,license__key,license__score,license__name,license__short_name,license__category,license__is_exception,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_expression,matched_rule__licenses,matched_rule__is_license_text,matched_rule__is_license_notice,matched_rule__is_license_reference,matched_rule__is_license_tag,copyright,copyright_holder,email,url,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__manifest_path,package__contains_source_code
+/json2csv.rb,file,json2csv.rb,json2csv,.rb,912,2018-11-15,1236469a06a2bacbdd8e172ad718482af5b0a936,1307c281e0b153202e291b217eab85d5,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,apache-2.0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,100.00,Apache License 2.0,Apache 2.0,Permissive,False,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,13,apache-2.0_7.RULE,apache-2.0,[u'apache-2.0'],False,True,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,,,,Copyright (c) 2017 nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,,,,,nexB Inc. and others.,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,,,,,http://nexb.com/,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,,,,,https://github.com/nexB/scancode-toolkit/,,,,,,,,,,,,,,,,,,,,,,,,,
+/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,8,,,,,,,,,,,http://www.apache.org/licenses/LICENSE-2.0,,,,,,,,,,,,,,,,,,,,,,,,,
+/license,file,license,license,,679,2018-10-15,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/license,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/license,,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GNU General Public License 2.0 or later,GPL 2.0 or later,Copyleft,False,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0-or-later,https://spdx.org/licenses/GPL-2.0-or-later,1,12,gpl-2.0-plus.LICENSE,gpl-2.0-plus,[u'gpl-2.0-plus'],True,False,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,file,package.json,package,.json,2200,2018-10-15,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",,False,True,False,False,False,False,0,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,mit,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,99.40,MIT License,MIT License,Permissive,False,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_31.RULE,mit,[u'mit'],True,False,False,False,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,24,,,,,,,,Copyright (c) 2012 LearnBoost <tj@learnboost.com>,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,24,24,,,,,,,,,LearnBoost,,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12,,,,,,,,,,tj@learnboost.com,,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,16,16,,,,,,,,,,,https://github.com/visionmedia/node-cookie-signature.git,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,27,27,,,,,,,,,,,https://github.com/visionmedia/node-cookie-signature/issues,,,,,,,,,,,,,,,,,,,,,,,,,
+/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,npm,,cookie-signature,v 1.0.3,,,JavaScript,Sign and unsign cookies,,,https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz,,,,,,https://github.com/visionmedia/node-cookie-signature/issues,,git+https://github.com/visionmedia/node-cookie-signature.git,,,,,,