Merge pull request #1285 from nexB/211-scan-headers

Add new "headers" top level attribute

Author: pombredanne

etc/scripts/add_spdx_keys_as_rules.py

@@ -221,7 +221,7 @@ def add_spdx_key_rules():
 
     """
     by_key = load_licenses(with_deprecated=True)
-    by_spdx_key = synclic.get_by_spdx(by_key.values(), include_other=True)
+    by_spdx_key = synclic.get_licenses_by_spdx_key(by_key.values(), include_other=True)
 
     click.echo('Checking all SPDX ids.')
     # first accumulate non-matches

etc/scripts/synclic.py

@@ -76,12 +76,12 @@ class ScanCodeLicenses(object):
 
     def __init__(self):
         self.by_key = load_licenses(with_deprecated=True)
-        self.by_spdx_key = get_by_spdx(self.by_key.values())
+        self.by_spdx_key = get_licenses_by_spdx_key(self.by_key.values())
 
         # TODO: not yet used
         foreign_dir = join(licensedcode.models.data_dir, 'non-english', 'licenses')
         self.non_english_by_key = load_licenses(foreign_dir, with_deprecated=True)
-        self.non_english_by_spdx_key = get_by_spdx(self.non_english_by_key.values())
+        self.non_english_by_spdx_key = get_licenses_by_spdx_key(self.non_english_by_key.values())
 
     def clean(self):
         """
@@ -105,7 +105,7 @@ def _clean(licenses):
             _clean(lics)
 
 
-def get_by_spdx(licenses, include_other=False):
+def get_licenses_by_spdx_key(licenses, include_other=False):
     """
     Return a mapping of {spdx_key: license object} given a sequence of License objects.
     """
@@ -966,9 +966,9 @@ def synchronize_licenses(scancode_licenses, external_source, use_spdx_key=False,
 
     if use_spdx_key:
         scancodes_by_key = scancode_licenses.by_spdx_key
-        externals_by_key = get_by_spdx(externals_by_key.values())
+        externals_by_key = get_licenses_by_spdx_key(externals_by_key.values())
 
-    externals_by_spdx_key = get_by_spdx(externals_by_key.values())
+    externals_by_spdx_key = get_licenses_by_spdx_key(externals_by_key.values())
 
     # track changes with sets of license keys
     same = set()

src/formattedcode/output_csv.py

@@ -54,7 +54,7 @@ def is_enabled(self, csv, **kwargs):
         return csv
 
     def process_codebase(self, codebase, csv, **kwargs):
-        results = self.get_results(codebase, **kwargs)
+        results = self.get_files(codebase, **kwargs)
         write_csv(results, csv)
 
 

src/formattedcode/output_html.py

@@ -50,7 +50,6 @@
 from commoncode.fileutils import fsencode
 from commoncode.fileutils import parent_directory
 from commoncode.system import on_linux
-from formattedcode.utils import get_headings
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from scancode import CommandLineOption
@@ -81,8 +80,8 @@ def is_enabled(self, html, **kwargs):
         return html
 
     def process_codebase(self, codebase, html, **kwargs):
-        results = self.get_results(codebase, **kwargs)
-        _files_count, version, _notice, _scan_start, _options = get_headings(codebase)
+        results = self.get_files(codebase, **kwargs)
+        version = codebase.get_or_create_current_header().tool_version
         write_templated(html, results, version, template_or_format='html')
 
 
@@ -114,8 +113,8 @@ def is_enabled(self, custom_output, custom_template, **kwargs):
         return custom_output and custom_template
 
     def process_codebase(self, codebase, custom_output, custom_template, **kwargs):
-        results = self.get_results(codebase, **kwargs)
-        _files_count, version, _notice, _start, _options = get_headings(codebase)
+        results = self.get_files(codebase, **kwargs)
+        version = codebase.get_or_create_current_header().tool_version
 
         if on_linux:
             custom_template = fsencode(custom_template)
@@ -143,8 +142,8 @@ def is_enabled(self, html_app, **kwargs):
         return html_app
 
     def process_codebase(self, codebase, input, html_app, **kwargs):  # NOQA
-        results = self.get_results(codebase, **kwargs)
-        _files_count, version, _notice, _start, _options = get_headings(codebase)
+        results = self.get_files(codebase, **kwargs)
+        version = codebase.get_or_create_current_header().tool_version
         html_app.write(as_html_app(html_app, input, version))
         create_html_app_assets(results, html_app)
 

src/formattedcode/output_json.py

@@ -29,7 +29,6 @@
 
 import simplejson
 
-from formattedcode.utils import get_headings
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from scancode import CommandLineOption
@@ -77,8 +76,8 @@ def is_enabled(self, output_json, **kwargs):
         return output_json
 
     def process_codebase(self, codebase, output_json, **kwargs):
-        results = self.get_results(codebase, **kwargs)
-        write_json(codebase, results, output_file=output_json, pretty=False)
+        files = self.get_files(codebase, **kwargs)
+        write_json(codebase, files, output_file=output_json, pretty=False)
 
 
 @output_impl
@@ -97,36 +96,29 @@ def is_enabled(self, output_json_pp, **kwargs):
         return output_json_pp
 
     def process_codebase(self, codebase, output_json_pp, **kwargs):
-        results = self.get_results(codebase, **kwargs)
-        write_json(codebase, results, output_file=output_json_pp, pretty=True)
+        files = self.get_files(codebase, **kwargs)
+        write_json(codebase, files, output_file=output_json_pp, pretty=True, **kwargs)
 
 
-def write_json(codebase, results, output_file,
+def write_json(codebase, files, output_file,
                include_summary=False, include_score=False,
-               pretty=False):
+               pretty=False, **kwargs):
+    # NOTE: we write as binary, not text
 
-    files_count, version, notice, scan_start, options = get_headings(codebase)
-
-    scan = OrderedDict([
-        ('scancode_notice', notice),
-        ('scancode_version', version),
-        ('scancode_options', options),
-        ('scan_start', scan_start),
-        ('files_count', files_count),
-        # FIXME: we are missing top level codebase ERRORs!!!
-    ])
+    codebase.add_files_count_to_current_header()
+    scan = OrderedDict([(b'headers', codebase.get_headers()), ])
 
     # add codebase toplevel attributes such as summaries
     if codebase.attributes:
         scan.update(codebase.attributes.to_dict())
 
     if TRACE:
-        logger_debug('write_json: results')
-        results = list(results)
+        logger_debug('write_json: files')
+        files = list(files)
         from pprint import pformat
-        logger_debug(pformat(results))
+        logger_debug(pformat(files))
 
-    scan['files'] = results
+    scan[b'files'] = files
 
     kwargs = dict(iterable_as_array=True, encoding='utf-8')
     if pretty:

src/formattedcode/output_jsonlines.py

@@ -25,11 +25,8 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from collections import OrderedDict
-
 import simplejson
 
-from formattedcode.utils import get_headings
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from scancode import CommandLineOption
@@ -57,32 +54,31 @@ def is_enabled(self, output_json_lines, **kwargs):
         return output_json_lines
 
     def process_codebase(self, codebase, output_json_lines, **kwargs):
-        results = self.get_results(codebase, **kwargs)
-        files_count, version, notice, scan_start, options = get_headings(codebase)
+        #NOTE: we write as binary, not text
+        files = self.get_files(codebase, **kwargs)
+
+        codebase.add_files_count_to_current_header()
 
-        header = dict(header=OrderedDict([
-            ('scancode_notice', notice),
-            ('scancode_version', version),
-            ('scancode_options', options),
-            ('scan_start', scan_start),
-            ('files_count', files_count)
-        ]))
+        headers = dict(headers=codebase.get_headers())
 
-        kwargs = dict(
+        simplejson_kwargs = dict(
             iterable_as_array=True,
             encoding='utf-8',
             separators=(b',', b':',)
         )
-        output_json_lines.write(simplejson.dumps(header, **kwargs))
+        output_json_lines.write(
+            simplejson.dumps(headers, **simplejson_kwargs))
         output_json_lines.write(b'\n')
 
         for name, value in codebase.attributes.to_dict().items():
             if value:
                 smry = {name: value}
-                output_json_lines.write(simplejson.dumps(smry, **kwargs))
+                output_json_lines.write(
+                    simplejson.dumps(smry, **simplejson_kwargs))
                 output_json_lines.write(b'\n')
 
-        for scanned_file in results:
-            scanned_file_line = {'files': [scanned_file]}
-            output_json_lines.write(simplejson.dumps(scanned_file_line, **kwargs))
+        for scanned_file in files:
+            scanned_file_line = {b'files': [scanned_file]}
+            output_json_lines.write(
+                simplejson.dumps(scanned_file_line, **simplejson_kwargs))
             output_json_lines.write(b'\n')

src/formattedcode/output_spdx.py

@@ -46,7 +46,6 @@
 from spdx.utils import SPDXNone
 from spdx.version import Version
 
-from formattedcode.utils import get_headings
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
 from scancode import CommandLineOption
@@ -101,12 +100,12 @@ def _patch_license_list():
         from spdx.config import LICENSE_MAP
         from licensedcode.models import load_licenses
         licenses = load_licenses(with_deprecated=True)
-        spdx_licenses = get_by_spdx(licenses.values())
+        spdx_licenses = get_licenses_by_spdx_key(licenses.values())
         LICENSE_MAP.update(spdx_licenses)
         _spdx_list_is_patched = True
 
 
-def get_by_spdx(licenses):
+def get_licenses_by_spdx_key(licenses):
     """
     Return a mapping of {spdx_key: license object} given a sequence of License
     objects.
@@ -149,9 +148,13 @@ def is_enabled(self, spdx_tv, info, **kwargs):
 
     def process_codebase(self, codebase, spdx_tv, **kwargs):
         input = kwargs.get('input', '')  # NOQA
-        results = self.get_results(codebase, **kwargs)
-        _files_count, version, notice, _scan_start, _options = get_headings(codebase)
-        write_spdx(spdx_tv, results, version, notice, input, as_tagvalue=True)
+        files = self.get_files(codebase, **kwargs)
+        header = codebase.get_or_create_current_header()
+        tool_name = header.tool_name
+        tool_version = header.tool_version
+        notice = header.notice
+        write_spdx(
+            spdx_tv, files, tool_name, tool_version, notice, input, as_tagvalue=True)
 
 
 @output_impl
@@ -171,13 +174,16 @@ def is_enabled(self, spdx_rdf, info, **kwargs):
 
     def process_codebase(self, codebase, spdx_rdf, **kwargs):
         input = kwargs.get('input', '')  # NOQA
-        results = self.get_results(codebase, **kwargs)
-        _files_count, version, notice, _scan_start, _options = get_headings(codebase)
-        write_spdx(spdx_rdf, results, version, notice, input, as_tagvalue=False)
+        files = self.get_files(codebase, **kwargs)
+        header = codebase.get_or_create_current_header()
+        tool_name = header.tool_name
+        tool_version = header.tool_version
+        notice = header.notice
+        write_spdx(
+            spdx_rdf, files, tool_name, tool_version, notice, input, as_tagvalue=False)
 
 
-def write_spdx(output_file, results, scancode_version, scancode_notice,
-               input_file, as_tagvalue=True):
+def write_spdx(output_file, files, tool_name, tool_version, notice, input_file, as_tagvalue=True):
     """
     Write scan output as SPDX Tag/value or RDF.
     """
@@ -191,9 +197,9 @@ def write_spdx(output_file, results, scancode_version, scancode_notice,
         input_path = dirname(absinput)
 
     doc = Document(Version(2, 1), License.from_identifier('CC0-1.0'))
-    doc.comment = scancode_notice
-
-    doc.creation_info.add_creator(Tool('ScanCode ' + scancode_version))
+    doc.comment = notice
+    tool_name = tool_name or 'ScanCode'
+    doc.creation_info.add_creator(Tool(tool_name + ' ' + tool_version))
     doc.creation_info.set_created_now()
 
     package = doc.package = Package(
@@ -208,7 +214,7 @@ def write_spdx(output_file, results, scancode_version, scancode_notice,
     all_files_have_no_copyright = True
 
     # FIXME: this should walk the codebase instead!!!
-    for file_data in results:
+    for file_data in files:
         # Construct the absolute path in case we need to access the file
         # to calculate its SHA1.
         file_entry = File(join(input_path, file_data.get('path')))

src/formattedcode/utils.py

@@ -91,9 +91,10 @@ def process_codebase(self, codebase, output, **kwargs):
         raise NotImplementedError
 
     @classmethod
-    def get_results(cls, codebase, **kwargs):
+    def get_files(cls, codebase, **kwargs):
         """
-        Return an iterable of serialized scan results from a codebase.
+        Return an iterable of serialized files mapping from a codebase.
+        Include "info", "timing" and strip root as needed.
         """
         # FIXME: serialization SHOULD NOT be needed: only some format need it
         # (e.g. JSON) and only these should serialize