#267 wip: initial, untested and not working prototype

 * cache scan results on disk
 * stream json at the end 

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/scancode/__init__.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -23,3 +23,21 @@
 #  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
 
 __version__ = '2.0.0rc2'
+
+from os.path import dirname
+from os.path import abspath
+from os.path import getsize
+from os.path import getmtime
+from os.path import join
+from os.path import exists
+
+from commoncode import fileutils
+
+scan_src_dir = abspath(dirname(__file__))
+src_dir = dirname(scan_src_dir)
+root_dir = dirname(src_dir)
+cache_dir = join(root_dir, '.cache')
+scans_cache_dir = join(cache_dir, 'scan_results_caches')
+
+if not exists(scans_cache_dir):
+    fileutils.create_dir(scans_cache_dir)

src/scancode/api.py

@@ -142,11 +142,11 @@ def get_licenses(location, min_score=0):
             result['matched_rule']['licenses'] = match.rule.licenses
             # TODO: add debug details such as matcher
             # result['matched_rule']['matcher'] = match.matcher
-            
+
             yield result
 
 
-def get_file_infos(location):
+def get_file_infos(location, as_list=True):
     """
     Return a list of dictionaries of informations collected from the file or
     directory at location.
@@ -177,7 +177,10 @@ def get_file_infos(location):
     infos['is_media'] = is_file and T.is_media or None
     infos['is_source'] = is_file and T.is_source or None
     infos['is_script'] = is_file and T.is_script or None
-    return [infos]
+    if as_list:
+        return [infos]
+    else:
+        return infos
 
 
 def get_package_infos(location):

src/scancode/cache.py

@@ -0,0 +1,126 @@
+#
+# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import, print_function
+
+import os
+
+from commoncode import fileutils
+from commoncode import timeutils
+
+from scancode import scans_cache_dir
+from collections import OrderedDict
+
+"""
+Caching scans on disk: A cache of all the scan results.
+
+Each scan results for a file or directory is cached on disk.
+
+The approach is to use to cache:
+ - the results of a scan, excluding file infos keyed by the hash of a scanned file
+ - the file infos, keyed by the path of a scanned file
+
+Once a scan is completed, we iterate the caches to output the scan results using this
+procedure: iterate the cached file infos and for each lookup the scan details in the
+cached scan results. This iteration is driving the final streaming of results to the
+output format (e.g. JSON).
+
+Finally once a scan is completed the cache is destroyed to free up disk space.
+"""
+
+
+class ScanCache(object):
+    """
+    A file-based cache for scan results.
+    This is NOT thread-safe, but is multi-process safe.
+    """
+    def __init__(self, cache_dir):
+        fileutils.create_dir(cache_dir)
+
+        # create a unique temp directory in cache_dir
+        self.cache_base_dir = fileutils.get_temp_dir(cache_dir, prefix=timeutils.time2tstamp()+'-')
+
+        # and subdirs for infos and scans caches
+        self.cache_infos_dir = os.path.join(self.cache_base_dir, 'infos')
+        fileutils.create_dir(self.cache_infos_dir)
+        self.cache_scans_dir = os.path.join(self.cache_base_dir, 'scans')
+        fileutils.create_dir(self.cache_scans_dir)
+
+        # and finially cache instances
+        from diskcache import Cache
+        self.infos = Cache(self.cache_infos_dir)
+        self.scans = Cache(self.cache_scans_dir)
+
+    def scan_key(self, path, file_infos):
+        """
+        Return a scan cache key for a path and file_infos.
+        """
+        sha1 = file_infos['sha1']
+        # we may eventually store directories, in which case we use the path as a key
+        return sha1 or path
+
+    def put_infos(self, path, file_infos):
+        """
+        Put file_infos for path in the cache and return True if the file referenced
+        in file_infos has already been scanned or False otherwise.
+        """
+        self.infos.set(path, file_infos)
+        return self.scan_key(path, file_infos) in self.scans
+
+    def put_scan(self, path, file_infos, scan_result):
+        """
+        Put scan_result in the cache. Also put  file_infos in the cache if needed.
+        """
+        self.infos.add(path, file_infos)
+        scan_key = self.scan_key(path, file_infos)
+        self.scans.add(scan_key, scan_result)
+
+    def iterate(self, with_infos=True):
+        """
+        Return an iterator of scan data for all cached scans e.g. the whole cache.
+        """
+        for path in self.infos:
+            file_infos = self.infos[path]
+            scan_result = OrderedDict(path=path)
+            if with_infos:
+                # infos is always collected but only returnedd if asked:
+                # we flatten these as direct attributes of a file object
+                scan_result.update(file_infos.items())
+
+            scan_key = self.scan_key(path, file_infos)
+            scan_details = self.scans[scan_key]
+            scan_result.update(scan_details)
+            yield scan_result
+
+    def clear(self, *args):
+        """
+        Purge the cache by deleting the corresponding cached data files.
+        """
+        self.infos.close()
+        self.scans.close()
+        fileutils.delete(self.cache_base_dir)
+
+
+def get_scans_cache():
+    return ScanCache(cache_dir=scans_cache_dir)

src/scancode/cli.py

@@ -26,13 +26,13 @@
 
 from collections import OrderedDict
 from functools import partial
-import json
 import os
 import sys
 from types import GeneratorType
 
 import click
 from click.termui import style
+import simplejson as json
 
 from commoncode import ignore
 from commoncode import fileutils
@@ -43,6 +43,8 @@
 from scancode import __version__ as version
 from scancode import utils
 
+from scancode.cache import get_scans_cache
+
 from scancode.format import as_template
 from scancode.format import as_html_app
 from scancode.format import create_html_app_assets
@@ -260,15 +262,27 @@ def scancode(ctx, input, output_file, copyright, license, package,
         license = True
         package = True
 
-    results = scan(input, copyright, license, package, email, url, info, license_score, verbose, quiet)
-    save_results(results, format, input, output_file)
+    scans_cache = get_scans_cache()
+    try:
+        files_count, results = scan(input, copyright, license, package, email, url, info, license_score, verbose, quiet, scans_cache)
+        save_results(files_count, results, format, input, output_file)
+    finally:
+        # cleanup
+        scans_cache.clear()
 
 
 def scan(input_path, copyright=True, license=True, package=True,
-         email=False, url=False, info=True, license_score=0, verbose=False, quiet=False):
+         email=False, url=False, info=True, license_score=0,
+         verbose=False, quiet=False,
+         scans_cache=None):
     """
-    Do the scans proper, return a list of file_results.
+    Return a tuple of (file_count, scan_results) where scan_results is an iterable.
+    Run each requested scan proper: each individual file scan is cached on disk to
+    free memory. Then the whole set of scans is loaded from the cache and streamed at
+    the end.
     """
+    assert scans_cache
+
     # save paths to report paths relative to the original input
     original_input = fileutils.as_posixpath(input_path)
     abs_input = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input_path)))
@@ -278,16 +292,14 @@ def scan(input_path, copyright=True, license=True, package=True,
     # note: "flag and function" expressions return the function if flag is True
     # note: the order of the scans matters to show things in logical order
     scanners = OrderedDict([
-        ('infos' , info and get_file_infos),
+        # ('infos' , info and get_file_infos),
         ('licenses' , license and get_licenses_with_score),
         ('copyrights' , copyright and get_copyrights),
         ('packages' , package and get_package_infos),
         ('emails' , email and get_emails),
         ('urls' , url and get_urls),
     ])
 
-    file_results = []
-
     # note: we inline progress display functions to close on some args
 
     def scan_start():
@@ -323,48 +335,64 @@ def scan_end():
                                quiet=quiet
                                ) as progressive_resources:
 
-        for resource in progressive_resources:
+        for files_count, resource in enumerate(progressive_resources):
+            # actual path of the file being scanned
             res = fileutils.as_posixpath(resource)
-
             # fix paths: keep the path as relative to the original input
             relative_path = utils.get_relative_path(original_input, abs_input, res)
-            scan_result = OrderedDict(path=relative_path)
-            # Should we yield instead?
-            scan_result.update(scan_one(res, scanners))
-            file_results.append(scan_result)
 
-    # TODO: eventually merge scans for the same files path...
-    # TODO: fix absolute paths as relative to original input argument...
+            # always fetch infos and cache.
+            infos = scan_infos(res)
+            is_cached = scans_cache.put_infos(relative_path, infos)
 
-    return file_results
+            # Skip other scans if already cached
+            if is_cached:
+                continue
+            scan_result = scan_one(res, scanners)
+            scans_cache.put_scan(relative_path, infos, scan_result)
+        files_count += 1
+    return files_count, scans_cache.iterate(with_infos=info)
+
+
+def scan_infos(input_file):
+    """
+    Scan one file or directory and return file_infos data.
+    """
+    infos = OrderedDict()
+    try:
+        infos = get_file_infos(input_file, as_list=False)
+    except Exception, e:
+        raise
+        # never fail but instead add an error message.
+        # FIXME: this should not be stored at the individual scan level
+        return dict(errors=e.message)
+    return infos
 
 
 def scan_one(input_file, scans):
     """
     Scan one file or directory and return a scanned data, calling every scan in
     the `scans` mapping of (scan name -> scan function).
     """
-    scanned_file = OrderedDict()
+    scan_result = OrderedDict()
     for scan_name, scan_func in scans.items():
         if not scan_func:
             continue
         try:
             scan = scan_func(input_file)
+            # consume generators
             if isinstance(scan, GeneratorType):
                 scan = list(scan)
-            # this is special and we flatten these as direct attributes of a file object
-            if scan_name == 'infos':
-                for file_infos in scan:
-                    scanned_file.update(file_infos.items())
-            else:
-                scanned_file[scan_name] = scan
+            scan_result[scan_name] = scan
         except Exception, e:
+            raise
             # never fail but instead add an error message.
-            scanned_file[scan_name] = {'errors': e.message}
-    return scanned_file
+            # FIXME: this should not be stored at the individual scan level
+            scan_result[scan_name] = {'errors': e.message}
+    return scan_result
 
 
-def save_results(scanned_files, format, input, output_file):
+def save_results(files_count, scanned_files, format, input, output_file):
     """
     Save results to file or screen.
     """
@@ -397,9 +425,10 @@ def save_results(scanned_files, format, input, output_file):
         meta = OrderedDict()
         meta['scancode_notice'] = acknowledgment_text_json
         meta['scancode_version'] = version
-        meta['files_count'] = len(scanned_files)
+        meta['files_count'] = files_count
         # TODO: add scanning options to meta
         meta['files'] = scanned_files
-        output_file.write(json.dumps(meta, indent=2))
+        # json.dump(meta, output_file, indent=2)
+        json.dump(meta, output_file, indent=2 * ' ', iterable_as_array=True)
     else:
         raise Exception('unknown format')

src/scancode/format.py

@@ -34,6 +34,8 @@
 from os.path import isfile
 from os.path import join
 
+import simplejson as json
+
 from commoncode import fileutils
 
 
@@ -130,10 +132,10 @@ def create_html_app_assets(results, output_file):
         fileutils.copytree(assets_dir, target_dir)
 
         # write json data
-        import json
         root_path, assets_dir = get_html_app_files_dirs(output_file)
         with open(join(root_path, assets_dir, 'data.json'), 'w') as f:
-            f.write('data=' + json.dumps(results))
+            f.write('data=')
+            json.dump(results, f, iterable_as_array=True)
 
         # create help file
         with open(join(root_path, assets_dir, 'help.html'), 'w') as f: