Rename resource_cache_index to results_cache_index

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/scancode/cli.py

@@ -65,14 +65,13 @@ class WindowsError(Exception):
 from scancode import notice
 from scancode import print_about
 from scancode import Scanner
-from scancode import resource_cache
+from scancode import results_cache
 from scancode.help import epilog_text
 from scancode.help import examples_text
 from scancode.interrupt import DEFAULT_TIMEOUT
 from scancode.interrupt import fake_interruptible
 from scancode.interrupt import interruptible
 from scancode.pool import ScanCodeTimeoutError
-from scancode_config import USE_CACHED_RESULTS
 
 # Tracing flags
 TRACE = False
@@ -394,6 +393,13 @@ def default_processes():
     # not yet supported in Click 6.7 but added in PluggableCommandLineOption
     hidden=True,
     help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption)
+
+@click.option('--no-cached-results',
+    is_flag=True,
+    default=False,
+    hidden=True,
+    help='ScanCode will not use cached results during scan time.',
+    help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption)
 def scancode(
     ctx,
     input,  # NOQA
@@ -412,6 +418,7 @@ def scancode(
     test_error_mode,
     keep_temp_files,
     check_version,
+    no_cached_results,
     echo_func=echo_stderr,
     *args,
     **kwargs,
@@ -526,6 +533,7 @@ def scancode(
             return_results=False,
             echo_func=echo_func,
             outdated=outdated,
+            use_cached_results=not no_cached_results,
             *args,
             **kwargs
         )
@@ -569,6 +577,7 @@ def run_scan(
     pretty_params=None,
     plugin_options=plugin_options,
     outdated=None,
+    use_cached_results=False,
     *args,
     **kwargs
 ):
@@ -973,6 +982,7 @@ def echo_func(*_args, **_kwargs):
             verbose=verbose,
             kwargs=requested_options,
             echo_func=echo_func,
+            use_cached_results=use_cached_results,
         )
         success = success and scan_success
 
@@ -1165,6 +1175,7 @@ def run_scanners(
     verbose=False,
     kwargs=None,
     echo_func=echo_stderr,
+    use_cached_results=False,
 ):
     """
     Run the list of `stage` ScanPlugin `plugins` on `codebase`.
@@ -1206,7 +1217,8 @@ def run_scanners(
     # TODO: add CLI option to bypass cache entirely?
     scan_success = scan_codebase(
         codebase, scanners, processes, timeout,
-        with_timing=timing, progress_manager=progress_manager)
+        with_timing=timing, progress_manager=progress_manager,
+        use_cached_results=use_cached_results)
 
     # TODO: add progress indicator
     # run the process codebase of each scan plugin (most often a no-op)
@@ -1244,6 +1256,7 @@ def scan_codebase(
     with_timing=False,
     progress_manager=None,
     echo_func=echo_stderr,
+    use_cached_results=False,
 ):
     """
     Run the `scanners` Scanner objects on the `codebase` Codebase. Return True
@@ -1273,7 +1286,8 @@ def scan_codebase(
         scanners=scanners,
         timeout=timeout,
         with_timing=with_timing,
-        with_threading=use_threading
+        with_threading=use_threading,
+        use_cached_results=use_cached_results,
     )
 
     if TRACE:
@@ -1406,6 +1420,7 @@ def scan_resource(
     timeout=DEFAULT_TIMEOUT,
     with_timing=False,
     with_threading=True,
+    use_cached_results=False,
 ):
     """
     Given a ``location_path`` tuple pf (location, path), return a tuple of:
@@ -1443,16 +1458,17 @@ def scan_resource(
     # and start returning values. The kill timeout is otherwise there
     # as a gatekeeper for runaway processes.
 
-    # compute resource_cache_index
-    resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path)
+    results_cache_index = ''
+    if use_cached_results:
+        # compute results_cache_index
+        results_cache_index = results_cache.compute_results_cache_index(location=location, path=path)
 
-    if USE_CACHED_RESULTS:
         # update `results` with cached data or add scanner to scanners_to_run if no
         # cache data is available
         for scanner in scanners:
             # get resource_cache_data
-            resource_cache_data = resource_cache.get_resource_cache_data(
-                resource_cache_index=resource_cache_index,
+            resource_cache_data = results_cache.get_results_cache_data(
+                results_cache_index=results_cache_index,
                 plugin_name=scanner.name
             )
             if resource_cache_data:
@@ -1480,9 +1496,9 @@ def scan_resource(
             # the return value of a scanner fun MUST be a mapping
             if values_mapping:
                 results.update(values_mapping)
-                if USE_CACHED_RESULTS:
-                    resource_cache.update_resource_cache_data(
-                        resource_cache_index=resource_cache_index,
+                if use_cached_results:
+                    results_cache.update_results_cache_data(
+                        results_cache_index=results_cache_index,
                         plugin_name=scanner.name,
                         results=values_mapping,
                     )

src/scancode/resource_cache.py

@@ -0,0 +1,90 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import hashlib
+import json
+import os
+
+from commoncode.fileutils import create_dir
+from commoncode.hash import binary_chunks
+from scancode_config import results_cache_dir
+
+
+def hasher_from_chunks(chunks):
+    """
+    Return a sha256 hasher loaded with `chunks`.
+    """
+    hasher = hashlib.sha256()
+    for chunk in chunks:
+        hasher.update(chunk)
+    return hasher
+
+
+def compute_results_cache_index(location, path):
+    """
+    Compute results_cache_index value for a Resource at `location`.
+    """
+    chunks = binary_chunks(location=location)
+    sha256_hasher = hasher_from_chunks(chunks=chunks)
+    # TODO: consider using filename instead of path
+    sha256_hasher.update(path.encode('utf-8', 'surrogateescape'))
+    return sha256_hasher.hexdigest()
+
+
+def get_results_cache_directory_location(results_cache_index):
+    """
+    Return the location of the directory containing the cache files for a given
+    `results_cache_index` hexstring.
+    """
+    # Split the hash into two subdirectories using the first two prefix pairs
+    prefix1 = results_cache_index[:2]
+    prefix2 = results_cache_index[2:4]
+    directory_name = results_cache_index[4:]
+    return os.path.join(results_cache_dir, prefix1, prefix2, directory_name)
+
+
+def get_results_cache_file_location(results_cache_index, plugin_name):
+    """
+    Return the location of the file containing the cached results of the scanner
+    `plugin_name` for a resource keyed by `results_cache_index` hexstring.
+    """
+    results_cache_directory_location = get_results_cache_directory_location(results_cache_index=results_cache_index)
+    return os.path.join(results_cache_directory_location, plugin_name)
+
+
+def get_results_cache_data(results_cache_index, plugin_name):
+    """
+    Return a mapping containing the results of scan plugin, `plugin_name`, for a
+    resource keyed by `resource_cache_index` hexstring. If the cache file does
+    not exist, an empty mapping is returned.
+    """
+    results_cache_file_location = get_results_cache_file_location(
+        results_cache_index=results_cache_index,
+        plugin_name=plugin_name
+    )
+    if os.path.exists(results_cache_file_location):
+        with open(results_cache_file_location) as f:
+            return json.load(f)
+    else:
+        return {}
+
+
+def update_results_cache_data(results_cache_index, plugin_name, results):
+    """
+    Update the results cache with the `results` of the scanner `plugin_name`
+    for the resource keyed by `results_cache_index`.
+    """
+    results_cache_directory_location = get_results_cache_directory_location(
+        results_cache_index=results_cache_index
+    )
+    if not os.path.exists(results_cache_directory_location):
+        create_dir(results_cache_directory_location)
+    results_cache_file_location = os.path.join(results_cache_directory_location, plugin_name)
+    with open(results_cache_file_location, 'w') as f:
+        json.dump(results, f)

src/scancode/results_cache.py

@@ -190,10 +190,14 @@ def _create_dir(location):
 __env_package_cache_dir = os.getenv('SCANCODE_PACKAGE_INDEX_CACHE')
 packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir)
 
+std_results_cache_dir = join(scancode_cache_dir, 'results')
+__env_results_cache_dir = os.getenv('SCANCODE_RESULTS_CACHE')
+results_cache_dir = (__env_results_cache_dir or std_results_cache_dir)
 
 _create_dir(licensedcode_cache_dir)
 _create_dir(packagedcode_cache_dir)
 _create_dir(scancode_cache_dir)
+_create_dir(results_cache_dir)
 
 # - scancode_temp_dir: for short-lived temporary files which are import- or run-
 # specific that may live for the duration of a function call or for the duration
@@ -221,6 +225,3 @@ def _create_dir(location):
 
 # Used for tests to regenerate fixtures with regen=True
 REGEN_TEST_FIXTURES = SCANCODE_REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False)
-
-# Used to control whether or not we use cached results during scan time
-USE_CACHED_RESULTS = SCANCODE_USE_CACHED_RESULTS = os.getenv('SCANCODE_USE_CACHED_RESULTS', True)