Update scan_resource to get and save data to cache

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/scancode/cli.py

@@ -65,6 +65,7 @@ class WindowsError(Exception):
 from scancode import notice
 from scancode import print_about
 from scancode import Scanner
+from scancode import resource_cache
 from scancode.help import epilog_text
 from scancode.help import examples_text
 from scancode.interrupt import DEFAULT_TIMEOUT
@@ -1430,6 +1431,7 @@ def scan_resource(
     results = {}
     scan_errors = []
     timings = {} if with_timing else None
+    scanners_to_run = []
 
     if not with_threading:
         interruptor = fake_interruptible
@@ -1440,8 +1442,24 @@ def scan_resource(
     # and start returning values. The kill timeout is otherwise there
     # as a gatekeeper for runaway processes.
 
-    # run each scanner in sequence in its own interruptible
+    # compute resource_cache_index
+    resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path)
+
+    # update `results` with cached data or add scanner to scanners_to_run if no
+    # cache data is available
     for scanner in scanners:
+        # get resource_cache_data
+        resource_cache_data = resource_cache.get_resource_cache_data(
+            resource_cache_index=resource_cache_index,
+            plugin_name=scanner.name
+        )
+        if resource_cache_data:
+            results.update(resource_cache_data)
+        else:
+            scanners_to_run.append(scanner)
+
+    # run each scanner in sequence in its own interruptible
+    for scanner in scanners_to_run:
         if with_timing:
             start = time()
 
@@ -1460,6 +1478,10 @@ def scan_resource(
             # the return value of a scanner fun MUST be a mapping
             if values_mapping:
                 results.update(values_mapping)
+                resource_cache.update_resource_cache_data(
+                    resource_cache_index=resource_cache_index,
+                    plugin_name=scanner.name
+                )
 
         except Exception:
             msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc()

src/scancode/plugin_resource_cache.py

@@ -0,0 +1,90 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import hashlib
+import json
+import os
+
+from commoncode.hash import binary_chunks
+from scancode_config import scancode_cache_dir
+
+
+RESOURCE_CACHE_DIR = os.path.join(scancode_cache_dir, "resource_cache_index")
+
+
+def hasher_from_chunks(chunks):
+    """
+    Return a sha256 hasher loaded with `chunks`.
+    """
+    hasher = hashlib.sha256()
+    for chunk in chunks:
+        hasher.update(chunk)
+    return hasher
+
+
+def compute_resource_cache_index(location, path):
+    """
+    Compute resource_cache_index value for Resource at `location`.
+    """
+    chunks = binary_chunks(location=location)
+    sha256_hasher = hasher_from_chunks(chunks=chunks)
+    # TODO: consider using filename instead of path
+    sha256_hasher.update(path.encode('utf-8', 'surrogateescape'))
+    return sha256_hasher.hexdigest()
+
+
+def get_resource_cache_directory_location(resource_cache_index):
+    """
+    Return the location of the directory containing the cache files for a given
+    `resource_cache_index` hexstring.
+    """
+    # Split the hash into two subdirectories using the first two prefix pairs
+    prefix1 = resource_cache_index[:2]
+    prefix2 = resource_cache_index[2:4]
+    directory_name = resource_cache_index[4:]
+    return os.path.join(RESOURCE_CACHE_DIR, prefix1, prefix2, directory_name)
+
+
+def get_resource_cache_file_location(resource_cache_index, plugin_name):
+    """
+    Return the location of the file containing the cached results of the scanner
+    `plugin_name` for a resource keyed by `resource_cache_index` hexstring.
+    """
+    resource_cache_directory_location = get_resource_cache_directory_location(resource_cache_index=resource_cache_index)
+    return os.path.join(resource_cache_directory_location, plugin_name)
+
+
+def get_resource_cache_data(resource_cache_index, plugin_name):
+    """
+    Return a mapping containing the results of scan plugin, `plugin_name`, for a
+    resource keyed by `resource_cache_index` hexstring. If the cache file does
+    not exist, an empty mapping is returned.
+    """
+    resource_cache_file_location = get_resource_cache_file_location(
+        resource_cache_index=resource_cache_index,
+        plugin_name=plugin_name
+    )
+    if os.path.exists(resource_cache_file_location):
+        with open(resource_cache_file_location) as f:
+            return json.load(f)
+    else:
+        return {}
+
+
+def update_resource_cache_data(resource_cache_index, plugin_name, results):
+    """
+    Update the resource cache with the `results` of the scanner `plugin_name`
+    for the resource keyed by `resource_cache_index`.
+    """
+    resource_cache_file_location = get_resource_cache_file_location(
+        resource_cache_index=resource_cache_index,
+        plugin_name=plugin_name
+    )
+    with open(resource_cache_file_location, 'w') as f:
+        json.dump(results, f)