Create directories before opening or creating cache files

    * Create envvar for controlling whether or not the scan results cache is used

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

pyproject.toml

@@ -276,7 +276,6 @@ packages = "packagedcode.plugin_package:PackageScanner"
 emails = "cluecode.plugin_email:EmailScanner"
 urls = "cluecode.plugin_url:UrlScanner"
 generated = "summarycode.generated:GeneratedCodeDetector"
-resource_cache_index = "scancode.plugin_resource_cache:ResourceCacheIndexScanner"
 
 
 # scancode_post_scan is the entry point for post_scan plugins executed after the

src/scancode/cli.py

@@ -72,6 +72,7 @@ class WindowsError(Exception):
 from scancode.interrupt import fake_interruptible
 from scancode.interrupt import interruptible
 from scancode.pool import ScanCodeTimeoutError
+from scancode_config import USE_CACHED_RESULTS
 
 # Tracing flags
 TRACE = False
@@ -1445,21 +1446,22 @@ def scan_resource(
     # compute resource_cache_index
     resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path)
 
-    # update `results` with cached data or add scanner to scanners_to_run if no
-    # cache data is available
-    for scanner in scanners:
-        # get resource_cache_data
-        resource_cache_data = resource_cache.get_resource_cache_data(
-            resource_cache_index=resource_cache_index,
-            plugin_name=scanner.name
-        )
-        if resource_cache_data:
-            results.update(resource_cache_data)
-        else:
-            scanners_to_run.append(scanner)
+    if USE_CACHED_RESULTS:
+        # update `results` with cached data or add scanner to scanners_to_run if no
+        # cache data is available
+        for scanner in scanners:
+            # get resource_cache_data
+            resource_cache_data = resource_cache.get_resource_cache_data(
+                resource_cache_index=resource_cache_index,
+                plugin_name=scanner.name
+            )
+            if resource_cache_data:
+                results.update(resource_cache_data)
+            else:
+                scanners_to_run.append(scanner)
 
     # run each scanner in sequence in its own interruptible
-    for scanner in scanners_to_run:
+    for scanner in scanners_to_run or scanners:
         if with_timing:
             start = time()
 
@@ -1478,10 +1480,12 @@ def scan_resource(
             # the return value of a scanner fun MUST be a mapping
             if values_mapping:
                 results.update(values_mapping)
-                resource_cache.update_resource_cache_data(
-                    resource_cache_index=resource_cache_index,
-                    plugin_name=scanner.name
-                )
+                if USE_CACHED_RESULTS:
+                    resource_cache.update_resource_cache_data(
+                        resource_cache_index=resource_cache_index,
+                        plugin_name=scanner.name,
+                        results=values_mapping,
+                    )
 
         except Exception:
             msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc()

src/scancode/resource_cache.py

@@ -11,6 +11,7 @@
 import json
 import os
 
+from commoncode.fileutils import create_dir
 from commoncode.hash import binary_chunks
 from scancode_config import scancode_cache_dir
 
@@ -82,9 +83,11 @@ def update_resource_cache_data(resource_cache_index, plugin_name, results):
     Update the resource cache with the `results` of the scanner `plugin_name`
     for the resource keyed by `resource_cache_index`.
     """
-    resource_cache_file_location = get_resource_cache_file_location(
-        resource_cache_index=resource_cache_index,
-        plugin_name=plugin_name
+    resource_cache_directory_location = get_resource_cache_directory_location(
+        resource_cache_index=resource_cache_index
     )
+    if not os.path.exists(resource_cache_directory_location):
+        create_dir(resource_cache_directory_location)
+    resource_cache_file_location = os.path.join(resource_cache_directory_location, plugin_name)
     with open(resource_cache_file_location, 'w') as f:
         json.dump(results, f)

src/scancode_config.py

@@ -221,3 +221,6 @@ def _create_dir(location):
 
 # Used for tests to regenerate fixtures with regen=True
 REGEN_TEST_FIXTURES = SCANCODE_REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False)
+
+# Used to control whether or not we use cached results during scan time
+USE_CACHED_RESULTS = SCANCODE_USE_CACHED_RESULTS = os.getenv('SCANCODE_USE_CACHED_RESULTS', True)