#267 Add SCANCODE_EXPERIMENTAL_MAX_MEMORY feature switch

 * if the SCANCODE_EXPERIMENTAL_MAX_MEMORY env var exists
   then the memory limit will be enforced. Otherwise only
   timeouts are enforced

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/scancode/cli.py

@@ -31,11 +31,12 @@
 # FIXME: unknown license
 ###########################################################################
 from multiprocessing.pool import IMapIterator, IMapUnorderedIterator
-from commoncode import filetype
+
 def wrapped(func):
     def wrap(self, timeout=None):
         return func(self, timeout=timeout or 1e10)
     return wrap
+
 IMapIterator.next = wrapped(IMapIterator.next)
 IMapIterator.__next__ = IMapIterator.next
 IMapUnorderedIterator.next = wrapped(IMapUnorderedIterator.next)
@@ -61,10 +62,12 @@ def wrap(self, timeout=None):
 
 from commoncode import ignore
 from commoncode import fileutils
+from commoncode import filetype
 
 from scancode import __version__ as version
 
 from scancode.interrupt import interruptible
+from scancode.interrupt import time_and_ram_interruptible
 from scancode import utils
 
 from scancode.cache import get_scans_cache_class
@@ -403,7 +406,7 @@ def scan_event(item):
             click.secho(' ' + errored_path, fg='red', err=to_stdout)
 
     click.secho('Scan statistics: %(files_count)d files scanned in %(total_time)ds.' % locals(), err=to_stdout)
-    click.secho('Scan options:    %(_scans)s with %(processes)d processes.' % locals(), err=to_stdout)
+    click.secho('Scan options:    %(_scans)s with %(processes)d processe(s).' % locals(), err=to_stdout)
     click.secho('Scanning speed:  {:.2} files per sec.'.format(files_scanned_per_second), err=to_stdout)
     click.secho('Scanning time:   %(scanning_time)ds.' % locals(), err=to_stdout, reset=True,)
     click.secho('Indexing time:   %(indexing_time)ds.' % locals(), err=to_stdout)
@@ -416,11 +419,17 @@ def scan_event(item):
 TEST_TIMEOUT = 0
 MAX_SCAN_TIMEOUT = 600
 
+TEST_MAX_MEMORY = 0 #1024 * 1024 * 1024 * 80  # 100MB
+MAX_SCAN_MEMORY = 1024 * 1024 * 1024 * 1024  # 1GB
+
+SCANCODE_EXPERIMENTAL_MAX_MEMORY = os.environ.get('SCANCODE_EXPERIMENTAL_MAX_MEMORY', False)
+
 
 def scan_timeout(size):
     """
-    Return a timeout in seconds computed based on a file size.
+    Return a scan timeout in seconds computed based on a file size.
     """
+    # at least 60 seconds
     timeout = 60
     if size > 1024 * 1024 * 1024:
         # add extra seconds for each megabyte
@@ -430,6 +439,20 @@ def scan_timeout(size):
     return timeout
 
 
+def scan_max_memory(size):
+    """
+    Return a scan not-to-exceed maximum memory in bytes computed based on a file size.
+    """
+
+    max_memory = 1024 * 1024 * 1024 * 700  # 700MB
+    if size > 1024 * 1024 * 1024:
+        # add extra quota for each byte: 5x
+        max_memory += size * 5
+
+    max_memory = min((max_memory, MAX_SCAN_MEMORY))
+    return max_memory
+
+
 def _scanit(paths, scanners, scans_cache_class):
     """
     Run scans and cache results. Used as an execution unit for parallel processing.
@@ -449,11 +472,18 @@ def _scanit(paths, scanners, scans_cache_class):
         # ENSURE we only do tghis for files not directories
         if not is_cached:
             # run the scan as an interruptiple task
-            # use TEST_TIMEOUT for tests if provided
-            timeout = TEST_TIMEOUT or scan_timeout(infos.get('size', 0))
             scans_runner = partial(scan_one, abs_path, scanners)
-            success, scan_result = interruptible(scans_runner, timeout=timeout)
 
+            file_size = infos.get('size', 0)
+            # use TEST_TIMEOUT for tests if provided
+            timeout = TEST_TIMEOUT or scan_timeout(file_size)
+            # feature switch
+            if SCANCODE_EXPERIMENTAL_MAX_MEMORY:
+                # use TEST_MAX_MEMORY for tests if provided
+                max_memory = TEST_MAX_MEMORY or scan_max_memory(file_size)
+                success, scan_result = time_and_ram_interruptible(scans_runner, timeout=timeout, max_memory=max_memory)
+            else: 
+                success, scan_result = interruptible(scans_runner, timeout=timeout)                
             if not success:
                 # Use scan errors as the scan result for that file on failure
                 scan_result = dict(scan_errors=[scan_result, ''])