Merge pull request #371 from nexB/267-cli-options-for-timeout-and-max_memory

#267 Add command options for --timeout and --max-memory

Author: pombredanne

src/scancode/cli.py

@@ -68,8 +68,9 @@ def wrap(self, timeout=None):
 from scancode import __version__ as version
 
 from scancode.interrupt import interruptible
-from scancode.interrupt import compute_memory_quota
-from scancode.interrupt import compute_timeout
+from scancode.interrupt import DEFAULT_TIMEOUT
+from scancode.interrupt import DEFAULT_MAX_MEMORY
+
 
 from scancode import utils
 
@@ -89,10 +90,6 @@ def wrap(self, timeout=None):
 from scancode.api import get_urls
 
 
-# set a value only for testing scan quotas
-TEST_TIMEOUT = 0
-TEST_MAX_MEMORY = 0
-
 
 info_text = '''
 ScanCode scans code and other files for origin and license.
@@ -267,29 +264,32 @@ def validate_formats(ctx, param, value):
 @click.option('-c', '--copyright', is_flag=True, default=False, help='Scan <input> for copyrights. [default]')
 @click.option('-l', '--license', is_flag=True, default=False, help='Scan <input> for licenses. [default]')
 @click.option('-p', '--package', is_flag=True, default=False, help='Scan <input> for packages. [default]')
-@click.option('--email', is_flag=True, default=False, help='Scan <input> for emails.')
-@click.option('--url', is_flag=True, default=False, help='Scan <input> for urls.')
-@click.option('-i', '--info', is_flag=True, default=False, help='Scan <input> for files information.')
+@click.option('-e', '--email', is_flag=True, default=False, help='Scan <input> for emails.')
+@click.option('-u', '--url', is_flag=True, default=False, help='Scan <input> for urls.')
+@click.option('-i', '--info', is_flag=True, default=False, help='Include information such as size, type, etc.')
 @click.option('--license-score', is_flag=False, default=0, type=int, show_default=True,
-              help='Matches with scores lower than this score are not returned. A number between 0 and 100.')
+              help='Do not return license matches with scores lower than this score. A number between 0 and 100.')
 
 @click.option('-f', '--format', is_flag=False, default='json', show_default=True, metavar='<style>',
               help=('Set <output_file> format <style> to one of the standard formats: %s '
                     'or the path to a custom template' % ' or '.join(formats)),
               callback=validate_formats)
 @click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.')
-@click.option('--quiet', is_flag=True, default=False, help='Do not print any progress message.')
-@click.option('-n', '--processes', is_flag=False, default=1, type=int, help='Scan <input> using n parallel processes.')
+@click.option('--quiet', is_flag=True, default=False, help='Do not print progress messages.')
+@click.option('-n', '--processes', is_flag=False, default=1, type=int, show_default=True, help='Scan <input> using n parallel processes.')
 
 @click.help_option('-h', '--help')
 @click.option('--examples', is_flag=True, is_eager=True, callback=print_examples, help=('Show command examples and exit.'))
 @click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ScanCode and licensing and exit.')
 @click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
-@click.option('--diag', is_flag=True, default=False, help='Include detailed diagnnostic messages for scanning errors.')
+@click.option('--diag', is_flag=True, default=False, help='Include detailed diagnostic messages in results if there are scanning errors.')
+@click.option('--timeout', is_flag=False, default=DEFAULT_TIMEOUT, type=int, show_default=True, help='Stop scanning a file if it takes longer than a timeout in seconds.')
+@click.option('--max-memory', is_flag=False, default=DEFAULT_MAX_MEMORY, type=int, show_default=True, help='Stop scanning a file if it its scan requires more than a maximum amount of memory in megabytes.')
 
 def scancode(ctx, input, output_file, copyright, license, package,
              email, url, info, license_score, format,
-             verbose, quiet, processes, diag,
+             verbose, quiet, processes,
+             diag, timeout, max_memory,
              *args, **kwargs):
     """scan the <input> file or directory for origin clues and license and save results to the <output_file>.
 
@@ -305,12 +305,9 @@ def scancode(ctx, input, output_file, copyright, license, package,
     scans_cache_class = get_scans_cache_class()
     try:
         to_stdout = output_file == sys.stdout
-        # for tests only
-        _timeout = float(os.environ.get('SCANCODE_TEST_TIMEOUT', '0'))
-        _max_memory = int(os.environ.get('SCANCODE_TEST_MAX_MEMORY', '0'))
         files_count, results = scan(input, copyright, license, package, email, url, info, license_score,
                                     verbose, quiet, processes, scans_cache_class, to_stdout,
-                                    diag, _timeout, _max_memory)
+                                    diag, timeout, max_memory)
         save_results(files_count, results, format, input, output_file)
     finally:
         # cleanup
@@ -322,7 +319,7 @@ def scan(input_path, copyright=True, license=True, package=True,
          email=False, url=False, info=True, license_score=0,
          verbose=False, quiet=False, processes=1,
          scans_cache_class=None, to_stdout=False,
-         diag=False, _timeout=0, _max_memory=0):
+         diag=False, timeout=DEFAULT_TIMEOUT, max_memory=DEFAULT_MAX_MEMORY):
     """
     Return a tuple of (file_count, indexing_time, scan_results) where
     scan_results is an iterable. Run each requested scan proper: each individual file
@@ -372,7 +369,7 @@ def scan(input_path, copyright=True, license=True, package=True,
     pool = Pool(processes=processes, maxtasksperchild=1000)
     resources = resource_paths(input_path)
     scanit = partial(_scanit, scanners=scanners, scans_cache_class=scans_cache_class,
-                     diag=diag, _timeout=_timeout, _max_memory=_max_memory)
+                     diag=diag, timeout=timeout, max_memory=max_memory)
     # Using chunksize is documented as much more efficient in the Python doc.
     # Yet "1" still provides a better and more progressive feedback.
     # With imap_unordered, results are returned as soon as ready and out of order.
@@ -434,7 +431,7 @@ def scan_event(item):
     return files_count, cached_scan.iterate(with_infos=info)
 
 
-def _scanit(paths, scanners, scans_cache_class, diag, _timeout=0, _max_memory=0):
+def _scanit(paths, scanners, scans_cache_class, diag, timeout=DEFAULT_TIMEOUT, max_memory=DEFAULT_MAX_MEMORY):
     """
     Run scans and cache results on disk. Return a tuple of (success, scanned relative
     path) where sucess is True on success, False on error. Note that this is really
@@ -459,10 +456,7 @@ def _scanit(paths, scanners, scans_cache_class, diag, _timeout=0, _max_memory=0)
             file_size = infos.get('size', 0)
 
             # quota keyword args for interruptible
-            # use _timeout or _max_memory for tests if provided or compute these quotas
-            kwargs = dict()
-            kwargs['timeout'] = _timeout or compute_timeout(file_size)
-            kwargs['max_memory'] = _max_memory or compute_memory_quota(file_size)
+            kwargs = dict(timeout=timeout, max_memory=max_memory)
 
             success, scan_result = interruptible(scans_runner, **kwargs)
 

src/scancode/interrupt.py

@@ -54,13 +54,10 @@ def wrap(self, timeout=None):
 
 import psutil
 
-
-MIN_TIMEOUT = 60  # seconds
-MAX_TIMEOUT = 600  # seconds
+DEFAULT_TIMEOUT = 120  # seconds
 RUNTIME_EXCEEDED = 1
 
-MIN_MEMORY = 2 * 1024 * 1024 * 1024  # 2GB
-MAX_MEMORY = 4 * 1024 * 1024 * 1024  # 4GB
+DEFAULT_MAX_MEMORY = 1000  # megabytes
 MEMORY_EXCEEDED = 2
 
 
@@ -69,12 +66,12 @@ def interruptible(func, *args, **kwargs):
     Call `func` function with `args` arguments and return a tuple of (success, return
     value). `func` is invoked through a wrapper and will be interrupted if it does
     not return within `timeout` seconds of execution or uses more than 'max_memory`
-    bytes of memory. `func` returned results should be pickable.
+    MEGABYTES of memory. `func` returned results should be pickable.
 
     `timeout` in seconds should be provided as a keyword argument.
     MIN_TIMEOUT is always enforced even if no timeout keyword is present.
 
-    `max_memory` in bytes should be provided as a keyword argument.
+    `max_memory` in megabytes should be provided as a keyword argument.
     If not present a memory quota is not enforced.
 
     Only `args` are passed to `func`, not any `kwargs`.
@@ -87,44 +84,24 @@ def interruptible(func, *args, **kwargs):
     item in the tuple is an error message string.
     """
 
+    timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT)
+    max_memory = kwargs.pop('max_memory', DEFAULT_MAX_MEMORY) * 1024 * 1024
+
     # We use a pool of two threads that race to finish against each other:
     # - one runs the func proper
     # - one runs a loop until a timeout to check memory usage and return when it
     #   exceeds max_memory or the timeout
     # The first thread to complete return its result. The other thread is terminated.
 
     pool = ThreadPool(2)
+    execution_units = [(func, args,), (time_and_memory_guard, [max_memory, timeout],)]
 
-    # our execution units contain at least the the function to run proper
-    execution_units = [(func, args,)]
-
-    only_monitor_timeout = True
-    timeout = MIN_TIMEOUT
-
-    # add timeout only if present
-    if 'timeout' in kwargs:
-        timeout = kwargs.pop('timeout', MIN_TIMEOUT)
-
-    # add memory quota only if present
-    if 'max_memory' in kwargs:
-        max_memory = kwargs.pop('max_memory', MIN_MEMORY)
-        only_monitor_timeout = False
-
-    if only_monitor_timeout:
-        # monitor using a simple time guard
-        execution_units.append((time_guard, [timeout],))
-    else:
-        # monitor using a combined time + memory guard
-        execution_units.append((time_and_memory_guard, [max_memory, timeout],))
-
-    # submit our threads: whichever finishes first thanks to imap_unordered
-    # will be returned by the call to next()
+    # run our threads: whichever finishes first thanks to imap_unordered will be
+    # returned by the call to next()
     threads = pool.imap_unordered(runner, execution_units, chunksize=1)
     pool.close()
-
     try:
-        # always use MAX_TIMEOUT
-        result = threads.next(MAX_TIMEOUT)
+        result = threads.next(timeout)
         if result == MEMORY_EXCEEDED:
             max_mb = megabytes(max_memory)
             return False, 'Processing interrupted: excessive memory usage of more than %(max_mb)s.' % locals()
@@ -139,6 +116,7 @@ def interruptible(func, *args, **kwargs):
 
     except KeyboardInterrupt:
         return False, 'Processing interrupted with Ctrl-C.'
+
     finally:
         # stop processing
         pool.terminate()
@@ -153,20 +131,12 @@ def runner(arg):
     return func(*args)
 
 
-def time_guard(timeout):
-    """
-    Return when a timeout has expired.
+def time_and_memory_guard(max_memory=DEFAULT_MAX_MEMORY, timeout=DEFAULT_TIMEOUT, interval=2):
     """
-    sleep(timeout)
-    return RUNTIME_EXCEEDED
-
-
-def time_and_memory_guard(max_memory, timeout=MAX_TIMEOUT, interval=2):
-    """
-    Return when max_memory hass been used or when a timeout has expired.
+    Return when max_memory bytes has been used or when a timeout has expired.
     Check memory usage every `interval` seconds during up to `timeout` seconds. Run
-    until the memory usage in the current process exceeds `max_memory`. If it does,
-    return `MEMORY_EXCEEDED`. If the memory usage does not go over `max_memory`
+    until the memory usage in the current process exceeds `max_memory` bytes. If it does,
+    return `MEMORY_EXCEEDED`. If the memory usage does not go over `max_memory` bytes
     within `timeout` seconds, return RUNTIME_EXCEEDED.
     """
     process = psutil.Process()
@@ -183,24 +153,6 @@ def time_and_memory_guard(max_memory, timeout=MAX_TIMEOUT, interval=2):
     return RUNTIME_EXCEEDED
 
 
-def compute_timeout(size, extra_sec_per_mb=30):
-    """
-    Return a scan timeout in seconds computed from a file size.
-    """
-    # add extra seconds for each megabyte
-    timeout = MIN_TIMEOUT + ((size // (1024 * 1024)) * extra_sec_per_mb)
-    return min([timeout, MAX_TIMEOUT])
-
-
-def compute_memory_quota(size, extra_ram_multiplier=50):
-    """
-    Return a not-to-exceed maximum memory_quota in bytes computed from a file size.
-    """
-    # add extra quota for each byte of a file bigger than 1MB
-    memory_quota = MIN_MEMORY + (size * extra_ram_multiplier)
-    return min([memory_quota, MAX_MEMORY])
-
-
 def megabytes(n):
     """
     Return a megabytes string representation of an `n` number of bytes.

tests/scancode/test_cli.py

@@ -368,17 +368,10 @@ def test_scan_works_with_multiple_processes_and_timeouts(monkeypatch):
     runner = CliRunner()
     result_file = test_env.get_temp_file('json')
 
-    patched_environ = dict(
-        # set small memory quota for test
-        SCANCODE_TEST_MAX_MEMORY='0',  # use default
-        SCANCODE_TEST_TIMEOUT='1',
-    )
-
     result = runner.invoke(
         cli.scancode,
-        [ '--copyright', '--license', '--processes', '2', '--format', 'json', test_dir, result_file],
-        catch_exceptions=True,
-        env=patched_environ)
+        [ '--copyright', '--license', '--processes', '2', '--timeout', '1', '--format', 'json', test_dir, result_file],
+        catch_exceptions=True)
 
     assert result.exit_code == 0
     assert 'Scanning done' in result.output
@@ -405,17 +398,10 @@ def test_scan_works_with_multiple_processes_and_memory_quota(monkeypatch):
     runner = CliRunner()
     result_file = test_env.get_temp_file('json')
 
-    patched_environ = dict(
-        # set small memory quota for test
-        SCANCODE_TEST_MAX_MEMORY=str(1 * 1024 * 1024),
-        SCANCODE_TEST_TIMEOUT='0',  # use default
-    )
-
     result = runner.invoke(
         cli.scancode,
-        [ '--copyright', '--license', '--processes', '2', '--format', 'json', test_dir, result_file],
+        [ '--copyright', '--license', '--processes', '2', '--max-memory', '1', '--format', 'json', test_dir, result_file],
         catch_exceptions=True,
-        env=patched_environ
     )
 
     assert result.exit_code == 0

tests/scancode/test_interrupt.py

@@ -34,18 +34,6 @@
 class TestInterrupt(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
-    def test_compute_timeout(self):
-        assert interrupt.MIN_TIMEOUT == interrupt.compute_timeout(0)
-        assert interrupt.MIN_TIMEOUT == interrupt.compute_timeout(1000)
-        assert interrupt.MIN_TIMEOUT + 300 == interrupt.compute_timeout(10 * 1024 * 1024)
-        assert interrupt.MAX_TIMEOUT == interrupt.compute_timeout(1000 * 1024 * 1024)
-
-    def test_compute_memory_quota(self):
-        assert interrupt.MIN_MEMORY == interrupt.compute_memory_quota(0)
-        assert interrupt.MIN_MEMORY <= interrupt.compute_memory_quota(1000)
-        assert interrupt.MIN_MEMORY < interrupt.compute_memory_quota(10 * 1024 * 1024)
-        assert interrupt.MIN_MEMORY < interrupt.compute_memory_quota(1000 * 1024 * 1024) <= interrupt.MAX_MEMORY
-
     def test_megabytes(self):
         assert '12MB' == interrupt.megabytes(12 * 1024 * 1024)
         assert '1MB' == interrupt.megabytes(1 * 1024 * 1024)
@@ -57,17 +45,14 @@ def test_memory_guard(self):
         # should fail after 2 seconds
         assert interrupt.RUNTIME_EXCEEDED == interrupt.time_and_memory_guard(max_memory=1024 * 1024 * 1024 * 1024, timeout=2, interval=1)
 
-    def test_time_guard(self):
-        assert interrupt.RUNTIME_EXCEEDED == interrupt.time_guard(0.1)
-
     def test_interruptible_can_run_function(self):
         from time import sleep
 
         def some_long_function(exec_time):
             sleep(exec_time)
             return 'OK'
 
-        result = interrupt.interruptible(some_long_function, 0.01, timeout=10, max_memory=1024 * 1024 * 1024)
+        result = interrupt.interruptible(some_long_function, 0.01, timeout=10, max_memory=1024)
         assert (True, 'OK') == result
 
     def test_interruptible_stops_execution_on_timeout(self):
@@ -89,6 +74,6 @@ def some_hungry_function(exec_time):
             _ram = range(1000000)
             return 'OK'
 
-        success, result = interrupt.interruptible(some_hungry_function, 0.1, timeout=5, max_memory=1000)
+        success, result = interrupt.interruptible(some_hungry_function, 0.1, timeout=5, max_memory=1)
         assert success == False
         assert 'Processing interrupted: excessive memory usage of more than' in result