add --replace-originals flag to extractcode

Signed-off-by: Maximilian Huber <maximilian.huber@tngtech.com>

Author: maxhbr

src/extractcode/extract.py

@@ -97,7 +97,7 @@
 ExtractEvent = namedtuple('ExtractEvent', 'source target done warnings errors')
 
 
-def extract(location, kinds=extractcode.default_kinds, recurse=False):
+def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False):
     """
     Walk and extract any archives found at `location` (either a file or
     directory). Extract only archives of a kind listed in the `kinds` kind tuple.
@@ -151,21 +151,21 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False):
             target = join(abspath(top), extractcode.get_extraction_path(loc))
             if TRACE:
                 logger.debug('extract:target: %(target)r' % locals())
-            for xevent in extract_file(loc, target, kinds):
+            for xevent in extract_file(loc, target, kinds, replace_originals=replace_originals):
                 if TRACE:
                     logger.debug('extract:walk:extraction event: %(xevent)r' % locals())
                 yield xevent
 
             if recurse:
                 if TRACE:
                     logger.debug('extract:walk: recursing on target: %(target)r' % locals())
-                for xevent in extract(target, kinds, recurse):
+                for xevent in extract(target, kinds, recurse, replace_originals=replace_originals):
                     if TRACE:
                         logger.debug('extract:walk:recurse:extraction event: %(xevent)r' % locals())
                     yield xevent
 
 
-def extract_file(location, target, kinds=extractcode.default_kinds, verbose=False):
+def extract_file(location, target, kinds=extractcode.default_kinds, verbose=False, replace_originals=False):
     """
     Extract a single archive at `location` in the `target` directory if it is
     of a kind supported in the `kinds` kind tuple.
@@ -178,16 +178,22 @@ def extract_file(location, target, kinds=extractcode.default_kinds, verbose=Fals
                      + getattr(extractor, '__module__', '')
                      + '.' + getattr(extractor, '__name__', ''))
     if extractor:
-        yield ExtractEvent(location, target, done=False, warnings=[], errors=[])
+        yield ExtractEvent(location, target if not replace_originals else location, done=False, warnings=[], errors=[])
         try:
             # extract first to a temp directory: if there is an error,  the
             # extracted files will not be moved to target
             tmp_tgt = fileutils.get_temp_dir(prefix='scancode-extract-')
             abs_location = abspath(expanduser(location))
             warns = extractor(abs_location, tmp_tgt) or []
             warnings.extend(warns)
-            fileutils.copytree(tmp_tgt, target)
-            fileutils.delete(tmp_tgt)
+            if not replace_originals:
+                fileutils.copytree(tmp_tgt, target)
+                fileutils.delete(tmp_tgt)
+            else:
+                if TRACE:
+                    logger.debug('extract_file: replace original with extracted content: %(location)r' % locals())
+                fileutils.delete(location)
+                fileutils.copytree(tmp_tgt, location)
         except Exception as e:
             errors = [str(e).strip(' \'"')]
             if verbose:

src/scancode/api.py

@@ -312,7 +312,7 @@ def get_file_info(location, **kwargs):
     return result
 
 
-def extract_archives(location, recurse=True):
+def extract_archives(location, recurse=True, replace_originals=False):
     """
     Yield ExtractEvent while extracting archive(s) and compressed files at
     `location`. If `recurse` is True, extract nested archives-in-archives
@@ -323,5 +323,5 @@ def extract_archives(location, recurse=True):
     """
     from extractcode.extract import extract
     from extractcode import default_kinds
-    for xevent in extract(location, kinds=default_kinds, recurse=recurse):
+    for xevent in extract(location, kinds=default_kinds, recurse=recurse, replace_originals=replace_originals):
         yield xevent

src/scancode/extract_cli.py

@@ -89,11 +89,12 @@ class ExtractCommand(utils.BaseCommand):
 @click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.')
 @click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.')
 @click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).')
+@click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.')
 
 @click.help_option('-h', '--help')
 @click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ScanCode and licensing and exit.')
 @click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
-def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs):  # NOQA
+def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, *args, **kwargs):  # NOQA
     """extract archives and compressed files found in the <input> file or directory tree.
 
     Use this command before scanning proper as an <input> preparation step.
@@ -157,7 +158,7 @@ def display_extract_summary():
 
     extract_results = []
     has_extract_errors = False
-    extractibles = extract_archives(abs_location, recurse=not shallow)
+    extractibles = extract_archives(abs_location, recurse=not shallow, replace_originals=replace_originals)
 
     if not quiet:
         echo_stderr('Extracting archives...', fg='green')

tests/extractcode/test_extract.py

@@ -238,6 +238,55 @@ def test_extract_tree_recursive(self):
         check_no_error(result)
         check_files(test_dir, expected)
 
+    def test_extract_tree_recursive_remove_originals(self):
+        expected = (
+            'a/a.txt',
+            'a/a.tar.gz/a/b/a.txt',
+            'a/a.tar.gz/a/b/b.txt',
+            'a/a.tar.gz/a/c/c.txt',
+            'b/a.txt',
+            'b/b.tar.gz/b/.svn/all-wcprops',
+            'b/b.tar.gz/b/.svn/entries',
+            'b/b.tar.gz/b/.svn/format',
+            'b/b.tar.gz/b/a/a.txt',
+            'b/b.tar.gz/b/a/.svn/all-wcprops',
+            'b/b.tar.gz/b/a/.svn/entries',
+            'b/b.tar.gz/b/a/.svn/format',
+            'b/b.tar.gz/b/a/.svn/prop-base/a.tar.gz.svn-base',
+            'b/b.tar.gz/b/a/.svn/text-base/a.tar.gz.svn-base',
+            'b/b.tar.gz/b/a/.svn/text-base/a.txt.svn-base',
+            'b/b.tar.gz/b/a/a.tar.gz/a/b/a.txt',
+            'b/b.tar.gz/b/a/a.tar.gz/a/b/b.txt',
+            'b/b.tar.gz/b/a/a.tar.gz/a/c/c.txt',
+            'b/b.tar.gz/b/b/a.txt',
+            'b/b.tar.gz/b/b/.svn/all-wcprops',
+            'b/b.tar.gz/b/b/.svn/entries',
+            'b/b.tar.gz/b/b/.svn/format',
+            'b/b.tar.gz/b/b/.svn/text-base/a.txt.svn-base',
+            'b/b.tar.gz/b/c/a.txt',
+            'b/b.tar.gz/b/c/.svn/all-wcprops',
+            'b/b.tar.gz/b/c/.svn/entries',
+            'b/b.tar.gz/b/c/.svn/format',
+            'b/b.tar.gz/b/c/.svn/prop-base/a.tar.gz.svn-base',
+            'b/b.tar.gz/b/c/.svn/text-base/a.tar.gz.svn-base',
+            'b/b.tar.gz/b/c/.svn/text-base/a.txt.svn-base',
+            'b/b.tar.gz/b/c/a.tar.gz/a/b/a.txt',
+            'b/b.tar.gz/b/c/a.tar.gz/a/b/b.txt',
+            'b/b.tar.gz/b/c/a.tar.gz/a/c/c.txt',
+            'c/a.txt',
+            'c/a.tar.gz/a/b/a.txt',
+            'c/a.tar.gz/a/b/b.txt',
+            'c/a.tar.gz/a/c/c.txt',
+        )
+        test_dir = self.get_test_loc('extract/tree', copy=True)
+        result = list(extract.extract(test_dir, recurse=True, replace_originals=True))
+        check_no_error(result)
+        check_files(test_dir, expected)
+        # again
+        result = list(extract.extract(test_dir, recurse=True))
+        check_no_error(result)
+        check_files(test_dir, expected)
+
     def test_extract_tree_shallow_then_recursive(self):
         shallow = (
             'a/a.tar.gz',