diff --git a/src/extractcode/extract.py b/src/extractcode/extract.py
index 40fefb93d06..7dbc2445d66 100644
--- a/src/extractcode/extract.py
+++ b/src/extractcode/extract.py
@@ -97,7 +97,7 @@
ExtractEvent = namedtuple('ExtractEvent', 'source target done warnings errors')
-def extract(location, kinds=extractcode.default_kinds, recurse=False):
+def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False):
"""
Walk and extract any archives found at `location` (either a file or
directory). Extract only archives of a kind listed in the `kinds` kind tuple.
@@ -116,10 +116,27 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False):
archives If `recurse` is false, then do not extract further an already
extracted archive identified by the corresponding extract suffix location.
+ If `replace_originals` is True, the extracted archives are replaced by the
+ extracted content.
+
Note that while the original file system is walked top-down, breadth-first,
if recurse and a nested archive is found, it is extracted to full depth
first before resuming the file system walk.
"""
+ events = list(extract_files(location, kinds, recurse))
+ if replace_originals:
+ for xevent in reversed(events):
+ if xevent.done:
+ source = xevent.source
+ target = xevent.target
+ if TRACE:
+ logger.debug('extract:replace_originals: replace %(source)r by %(target)r' % locals())
+ fileutils.delete(source)
+ fileutils.copytree(target, source)
+ fileutils.delete(target)
+ return events
+
+def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={})
if TRACE:
logger.debug('extract:start: %(location)r recurse: %(recurse)r\n' % locals())
diff --git a/src/scancode/api.py b/src/scancode/api.py
index 0a3d57deadc..fdc6fb7bd05 100644
--- a/src/scancode/api.py
+++ b/src/scancode/api.py
@@ -312,7 +312,7 @@ def get_file_info(location, **kwargs):
return result
-def extract_archives(location, recurse=True):
+def extract_archives(location, recurse=True, replace_originals=False):
"""
Yield ExtractEvent while extracting archive(s) and compressed files at
`location`. If `recurse` is True, extract nested archives-in-archives
@@ -323,5 +323,5 @@ def extract_archives(location, recurse=True):
"""
from extractcode.extract import extract
from extractcode import default_kinds
- for xevent in extract(location, kinds=default_kinds, recurse=recurse):
+ for xevent in extract(location, kinds=default_kinds, recurse=recurse, replace_originals=replace_originals):
yield xevent
diff --git a/src/scancode/extract_cli.py b/src/scancode/extract_cli.py
index e6230ed7437..e4334371fc9 100644
--- a/src/scancode/extract_cli.py
+++ b/src/scancode/extract_cli.py
@@ -89,11 +89,12 @@ class ExtractCommand(utils.BaseCommand):
@click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.')
@click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.')
@click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).')
+@click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.')
@click.help_option('-h', '--help')
@click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ScanCode and licensing and exit.')
@click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
-def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs): # NOQA
+def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, *args, **kwargs): # NOQA
"""extract archives and compressed files found in the file or directory tree.
Use this command before scanning proper as an preparation step.
@@ -157,7 +158,7 @@ def display_extract_summary():
extract_results = []
has_extract_errors = False
- extractibles = extract_archives(abs_location, recurse=not shallow)
+ extractibles = extract_archives(abs_location, recurse=not shallow, replace_originals=replace_originals)
if not quiet:
echo_stderr('Extracting archives...', fg='green')
diff --git a/tests/extractcode/test_extract.py b/tests/extractcode/test_extract.py
index d20069df959..44bf7451516 100644
--- a/tests/extractcode/test_extract.py
+++ b/tests/extractcode/test_extract.py
@@ -238,6 +238,55 @@ def test_extract_tree_recursive(self):
check_no_error(result)
check_files(test_dir, expected)
+ def test_extract_tree_recursive_replace_originals(self):
+ expected = (
+ 'a/a.txt',
+ 'a/a.tar.gz/a/b/a.txt',
+ 'a/a.tar.gz/a/b/b.txt',
+ 'a/a.tar.gz/a/c/c.txt',
+ 'b/a.txt',
+ 'b/b.tar.gz/b/.svn/all-wcprops',
+ 'b/b.tar.gz/b/.svn/entries',
+ 'b/b.tar.gz/b/.svn/format',
+ 'b/b.tar.gz/b/a/a.txt',
+ 'b/b.tar.gz/b/a/.svn/all-wcprops',
+ 'b/b.tar.gz/b/a/.svn/entries',
+ 'b/b.tar.gz/b/a/.svn/format',
+ 'b/b.tar.gz/b/a/.svn/prop-base/a.tar.gz.svn-base',
+ 'b/b.tar.gz/b/a/.svn/text-base/a.tar.gz.svn-base',
+ 'b/b.tar.gz/b/a/.svn/text-base/a.txt.svn-base',
+ 'b/b.tar.gz/b/a/a.tar.gz/a/b/a.txt',
+ 'b/b.tar.gz/b/a/a.tar.gz/a/b/b.txt',
+ 'b/b.tar.gz/b/a/a.tar.gz/a/c/c.txt',
+ 'b/b.tar.gz/b/b/a.txt',
+ 'b/b.tar.gz/b/b/.svn/all-wcprops',
+ 'b/b.tar.gz/b/b/.svn/entries',
+ 'b/b.tar.gz/b/b/.svn/format',
+ 'b/b.tar.gz/b/b/.svn/text-base/a.txt.svn-base',
+ 'b/b.tar.gz/b/c/a.txt',
+ 'b/b.tar.gz/b/c/.svn/all-wcprops',
+ 'b/b.tar.gz/b/c/.svn/entries',
+ 'b/b.tar.gz/b/c/.svn/format',
+ 'b/b.tar.gz/b/c/.svn/prop-base/a.tar.gz.svn-base',
+ 'b/b.tar.gz/b/c/.svn/text-base/a.tar.gz.svn-base',
+ 'b/b.tar.gz/b/c/.svn/text-base/a.txt.svn-base',
+ 'b/b.tar.gz/b/c/a.tar.gz/a/b/a.txt',
+ 'b/b.tar.gz/b/c/a.tar.gz/a/b/b.txt',
+ 'b/b.tar.gz/b/c/a.tar.gz/a/c/c.txt',
+ 'c/a.txt',
+ 'c/a.tar.gz/a/b/a.txt',
+ 'c/a.tar.gz/a/b/b.txt',
+ 'c/a.tar.gz/a/c/c.txt',
+ )
+ test_dir = self.get_test_loc('extract/tree', copy=True)
+ result = list(extract.extract(test_dir, recurse=True, replace_originals=True))
+ check_no_error(result)
+ check_files(test_dir, expected)
+ # again
+ result = list(extract.extract(test_dir, recurse=True))
+ check_no_error(result)
+ check_files(test_dir, expected)
+
def test_extract_tree_shallow_then_recursive(self):
shallow = (
'a/a.tar.gz',