diff --git a/src/extractcode/extract.py b/src/extractcode/extract.py index 40fefb93d06..7dbc2445d66 100644 --- a/src/extractcode/extract.py +++ b/src/extractcode/extract.py @@ -97,7 +97,7 @@ ExtractEvent = namedtuple('ExtractEvent', 'source target done warnings errors') -def extract(location, kinds=extractcode.default_kinds, recurse=False): +def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False): """ Walk and extract any archives found at `location` (either a file or directory). Extract only archives of a kind listed in the `kinds` kind tuple. @@ -116,10 +116,27 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False): archives If `recurse` is false, then do not extract further an already extracted archive identified by the corresponding extract suffix location. + If `replace_originals` is True, the extracted archives are replaced by the + extracted content. + Note that while the original file system is walked top-down, breadth-first, if recurse and a nested archive is found, it is extracted to full depth first before resuming the file system walk. """ + events = list(extract_files(location, kinds, recurse)) + if replace_originals: + for xevent in reversed(events): + if xevent.done: + source = xevent.source + target = xevent.target + if TRACE: + logger.debug('extract:replace_originals: replace %(source)r by %(target)r' % locals()) + fileutils.delete(source) + fileutils.copytree(target, source) + fileutils.delete(target) + return events + +def extract_files(location, kinds=extractcode.default_kinds, recurse=False): ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={}) if TRACE: logger.debug('extract:start: %(location)r recurse: %(recurse)r\n' % locals()) diff --git a/src/scancode/api.py b/src/scancode/api.py index 0a3d57deadc..fdc6fb7bd05 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -312,7 +312,7 @@ def get_file_info(location, **kwargs): return result -def extract_archives(location, recurse=True): +def extract_archives(location, recurse=True, replace_originals=False): """ Yield ExtractEvent while extracting archive(s) and compressed files at `location`. If `recurse` is True, extract nested archives-in-archives @@ -323,5 +323,5 @@ def extract_archives(location, recurse=True): """ from extractcode.extract import extract from extractcode import default_kinds - for xevent in extract(location, kinds=default_kinds, recurse=recurse): + for xevent in extract(location, kinds=default_kinds, recurse=recurse, replace_originals=replace_originals): yield xevent diff --git a/src/scancode/extract_cli.py b/src/scancode/extract_cli.py index e6230ed7437..e4334371fc9 100644 --- a/src/scancode/extract_cli.py +++ b/src/scancode/extract_cli.py @@ -89,11 +89,12 @@ class ExtractCommand(utils.BaseCommand): @click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.') @click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.') @click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).') +@click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.') @click.help_option('-h', '--help') @click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ScanCode and licensing and exit.') @click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.') -def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs): # NOQA +def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, *args, **kwargs): # NOQA """extract archives and compressed files found in the file or directory tree. Use this command before scanning proper as an preparation step. @@ -157,7 +158,7 @@ def display_extract_summary(): extract_results = [] has_extract_errors = False - extractibles = extract_archives(abs_location, recurse=not shallow) + extractibles = extract_archives(abs_location, recurse=not shallow, replace_originals=replace_originals) if not quiet: echo_stderr('Extracting archives...', fg='green') diff --git a/tests/extractcode/test_extract.py b/tests/extractcode/test_extract.py index d20069df959..44bf7451516 100644 --- a/tests/extractcode/test_extract.py +++ b/tests/extractcode/test_extract.py @@ -238,6 +238,55 @@ def test_extract_tree_recursive(self): check_no_error(result) check_files(test_dir, expected) + def test_extract_tree_recursive_replace_originals(self): + expected = ( + 'a/a.txt', + 'a/a.tar.gz/a/b/a.txt', + 'a/a.tar.gz/a/b/b.txt', + 'a/a.tar.gz/a/c/c.txt', + 'b/a.txt', + 'b/b.tar.gz/b/.svn/all-wcprops', + 'b/b.tar.gz/b/.svn/entries', + 'b/b.tar.gz/b/.svn/format', + 'b/b.tar.gz/b/a/a.txt', + 'b/b.tar.gz/b/a/.svn/all-wcprops', + 'b/b.tar.gz/b/a/.svn/entries', + 'b/b.tar.gz/b/a/.svn/format', + 'b/b.tar.gz/b/a/.svn/prop-base/a.tar.gz.svn-base', + 'b/b.tar.gz/b/a/.svn/text-base/a.tar.gz.svn-base', + 'b/b.tar.gz/b/a/.svn/text-base/a.txt.svn-base', + 'b/b.tar.gz/b/a/a.tar.gz/a/b/a.txt', + 'b/b.tar.gz/b/a/a.tar.gz/a/b/b.txt', + 'b/b.tar.gz/b/a/a.tar.gz/a/c/c.txt', + 'b/b.tar.gz/b/b/a.txt', + 'b/b.tar.gz/b/b/.svn/all-wcprops', + 'b/b.tar.gz/b/b/.svn/entries', + 'b/b.tar.gz/b/b/.svn/format', + 'b/b.tar.gz/b/b/.svn/text-base/a.txt.svn-base', + 'b/b.tar.gz/b/c/a.txt', + 'b/b.tar.gz/b/c/.svn/all-wcprops', + 'b/b.tar.gz/b/c/.svn/entries', + 'b/b.tar.gz/b/c/.svn/format', + 'b/b.tar.gz/b/c/.svn/prop-base/a.tar.gz.svn-base', + 'b/b.tar.gz/b/c/.svn/text-base/a.tar.gz.svn-base', + 'b/b.tar.gz/b/c/.svn/text-base/a.txt.svn-base', + 'b/b.tar.gz/b/c/a.tar.gz/a/b/a.txt', + 'b/b.tar.gz/b/c/a.tar.gz/a/b/b.txt', + 'b/b.tar.gz/b/c/a.tar.gz/a/c/c.txt', + 'c/a.txt', + 'c/a.tar.gz/a/b/a.txt', + 'c/a.tar.gz/a/b/b.txt', + 'c/a.tar.gz/a/c/c.txt', + ) + test_dir = self.get_test_loc('extract/tree', copy=True) + result = list(extract.extract(test_dir, recurse=True, replace_originals=True)) + check_no_error(result) + check_files(test_dir, expected) + # again + result = list(extract.extract(test_dir, recurse=True)) + check_no_error(result) + check_files(test_dir, expected) + def test_extract_tree_shallow_then_recursive(self): shallow = ( 'a/a.tar.gz',