Skip to content

add --replace-originals flag to extractcode #1893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pombredanne merged 2 commits into from
Feb 17, 2020
Merged

add --replace-originals flag to extractcode #1893

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9fa41f6
add --replace-originals flag to extractcode
maxhbr Feb 10, 2020
7af031e
Use the same doc for replace_originals as in CLI
pombredanne Feb 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/extractcode/extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
ExtractEvent = namedtuple('ExtractEvent', 'source target done warnings errors')


def extract(location, kinds=extractcode.default_kinds, recurse=False):
def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False):
Comment thread
maxhbr marked this conversation as resolved.
"""
Walk and extract any archives found at `location` (either a file or
directory). Extract only archives of a kind listed in the `kinds` kind tuple.
Expand All @@ -116,10 +116,27 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False):
archives If `recurse` is false, then do not extract further an already
extracted archive identified by the corresponding extract suffix location.

If `replace_originals` is True, the extracted archives are replaced by the
extracted content.

Note that while the original file system is walked top-down, breadth-first,
if recurse and a nested archive is found, it is extracted to full depth
first before resuming the file system walk.
"""
events = list(extract_files(location, kinds, recurse))
if replace_originals:
for xevent in reversed(events):
if xevent.done:
source = xevent.source
target = xevent.target
if TRACE:
logger.debug('extract:replace_originals: replace %(source)r by %(target)r' % locals())
fileutils.delete(source)
fileutils.copytree(target, source)
fileutils.delete(target)
return events

def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={})
if TRACE:
logger.debug('extract:start: %(location)r recurse: %(recurse)r\n' % locals())
Expand Down
4 changes: 2 additions & 2 deletions src/scancode/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ def get_file_info(location, **kwargs):
return result


def extract_archives(location, recurse=True):
def extract_archives(location, recurse=True, replace_originals=False):
"""
Yield ExtractEvent while extracting archive(s) and compressed files at
`location`. If `recurse` is True, extract nested archives-in-archives
Expand All @@ -323,5 +323,5 @@ def extract_archives(location, recurse=True):
"""
from extractcode.extract import extract
from extractcode import default_kinds
for xevent in extract(location, kinds=default_kinds, recurse=recurse):
for xevent in extract(location, kinds=default_kinds, recurse=recurse, replace_originals=replace_originals):
yield xevent
Comment thread
pombredanne marked this conversation as resolved.
5 changes: 3 additions & 2 deletions src/scancode/extract_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,12 @@ class ExtractCommand(utils.BaseCommand):
@click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.')
@click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.')
@click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).')
@click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.')

@click.help_option('-h', '--help')
@click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ScanCode and licensing and exit.')
@click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs): # NOQA
def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, *args, **kwargs): # NOQA
"""extract archives and compressed files found in the <input> file or directory tree.

Use this command before scanning proper as an <input> preparation step.
Expand Down Expand Up @@ -157,7 +158,7 @@ def display_extract_summary():

extract_results = []
has_extract_errors = False
extractibles = extract_archives(abs_location, recurse=not shallow)
extractibles = extract_archives(abs_location, recurse=not shallow, replace_originals=replace_originals)

if not quiet:
echo_stderr('Extracting archives...', fg='green')
Expand Down
49 changes: 49 additions & 0 deletions tests/extractcode/test_extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,55 @@ def test_extract_tree_recursive(self):
check_no_error(result)
check_files(test_dir, expected)

def test_extract_tree_recursive_replace_originals(self):
expected = (
'a/a.txt',
'a/a.tar.gz/a/b/a.txt',
'a/a.tar.gz/a/b/b.txt',
'a/a.tar.gz/a/c/c.txt',
'b/a.txt',
'b/b.tar.gz/b/.svn/all-wcprops',
'b/b.tar.gz/b/.svn/entries',
'b/b.tar.gz/b/.svn/format',
'b/b.tar.gz/b/a/a.txt',
'b/b.tar.gz/b/a/.svn/all-wcprops',
'b/b.tar.gz/b/a/.svn/entries',
'b/b.tar.gz/b/a/.svn/format',
'b/b.tar.gz/b/a/.svn/prop-base/a.tar.gz.svn-base',
'b/b.tar.gz/b/a/.svn/text-base/a.tar.gz.svn-base',
'b/b.tar.gz/b/a/.svn/text-base/a.txt.svn-base',
'b/b.tar.gz/b/a/a.tar.gz/a/b/a.txt',
'b/b.tar.gz/b/a/a.tar.gz/a/b/b.txt',
'b/b.tar.gz/b/a/a.tar.gz/a/c/c.txt',
'b/b.tar.gz/b/b/a.txt',
'b/b.tar.gz/b/b/.svn/all-wcprops',
'b/b.tar.gz/b/b/.svn/entries',
'b/b.tar.gz/b/b/.svn/format',
'b/b.tar.gz/b/b/.svn/text-base/a.txt.svn-base',
'b/b.tar.gz/b/c/a.txt',
'b/b.tar.gz/b/c/.svn/all-wcprops',
'b/b.tar.gz/b/c/.svn/entries',
'b/b.tar.gz/b/c/.svn/format',
'b/b.tar.gz/b/c/.svn/prop-base/a.tar.gz.svn-base',
'b/b.tar.gz/b/c/.svn/text-base/a.tar.gz.svn-base',
'b/b.tar.gz/b/c/.svn/text-base/a.txt.svn-base',
'b/b.tar.gz/b/c/a.tar.gz/a/b/a.txt',
'b/b.tar.gz/b/c/a.tar.gz/a/b/b.txt',
'b/b.tar.gz/b/c/a.tar.gz/a/c/c.txt',
'c/a.txt',
'c/a.tar.gz/a/b/a.txt',
'c/a.tar.gz/a/b/b.txt',
'c/a.tar.gz/a/c/c.txt',
)
test_dir = self.get_test_loc('extract/tree', copy=True)
result = list(extract.extract(test_dir, recurse=True, replace_originals=True))
check_no_error(result)
check_files(test_dir, expected)
# again
result = list(extract.extract(test_dir, recurse=True))
check_no_error(result)
check_files(test_dir, expected)

def test_extract_tree_shallow_then_recursive(self):
shallow = (
'a/a.tar.gz',
Expand Down