diff --git a/.gitattributes b/.gitattributes index c446d38..96c89ce 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Ignore all Git auto CR/LF line endings conversions -* binary +* -text +pyproject.toml export-subst diff --git a/.travis.yml b/.travis.yml index 7a342df..8f9412e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,10 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9" # Scripts to run at install stage -install: ./configure +install: ./configure --dev # Scripts to run at script stage -script: tmp/bin/pytest +script: tmp/bin/pytest --ignore=tests/test_vmimage.py diff --git a/AUTHORS.rst b/AUTHORS.rst index 00bd7dc..eeaf661 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,10 +2,12 @@ The following organizations or individuals have contributed to this repo: - Abhishek Kumar @Abhishek-Dev09 - AlexB @a-tinsmith +- Konrad Weihmann @priv-kweihmann - Maximilian Huber @maxhbr - Michael Rupprecht @michaelrup - Philippe Ombredanne @pombredanne +- Pierre Tardy @tardyp - Qingmin Duanmu @qduanmu - Rakesh Balusa @balusarakesh - Ravi Jain @JRavi2 -- Steven Esser @majurg +- Steven Esser @majurg \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 39a0096..f666fe2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,36 +1,50 @@ -Release notes -============= +Changelog +========= -vNext ------ +v (next) +-------- -Version 21.1.21 ---------------- +v21.6.1 +-------- -- Bump dependencies and use latest typecode and binaries. This is to fix - installation problems on multiple OSes. +- Add support for VMDK, QCOW and VDI VM image filesystems extraction +- Add new configuration mechanism to get third-party binary paths: + - Use an environment variable + - Or use a plugin-provided path + - Or use well-known system installation locations + - Or use the system PATH + - Or fail with an informative error message -Version 21.1.21 ---------------- +- Update to use latest skeleton -- Add new [full] extra requires that install all the dependencies -- Fix bug related to commoncode libraries loading + +v2021-2-24 +---------- + +- Fix incorrect documentation link + + +v2021-1-21 +---------- + +- Fix bug related to CommonCode libraries loading - Improve the extra requirements - Set minimum version for dependencies - Improve documentation +- Reorganize tests files -Version 21.1.15 ---------------- +v2021-1-15 +---------- - Drop support for Python 2 - Use the latest CommonCode and TypeCode libraries - Add azure-pipelines CI support -Version 20.10 -------------- +v20.10 +------ -- Initial release. +- Initial release as a split from ScanCode toolkit diff --git a/NOTICE b/NOTICE index 65936b2..ad3576b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,19 +1,8 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/README.rst b/README.rst index 4c15702..23ae9cf 100644 --- a/README.rst +++ b/README.rst @@ -4,35 +4,253 @@ ExtractCode - license: Apache-2.0 - copyright: copyright (c) nexB. Inc. and others - homepage_url: https://github.com/nexB/extractcode -- keywords: archive, extraction, libarchive, 7zip, scancode-toolkit +- keywords: archive, extraction, libarchive, 7zip, scancode-toolkit, extractcode +Supports Windows, Linux and macOS on 64 bits processors and Python 3.6 to 3.9. -ExtractCode is a universal archive extractor. It uses behind the scenes -the Python standard library, a custom ctypes binding to libarchive and -the 7zip command line to extract a large number of common and -less common archives and compressed files. It tries to extract things -in the same way on all OSes, including auto-renaming files that would -not have valid names on certain filesystems or when there are multiple -copies of the same path in a given archive. -The extraction is driven from a "voting" system that considers the -file extension(s) and name, the file type and mime type (using a ctypes -binding to libmagic) to select the most appropriate extractor or -uncompressor function. It can handle multi-level archives such as tar.gz. +**ExtractCode is a (mostly) universal archive extractor.** +Install with:: + + pip install extractcode[full] + + +Why another extractor? +---------------------- + +**it will extract!** + +ExtractCode will extract things where other extractors may fail. + +- Say you want to extract the tarball of the Linux kernel source code on Windows. + It contains paths that are the same when ignoring the case and therefore will + not extract OK on Windows: some file may be munged or the extract may file. + +- Or a tarball (on any OS) may contain multiple times the exact same path. In + these cases the paths showing up earlier in the archive may be "hidden" and + overwritten by the same path showing up later in the archive giving the + impression that there is only one file. + +- Or an archive may be damaged a little but most files can still be extracted. + +- Or the extracted files are such permissions that you cannot read them and are + not owned by you. + +- Or the archive may contain weird paths inluding relative paths that may be + problematic to extract. + +- Or the archive may contain special file types (character/device files) that + may be problematic to extract. + +- Or an archive may be a virtual disk or some file system(s) images that would + typically need to be mounted to be accessed, and may require root access + and guesswork to find out which partition and filesystem are at play and + which driver to use. + +In all these cases, ExtractCode will extract and try hard do the right thing to +obtain the actual archived content when other tools may fail. + +It can also extract recursively any type of (nested) archives-in-archives + +As a downside, the extracted content may not be exactly what would be expected +to use the contained files: for instance ... but this it is perfectly OK for +file content analysis for software composition or forensic analysis. + +Behind the scene, ExtractCode uses multiple tools such as: + +- the Python standard library, +- a custom ctypes binding to libarchive, +- the 7zip command line tool, and +- optionally libguestfs on Linux. + +With these, it is possible to extract a large number of common and less common +archives and compressed file types. ExtractCode tries to extract things in the +same way on all supported OSes, including auto-renaming files that would have +invalid, non-extractible names on certain filesystems or when there are multiple +copies of the same path in a given archive (which is possible in a tar). + +The extraction is driven from a "voting" system that considers the file +extension(s) and name, the filetype and mimetype (using a ctypes binding to +libmagic) to select the most appropriate extractor or decompressor function. +It can handle multi-level archives such as tar.gz and can extract recursively +any nested archives. Visit https://aboutcode.org and https://github.com/nexB/ for support and download. + +We run CI tests on: + + - Azure pipelines https://dev.azure.com/nexB/extractcode/_build + + +Installation +------------ + +To install this package with its full capability (where the binaries for +7zip and libarchive are installed), use the `full` extra option:: + + pip install extractcode[full] + +If you want to use the version of binaries (possibly) provided by your operating +system, use the `minimal` option:: + + pip install extractcode + +In this case, you will need to provide a working and compatible libarchive and +7zip installed and configured in one of these ways such that ExtractCode can +find them: + +- **a typecode-libarchive and typecode-7z plugin**: See the standard ones at + https://github.com/nexB/scancode-plugins/tree/main/builtins + These can either bundle a libarchive library, a 7z executable or expose a + system-installed libraries. + It does so by providing plugin entry points as ``scancode_location_provider`` + for ``extractcode_libarchive`` that should point to a ``LocationProviderPlugin`` + subclass with a ``get_locations()`` method that must return a mapping with + this key: + + - 'extractcode.libarchive.dll': the absolute path to a **libarchive** shared object/DLL + + See for example: + + - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/extractcode_libarchive-linux/setup.py#L40 + - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/extractcode_libarchive-linux/src/extractcode_libarchive/__init__.py#L17 + + And in the same way, the ``scancode_location_provider`` for ``extractcode_7zip`` + should point to a ``LocationProviderPlugin`` subclass with a ``get_locations()`` + method that must return a mapping with this key: + + - 'extractcode.sevenzip.exe': the absolute path to a **7zip** executable + + See for example: + + - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/extractcode_7z-linux/setup.py#L40 + - https://github.com/nexB/scancode-plugins/blob/4da5fe8a5ab1c87b9b4af9e54d7ad60e289747f5/builtins/extractcode_7z-linux/src/extractcode_7z/__init__.py#L18 + +- use **environment variables** to point to installed binaries: + + - EXTRACTCODE_LIBARCHIVE_PATH: the absolute path to a libarchive DLL + - EXTRACTCODE_7Z_PATH: the absolute path to a 7zip executable + + +- **a system-installed libarchive and 7zip executable** available in the system **PATH**. + + +The supported binary tools versions are: + +- libarchive 3.5.x +- 7zip 16.5.x + + +Development +----------- + To set up the development environment:: - source configure + source configure --dev + To run unit tests:: pytest -vvs -n 2 + To clean up development environment:: ./configure --clean +To run the command line tool in the activated environment:: + + ./extractcode -h + + +Configuration with environment variables +---------------------------------------- + +ExtractCode will use these environment variables if set: + +- EXTRACTCODE_LIBARCHIVE_PATH : the path to the ``libarchive.so`` libarchive + shared library used to support some of the archive formats. If not provided, + ExtractCode will look for a plugin-provided libarchive library path. See + https://github.com/nexB/scancode-plugins/tree/main/builtins for such plugins. + If no plugin contributes libarchive, then a final attempt is made to look for + it in the PATH using standard DLL loading techniques. + +- EXTRACTCODE_7Z_PATH : the path to the ``7z`` 7zip executable used to support + some of the archive formats. If not provided, ExtractCode will look for a + plugin-provided 7z executable path. See + https://github.com/nexB/scancode-plugins/tree/main/builtins for such plugins. + If no plugin contributes 7z, then a final attempt is made to look for + it in the PATH. + +- EXTRACTCODE_GUESTFISH_PATH : the path to the ``guestfish`` tool from + libguestfs to use to extract VM images. If not provided, ExtractCode will look + in the PATH for an installed ``guestfish`` executable instead. + + + +Adding support for VM images extraction +--------------------------------------- + +Adding support for VM images requires the manual installation of the +libguestfs-tools system package. This is suported only on Linux. +On Debian and Ubuntu you can use this command:: + + sudo apt-get install libguestfs-tools + + +On Ubuntu only, an additional manual step is required as the kernel executable +file cannot be read by users as required by libguestfish. + +Run this command as a temporary and immediate fix:: + + sudo chmod 0644 /boot/vmlinuz-* + for k in /boot/vmlinuz-* + do sudo dpkg-statoverride --add --update root root 0644 /boot/vmlinuz-$k + done + +You likely want both this temporary fix and a more permanent fix; otherwise each +kernel update will revert to the default permissions and ExtractCode will stop +working for VM images extraction. + +Therefore follow these instructions: + +1. As sudo, create the file /etc/kernel/postinst.d/statoverride with this +content, devised by Kees Cook (@kees) in +https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725/comments/3 :: + + #!/bin/sh + version="$1" + # passing the kernel version is required + [ -z "${version}" ] && exit 0 + dpkg-statoverride --update --add root root 0644 /boot/vmlinuz-${version} + +2. Set executable permissions:: + + sudo chmod +x /etc/kernel/postinst.d/statoverride + +See also these links for a complete discussion: + + - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725 + - https://bugzilla.redhat.com/show_bug.cgi?id=1670790 + - https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662/comments/24 + + +Alternative +----------- + +These other tools are related and were considered before creating ExtractCode: + +These tools provide built-in, original extraction capabilities: + +- https://libarchive.org/ (integrated in ExtractCode) (BSD license) +- https://www.7-zip.org/ (integrated in ExtractCode) (LGPL license) +- https://theunarchiver.com/command-line (maintenance status unknown) (LGPL license) + +These tools are command line tools wrapping other extraction tools and are +similar to ExtractCode but with different goals: + +- https://github.com/wummel/patool (wrapper on many CLI tools) (GPL license) +- https://github.com/dtrx-py/dtrx (wrapper on a few CLI tools) (recently revived) (GPL license) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a4c950..ed2430f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,31 +7,31 @@ jobs: - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu16_cpython image_name: ubuntu-16.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -vvs + all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -vvs - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos1014_cpython image_name: macos-10.14 @@ -39,7 +39,7 @@ jobs: test_suites: all: tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos1015_cpython image_name: macos-10.15 diff --git a/configure b/configure index 1fdccc0..98d0cf7 100755 --- a/configure +++ b/configure @@ -1,44 +1,169 @@ #!/usr/bin/env bash # -# Copyright (c) nexB Inc. and others. +# Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/ for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # set -e #set -x -# source this script for a basic setup and configuration for local development +################################ +# A configuration script to set things up: +# create a virtualenv and install or update thirdparty packages. +# Source this script for initial configuration +# Use configure --help for details +# +# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz +# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +################################ +CLI_ARGS=$1 -CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +################################ +# Defaults. Change these variables to customize this script +################################ +# Requirement arguments passed to pip and used by default or with --dev. +REQUIREMENTS="--editable .[full]" +DEV_REQUIREMENTS="--editable .[full,testing]" -if [[ "$1" == "--clean" ]]; then - rm -rf "$CONFIGURE_ROOT_DIR/tmp" - exit +# where we create a virtualenv +VIRTUALENV_DIR=tmp + +# Cleanable files and directories with the --clean option +CLEANABLE=" + build + tmp" + +# extra arguments passed to pip +PIP_EXTRA_ARGS=" " + +# the URL to download virtualenv.pyz if needed +VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +################################ + + +################################ +# Current directory where this script lives +CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + + +################################ +# Set the quiet flag to empty if not defined +if [[ "$CFG_QUIET" == "" ]]; then + CFG_QUIET=" " fi -if [[ "$PYTHON_EXE" == "" ]]; then - PYTHON_EXE=python3 +################################ +# find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi fi -function setup { - # create a virtualenv on Python - mkdir -p $CONFIGURE_ROOT_DIR/tmp - wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz - $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp - source $CONFIGURE_ROOT_DIR/tmp/bin/activate - $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit } -setup +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} -$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[full] -e .[testing] -if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then - source "$CONFIGURE_ROOT_DIR/tmp/bin/activate" -fi +create_virtualenv() { + # create a virtualenv for Python + # Note: we do not use the bundled Python 3 "venv" because its behavior and + # presence is not consistent across Linux distro and sometimes pip is not + # included either by default. The virtualenv.pyz app cures all these issues. + + VENV_DIR="$1" + if [ ! -f "$CFG_BIN_DIR/python" ]; then + + mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" + + if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then + VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" + else + VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + fi + + $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ + --wheel embed --pip embed --setuptools embed \ + --seeder pip \ + --never-download \ + --no-periodic-update \ + --no-vcs-ignore \ + $CFG_QUIET \ + "$CFG_ROOT_DIR/$VENV_DIR" + fi +} + + +install_packages() { + # install requirements in virtualenv + # note: --no-build-isolation means that pip/wheel/setuptools will not + # be reinstalled a second time and reused from the virtualenv and this + # speeds up the installation. + # We always have the PEP517 build dependencies installed already. + + "$CFG_BIN_DIR/pip" install \ + --upgrade \ + --no-build-isolation \ + $CFG_QUIET \ + $PIP_EXTRA_ARGS \ + $1 +} + + +################################ +# Main command line entry point +CFG_DEV_MODE=0 +CFG_REQUIREMENTS=$REQUIREMENTS + +case "$CLI_ARGS" in + --help) cli_help;; + --clean) clean;; + --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; +esac + +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" set +e diff --git a/configure.bat b/configure.bat index b19aec3..19805e9 100644 --- a/configure.bat +++ b/configure.bat @@ -1,120 +1,180 @@ @echo OFF @setlocal -@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem Copyright (c) nexB Inc. and others. All rights reserved. +@rem SPDX-License-Identifier: Apache-2.0 +@rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +@rem See https://github.com/nexB/ for support or download. +@rem See https://aboutcode.org for more information about nexB OSS projects. + @rem ################################ -@rem # A configuration script for Windows -@rem # -@rem # The options and (optional) arguments are: -@rem # --clean : this is exclusive of anything else and cleans the environment -@rem # from built and installed files -@rem # -@rem # --python < path to python.exe> : this must be the first argument and set -@rem # the path to the Python executable to use. If < path to python.exe> is -@rem # set to "path", then the executable will be the python.exe available -@rem # in the PATH. +@rem # A configuration script to set things up: +@rem # create a virtualenv and install or update thirdparty packages. +@rem # Source this script for initial configuration +@rem # Use configure --help for details + +@rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz +@rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ -@rem Current directory where this .bat files lives -set CFG_ROOT_DIR=%~dp0 -@rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe -set PYTHON_EXECUTABLE= +@rem ################################ +@rem # Defaults. Change these variables to customize this script +@rem ################################ + +@rem # Requirement arguments passed to pip and used by default or with --dev. +set "REQUIREMENTS=--editable .[full]" +set "DEV_REQUIREMENTS=--editable .[full,testing]" + +@rem # where we create a virtualenv +set "VIRTUALENV_DIR=tmp" + +@rem # Cleanable files and directories to delete with the --clean option +set "CLEANABLE=build tmp" -@rem parse command line options and arguments -:collectopts -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b -if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts +@rem # extra arguments passed to pip +set "PIP_EXTRA_ARGS= " -@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run -if exist ""%CONFIGURED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% - goto run +@rem # the URL to download virtualenv.pyz if needed +set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +@rem ################################ + + +@rem ################################ +@rem # Current directory where this script lives +set CFG_ROOT_DIR=%~dp0 +set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + + +@rem ################################ +@rem # Set the quiet flag to empty if not defined +if not defined CFG_QUIET ( + set "CFG_QUIET= " ) -@rem If we have a command arg for Python use this as-is -if ""%PROVIDED_PYTHON%""==""path"" ( - @rem use a bare python available in the PATH - set PYTHON_EXECUTABLE=python - goto run + +@rem ################################ +@rem # Main command line entry point +set CFG_DEV_MODE=0 +set "CFG_REQUIREMENTS=%REQUIREMENTS%" + +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (goto clean) +if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + set CFG_DEV_MODE=1 ) -if exist ""%PROVIDED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% - goto run +if "%1" EQU "--python" ( + echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" + echo "variable instead. Run configure --help for details." + exit /b 0 ) +@rem ################################ +@rem # find a proper Python to run +@rem # Use environment variables or a file if available. +@rem # Otherwise the latest Python by default. +if not defined PYTHON_EXECUTABLE ( + @rem # check for a file named PYTHON_EXECUTABLE + if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( + set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" + ) else ( + set "PYTHON_EXECUTABLE=py" + ) +) -@rem otherwise we search for a suitable Python interpreter -:find_python -@rem First check the existence of the "py" launcher (available in Python 3) -@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 -@rem if not, check if we have an old py 2.7 -@rem exist if all fails +:create_virtualenv +@rem # create a virtualenv for Python +@rem # Note: we do not use the bundled Python 3 "venv" because its behavior and +@rem # presence is not consistent across Linux distro and sometimes pip is not +@rem # included either by default. The virtualenv.pyz app cures all these issues. -where py >nul 2>nul -if %ERRORLEVEL% == 0 ( - @rem we have a py launcher, check for the availability of our required Python 3 version - py -3.6 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -3.6 - ) else ( - @rem we have no required python 3, let's try python 2: - py -2 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -2 - ) else ( - @rem we have py and no python 3 and 2, exit - echo * Unable to find an installation of Python. - exit /b 1 - ) +if not exist ""%CFG_BIN_DIR%\python.exe"" ( + if not exist "%CFG_BIN_DIR%" ( + mkdir %CFG_BIN_DIR% ) -) else ( - @rem we have no py launcher, check for a default Python 2 installation - if not exist ""%DEFAULT_PYTHON2%"" ( - echo * Unable to find an installation of Python. - exit /b 1 + + if exist ""%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz"" ( + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% ) else ( - set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + if not exist ""%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz"" ( + curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% + + if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% + ) + ) + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% ) ) +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) + -:run -@rem without this things may not always work on Windows 10, but this makes things slower -set PYTHONDONTWRITEBYTECODE=1 +:install_packages +@rem # install requirements in virtualenv +@rem # note: --no-build-isolation means that pip/wheel/setuptools will not +@rem # be reinstalled a second time and reused from the virtualenv and this +@rem # speeds up the installation. +@rem # We always have the PEP517 build dependencies installed already. -call mkdir "%CFG_ROOT_DIR%tmp" -call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz -call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" -call "%CFG_ROOT_DIR%tmp\Scripts\activate" -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[full] -e .[testing] +%CFG_BIN_DIR%\pip install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% -@rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% ) -endlocal -goto activate +exit /b 0 + + +@rem ################################ :cli_help -echo A configuration script for Windows -echo usage: configure [options] [path/to/config/directory] -echo. -echo The options and arguments are: -echo --clean : this is exclusive of anything else and cleans the environment -echo from built and installed files -echo. -echo --python path/to/python.exe : this is set to the path of an alternative -echo Python executable to use. If path/to/python.exe is set to "path", -echo then the executable will be the python.exe available in the PATH. -echo. - - -:activate -@rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( - "%CFG_ROOT_DIR%tmp\Scripts\activate" + echo An initial configuration script + echo " usage: configure [options]" + echo " " + echo The default is to configure for regular use. Use --dev for development. + echo " " + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo " " + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + exit /b 0 + + +:clean +@rem # Remove cleanable file and directories and files from the root dir. +echo "* Cleaning ..." +for %%F in (%CLEANABLE%) do ( + rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 + del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) +exit /b 0 diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-mac.yml deleted file mode 100644 index 752ae2e..0000000 --- a/etc/ci/azure-mac.yml +++ /dev/null @@ -1,36 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - test_suites: {} - python_architecture: x64 - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - strategy: - matrix: - ${{ each pyver in parameters.python_versions }}: - ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} - test_suite_label: ${{ tsuite.key }} - test_suite: ${{ tsuite.value }} - steps: - - checkout: self - fetchDepth: 10 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' - - - script: ./configure - displayName: 'Run Configure' - - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-posix.yml similarity index 84% rename from etc/ci/azure-linux.yml rename to etc/ci/azure-posix.yml index 752ae2e..0921d9b 100644 --- a/etc/ci/azure-linux.yml +++ b/etc/ci/azure-posix.yml @@ -19,6 +19,7 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 @@ -29,7 +30,11 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: ./configure + - script: | + python3 --version + python$(python_version) --version + echo "python$(python_version)" > PYTHON_EXECUTABLE + ./configure --dev displayName: 'Run Configure' - script: $(test_suite) diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index afe1686..03d8927 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -29,7 +29,10 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: configure --python path + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --dev displayName: 'Run Configure' - script: $(test_suite) diff --git a/extractcode b/extractcode new file mode 100755 index 0000000..bb91c2a --- /dev/null +++ b/extractcode @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# ScanCode is a trademark of nexB Inc. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +# A minimal shell wrapper to the CLI entry point fo ExtractCode + + +################################################################################### +# from https://raw.githubusercontent.com/mkropat/sh-realpath/58c03982cfd8accbcf0c4426a4adf0f120a8b2bb/realpath.sh +# realpath emulation for portability on *nix +# this allow running scancode from arbitrary locations and from symlinks +# +# Copyright (c) 2014 Michael Kropat +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + path=$(readlink -- "$1") + if [ $? -eq 0 ]; then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} + +################################################################################### +# Now run scancode proper + +EXTRACTCODE_BIN="$( realpath "${BASH_SOURCE[0]}" )" +EXTRACTCODE_ROOT_DIR="$( cd "$( dirname "${EXTRACTCODE_BIN}" )" && pwd )" + +EXTRACTCODE_CONFIGURED_PYTHON="$EXTRACTCODE_ROOT_DIR/tmp/bin/python" +if [ ! -f "$EXTRACTCODE_CONFIGURED_PYTHON" ]; then + echo "* Configuring ExtractCode for first use..." + CONFIGURE_QUIET=1 "$EXTRACTCODE_ROOT_DIR/configure" +fi + +"$EXTRACTCODE_ROOT_DIR/tmp/bin/extractcode" "$@" diff --git a/extractcode.ABOUT b/extractcode.ABOUT index 7ebfb73..75e231d 100644 --- a/extractcode.ABOUT +++ b/extractcode.ABOUT @@ -1,9 +1,9 @@ about_resource: . copyright: copyright (c) nexB. Inc. and others -description: A mostly universal archive extractor using z7zip, libarchve, other - libraries and the Python standard library for reliable archive extraction. - It is used by ScanCode toolkit and related projects -keywords: archive, extraction, libarchive, 7zip, scancode-toolkit +description: A mostly universal archive extractor using 7zip, libarchive and the + Python standard library for reliable archive extraction on Linux, Windows and + macOS. It is used by ScanCode toolkit and related projects. +keywords: archive, extraction, libarchive, 7zip, gzip, xz, lzma, bzip2, tar, ar, cpio, scancode-toolkit homepage_url: https://github.com/nexB/extractcode holder: nexB. Inc. and others holder_contact: info@aboutcode.org diff --git a/extractcode.bat b/extractcode.bat new file mode 100644 index 0000000..d9d081f --- /dev/null +++ b/extractcode.bat @@ -0,0 +1,32 @@ +@echo OFF + +@rem Copyright (c) nexB Inc. and others. All rights reserved. +@rem SPDX-License-Identifier: Apache-2.0 +@rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +@rem ScanCode is a trademark of nexB Inc. +@rem See https://github.com/nexB/extractcode for support or download. +@rem See https://aboutcode.org for more information about nexB OSS projects. + +@rem A wrapper to ExtractCode command line entry point + +set EXTRACTCODE_ROOT_DIR=%~dp0 +set EXTRACTCODE_CONFIGURED_PYTHON=%EXTRACTCODE_ROOT_DIR%Scripts\python.exe + +if not exist "%EXTRACTCODE_CONFIGURED_PYTHON%" goto configure +goto extractcode + +:configure +echo * Configuring ExtractCode for first use... +set CONFIGURE_QUIET=1 +call "%EXTRACTCODE_ROOT_DIR%configure" + +@rem Return a proper return code on failure +if %errorlevel% neq 0 ( + exit /b %errorlevel% +) + +:extractcode +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +"%EXTRACTCODE_ROOT_DIR%Scripts\extractcode" %* diff --git a/pyproject.toml b/pyproject.toml index 55fb92c..852f0fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,11 @@ [build-system] -requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 4"] +requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 6"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] +# this is used populated when creating a git archive +# and when there is .git dir and/or there is no git installed +fallback_version = "9999.$Format:%h-%cs$" [tool.pytest.ini_options] norecursedirs = [ @@ -41,6 +44,6 @@ python_functions = "test" addopts = [ "-rfExXw", - "--strict", + "--strict-markers", "--doctest-modules" ] diff --git a/setup.cfg b/setup.cfg index 0c05301..4276949 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ author_email = info@aboutcode.org license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 -description = A mostly universal archive extractor using z7zip, libarchve, other libraries and the Python standard library for reliable archive extraction. +description = A mostly universal archive extractor using 7zip, libarchive and the Python standard library for reliable archive extraction. long_description = file:README.rst url = https://github.com/nexB/extractcode classifiers = @@ -26,18 +26,40 @@ keywords = extraction libarchive 7zip + 7z + gzip + bzip2 + xz + lzma + lz4 + lzip + zstd + Z + tar + xar + ar + cpio + vmdk + qcow2 + vhd + iso + deb + cab + rpm + patch scancode-toolkit [options] package_dir= =src -packages = find: +packages=find: include_package_data = true zip_safe = false install_requires = - commoncode >= 21.1.21 + attrs >= 18.1, !=20.1.0 + commoncode >= 21.5.25 plugincode >= 21.1.21 - typecode >= 21.2.23 + typecode >= 21.5.25 patch >= 1.16 setup_requires = setuptools_scm[toml] >= 4 @@ -47,9 +69,9 @@ where=src [options.extras_require] full = - extractcode-7z >= 16.5.210223 - extractcode_libarchive >= 3.5.1.210223 - typecode[full] >= 21.2.23 + extractcode-7z >= 16.5.210525 + extractcode_libarchive >= 3.5.1.210525 + typecode[full] >= 21.5.25 testing = # upstream diff --git a/src/README.rst b/src/README.rst deleted file mode 100644 index ec651fc..0000000 --- a/src/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python source code (and installable data) in this directory. - diff --git a/src/extractcode/NOTICE b/src/extractcode/NOTICE index 65936b2..ad3576b 100644 --- a/src/extractcode/NOTICE +++ b/src/extractcode/NOTICE @@ -1,19 +1,8 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/extractcode/__init__.py b/src/extractcode/__init__.py index b41efdc..fb6095d 100644 --- a/src/extractcode/__init__.py +++ b/src/extractcode/__init__.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import logging @@ -37,12 +26,10 @@ from commoncode.system import on_linux logger = logging.getLogger(__name__) -DEBUG = False -# import sys -# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) -# logger.setLevel(logging.DEBUG) - -root_dir = join(dirname(__file__), 'bin') +TRACE = False +if TRACE: + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + logger.setLevel(logging.DEBUG) # Suffix added to extracted target_dir paths EXTRACT_SUFFIX = '-extract' @@ -66,13 +53,26 @@ 7: 'special_package', } -# note: do not include special_package in all by default -all_kinds = (regular, regular_nested, package, file_system, docs, patches, special_package) -default_kinds = (regular, regular_nested, package) +# note: we do not include special_package in all_kinds by default +all_kinds = ( + regular, + regular_nested, + package, + file_system, + docs, + patches, + special_package, +) + +default_kinds = ( + regular, + regular_nested, + package, +) # map user-visible extract types to tuples of "kinds" extract_types = { - 'default': (regular, regular_nested, package,), + 'default': default_kinds, 'all': all_kinds, 'package': (package,), 'filesystem': (file_system,), @@ -137,11 +137,11 @@ def remove_backslashes_and_dotdots(directory): def new_name(location, is_dir=False): """ Return a new non-existing location from a `location` usable to write a file - or create directory without overwriting existing files or directories in the same - parent directory, ignoring the case of the filename. + or create directory without overwriting existing files or directories in the + same parent directory, ignoring the case of the filename. - The case of the filename is ignored to ensure that similar results are returned - across case sensitive (*nix) and case insensitive file systems. + The case of the filename is ignored to ensure that similar results are + returned across case sensitive (*nix) and case insensitive file systems. To find a new unique filename, this tries new names this way: * pad a directory name with _X where X is an incremented number. diff --git a/src/extractcode/api.py b/src/extractcode/api.py index a5bb86c..d250f86 100644 --- a/src/extractcode/api.py +++ b/src/extractcode/api.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # """ @@ -23,22 +12,37 @@ """ -def extract_archives(location, recurse=True, replace_originals=False, ignore_pattern=()): +def extract_archives( + location, + recurse=True, + replace_originals=False, + ignore_pattern=(), + all_formats=False, +): """ Yield ExtractEvent while extracting archive(s) and compressed files at - `location`. If `recurse` is True, extract nested archives-in-archives - recursively. + `location`. + + If `recurse` is True, extract nested archives-in-archives recursively. + If `all_formats` is True, extract all supported archives formats. + Archives and compressed files are extracted in a directory named "-extract" created in the same directory as the archive. + Note: this API is returning an iterable and NOT a sequence. """ + from extractcode.extract import extract from extractcode import default_kinds + from extractcode import all_kinds + + kinds = all_kinds if all_formats else default_kinds + for xevent in extract( location=location, - kinds=default_kinds, + kinds=kinds, recurse=recurse, replace_originals=replace_originals, - ignore_pattern=ignore_pattern + ignore_pattern=ignore_pattern, ): yield xevent diff --git a/src/extractcode/archive.py b/src/extractcode/archive.py index 32b52bc..ae5e268 100644 --- a/src/extractcode/archive.py +++ b/src/extractcode/archive.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # from collections import namedtuple @@ -26,7 +15,6 @@ from commoncode import filetype from commoncode import functional from commoncode.ignore import is_ignored - from typecode import contenttype from extractcode import all_kinds @@ -38,9 +26,11 @@ from extractcode import patches from extractcode import special_package +from extractcode import libarchive2 from extractcode import patch from extractcode import sevenzip -from extractcode import libarchive2 +from extractcode import vmimage + from extractcode.uncompress import uncompress_gzip from extractcode.uncompress import uncompress_bzip2 @@ -79,8 +69,19 @@ - http://en.wikipedia.org/wiki/List_of_file_formats#Archive_and_compressed """ -# if strict, all hanlders criteria must be matched for it to be selected -Handler = namedtuple('Handler', ['name', 'filetypes', 'mimetypes', 'extensions', 'kind', 'extractors', 'strict']) +# if strict, all handlers criteria must be matched for a handler to be selected +Handler = namedtuple( + 'Handler', + [ + 'name', + 'filetypes', + 'mimetypes', + 'extensions', + 'kind', + 'extractors', + 'strict', + ] +) def can_extract(location): @@ -96,32 +97,53 @@ def can_extract(location): def should_extract(location, kinds, ignore_pattern=()): """ - Return True if this location should be extracted based on the provided - kinds + Return True if this location should be extracted based on the provided kinds """ location = os.path.abspath(os.path.expanduser(location)) - ignore_pattern = {extension : 'User ignore: Supplied by --ignore' for extension in ignore_pattern} + ignore_pattern = {extension : 'User ignore: Supplied by --ignore' + for extension in ignore_pattern} should_ignore = is_ignored(location, ignore_pattern) - if get_extractor(location, kinds) and not should_ignore: + extractor = get_extractor(location, kinds=kinds) + + if TRACE_DEEP: + logger.debug( + f' should_extract: extractor: {extractor}, ' + f'should_ignore: {should_ignore}' + ) + + if extractor and not should_ignore: return True def get_extractor(location, kinds=all_kinds): """ - Return an extraction callable that can extract the file at location or - an None if no extract function is found. + Return an extraction callable that can extract the file at ``location`` or + None if no extraction callable function is found. + Limit the search for an extractor to the ``kinds`` list of archive kinds. + See extractcode.all_kinds for details. + + An extraction callable should accept these arguments: + - location of the file to extract + - target_dir where to extract + It should extract files from the `location` in the `target_dir` directory. + It must return a list of warning messages if any or an empty list. + It must raise Exceptions on errors. """ assert location location = os.path.abspath(os.path.expanduser(location)) - extractors = get_extractors(location, kinds) + extractors = get_extractors(location, kinds=kinds) if not extractors: + if TRACE_DEEP: + logger.debug(f' get_extractor: not extractors: {extractors}') return None if len(extractors) == 2: extractor1, extractor2 = extractors - nested_extractor = functional.partial(extract_twice, - extractor1=extractor1, - extractor2=extractor2) + nested_extractor = functional.partial( + extract_twice, + extractor1=extractor1, + extractor2=extractor2, + ) return nested_extractor elif len(extractors) == 1: return extractors[0] @@ -135,23 +157,38 @@ def get_extractors(location, kinds=all_kinds): location or an empty list. """ handler = get_best_handler(location, kinds) + if TRACE_DEEP: + logger.debug(f' get_extractors: handler: {handler}') + return handler and handler.extractors or [] def get_best_handler(location, kinds=all_kinds): """ - Return the best handler of None for the file at location. + Return the best handler for the file at `location` or None . """ location = os.path.abspath(os.path.expanduser(location)) if not filetype.is_file(location): return + handlers = list(get_handlers(location)) if TRACE_DEEP: - logger.debug('get_best_handler: handlers: %(handlers)r ' % locals()) + logger.debug(f' get_best_handler: handlers: {handlers}') + if not handlers: + return - if handlers: - candidates = score_handlers(handlers) - return candidates and pick_best_handler(candidates, kinds) + candidates = list(score_handlers(handlers)) + if TRACE_DEEP: + logger.debug(f' get_best_handler: candidates: {candidates}') + if not candidates: + if TRACE_DEEP: + logger.debug(f' get_best_handler: candidates: {candidates}') + return + + picked = pick_best_handler(candidates, kinds=kinds) + if TRACE_DEEP: + logger.debug(f' get_best_handler: picked: {picked}') + return picked def get_handlers(location): @@ -166,7 +203,9 @@ def get_handlers(location): mtype = T.mimetype_file if TRACE_DEEP: - logger.debug('get_handlers: processing %(location)s: ftype: %(ftype)s, mtype: %(mtype)s ' % locals()) + logger.debug( + 'get_handlers: processing %(location)s: ' + 'ftype: %(ftype)s, mtype: %(mtype)s ' % locals()) for handler in archive_handlers: if not handler.extractors: continue @@ -177,16 +216,29 @@ def get_handlers(location): # default to False type_matched = handler.filetypes and any(t in ftype for t in handler.filetypes) + if TRACE_DEEP: + logger.debug(f' get_handlers: handler.filetypes={handler.filetypes}') mime_matched = handler.mimetypes and any(m in mtype for m in handler.mimetypes) exts = handler.extensions if exts: extension_matched = exts and location.lower().endswith(exts) if TRACE_DEEP: - logger.debug(' get_handlers: matched type: %(type_matched)s, mime: %(mime_matched)s, ext: %(extension_matched)s' % locals()) - - if handler.strict and not all([type_matched, mime_matched, extension_matched]): - logger.debug(' get_handlers: skip strict' % locals()) + print( + f' get_handlers: matched type: {type_matched}, ' + f'mime: {mime_matched}, ext: {extension_matched}' % locals() + ) + + if ( + handler.strict + and not ( + type_matched + and mime_matched + and extension_matched + ) + ): + if TRACE_DEEP: + print(f' get_handlers: skip strict: {handler.name}') continue if type_matched or mime_matched or extension_matched: @@ -201,10 +253,18 @@ def score_handlers(handlers): Score candidate handlers. Higher score is better. """ for handler, type_matched, mime_matched, extension_matched in handlers: + if TRACE_DEEP: + logger.debug( + f' score_handlers: handler={handler}, ' + f'type_matched={type_matched}, ' + f'mime_matched={mime_matched}, ' + f'extension_matched={extension_matched}' + ) score = 0 # increment kind value: higher kinds numerical values are more # specific by design score += handler.kind + if TRACE_DEEP: logger.debug(f' score_handlers: score += handler.kind {score}') # increment score based on matched criteria if type_matched and mime_matched and extension_matched: @@ -255,6 +315,10 @@ def pick_best_handler(candidates, kinds): """ # sort by increasing scores scored = sorted(candidates, reverse=True) + + if TRACE_DEEP: + logger.debug(f' pick_best_handler: scored: {scored}') + if not scored: return @@ -396,17 +460,30 @@ def try_to_extract(location, target_dir, extractor): extract_deb = libarchive2.extract -# sevenzip is best for windows lib formats and works fine otherwise. libarchive works on standard ar formats. -extract_ar = functional.partial(extract_with_fallback, extractor1=libarchive2.extract, extractor2=sevenzip.extract) +# sevenzip is best for windows lib formats and works fine otherwise. libarchive +# works on standard ar formats. +extract_ar = functional.partial( + extract_with_fallback, + extractor1=libarchive2.extract, + extractor2=sevenzip.extract, +) extract_msi = sevenzip.extract extract_cpio = libarchive2.extract # sevenzip should be best at extracting 7zip but most often libarchive is better first -extract_7z = functional.partial(extract_with_fallback, extractor1=libarchive2.extract, extractor2=sevenzip.extract) +extract_7z = functional.partial( + extract_with_fallback, + extractor1=libarchive2.extract, + extractor2=sevenzip.extract, +) # libarchive is best for the run of the mill zips, but sevenzip sometimes is better -extract_zip = functional.partial(extract_with_fallback, extractor1=libarchive2.extract, extractor2=sevenzip.extract) +extract_zip = functional.partial( + extract_with_fallback, + extractor1=libarchive2.extract, + extractor2=sevenzip.extract, +) extract_springboot = functional.partial(try_to_extract, extractor=extract_zip) @@ -420,6 +497,7 @@ def try_to_extract(location, target_dir, extractor): extract_xz = sevenzip.extract extract_lzma = sevenzip.extract extract_squashfs = sevenzip.extract +extract_vm_image = vmimage.extract extract_cab = sevenzip.extract extract_nsis = sevenzip.extract extract_ishield = sevenzip.extract @@ -461,7 +539,12 @@ def try_to_extract(location, target_dir, extractor): OfficeDocHandler = Handler( name='Office doc', - filetypes=('zip archive', 'microsoft word 2007+', 'microsoft excel 2007+', 'microsoft powerpoint 2007+'), + filetypes=( + 'zip archive', + 'microsoft word 2007+', + 'microsoft excel 2007+', + 'microsoft powerpoint 2007+', + ), mimetypes=('application/zip', 'application/vnd.openxmlformats',), # Extensions of office documents that are zip files too extensions=( @@ -499,7 +582,7 @@ def try_to_extract(location, target_dir, extractor): strict=True ) - # see http://tools.android.com/tech-docs/new-build-system/aar-formats +# see http://tools.android.com/tech-docs/new-build-system/aar-formats AndroidLibHandler = Handler( name='Android library', filetypes=('zip archive',), @@ -773,8 +856,16 @@ def try_to_extract(location, target_dir, extractor): name='Tar bzip2', filetypes=('bzip2 compressed',), mimetypes=('application/x-bzip2',), - extensions=('.tar.bz2', '.tar.bz', '.tar.bzip', '.tar.bzip2', - '.tbz', '.tbz2', '.tb2', '.tarbz2',), + extensions=( + '.tar.bz2', + '.tar.bz', + '.tar.bzip', + '.tar.bzip2', + '.tbz', + '.tbz2', + '.tb2', + '.tarbz2', + ), kind=regular_nested, extractors=[extract_tar], strict=False @@ -822,10 +913,11 @@ def try_to_extract(location, target_dir, extractor): NugetHandler = Handler( name='Nuget', - # weirdly enough the detection by libmagic is sometimes wrong - # TODO file a bug upstream - # this is due to this: https://en.wikipedia.org/wiki/Open_Packaging_Conventions#File_formats_using_the_OPC + # TODO: file a bug upstream + # Weirdly enough the detection by libmagic is sometimes wrong + # this is due to this issue: # being recognized by libmagic as an OOXML file + # https://en.wikipedia.org/wiki/Open_Packaging_Conventions#File_formats_using_the_OPC filetypes=('zip archive', 'microsoft ooxml',), mimetypes=('application/zip', 'application/octet-stream',), extensions=('.nupkg',), @@ -867,7 +959,10 @@ def try_to_extract(location, target_dir, extractor): DebHandler = Handler( name='Debian package', filetypes=('debian binary package',), - mimetypes=('application/vnd.debian.binary-package', 'application/x-archive',), + mimetypes=( + 'application/vnd.debian.binary-package', + 'application/x-archive', + ), extensions=('.deb', '.udeb',), kind=package, extractors=[extract_deb], @@ -985,7 +1080,7 @@ def try_to_extract(location, target_dir, extractor): ) SquashfsHandler = Handler( - name='squashfs FS', + name='SquashFS disk image', filetypes=('squashfs',), mimetypes=(), extensions=(), @@ -994,6 +1089,37 @@ def try_to_extract(location, target_dir, extractor): strict=False ) +QCOWHandler = Handler( + # note that there are v1, v2 and v3 formats. + name='QEMU QCOW2 disk image', + filetypes=('qemu qcow2 image', 'qemu qcow image',), + mimetypes=('application/octet-stream',), + extensions=('.qcow2', '.qcow', '.qcow2c', '.img',), + kind=file_system, + extractors=[extract_vm_image], + strict=True, +) + +VMDKHandler = Handler( + name='VMDK disk image', + filetypes=('vmware4 disk image',), + mimetypes=('application/octet-stream',), + extensions=('.vmdk',), + kind=file_system, + extractors=[extract_vm_image], + strict=True, +) + +VirtualBoxHandler = Handler( + name='VirtualBox disk image', + filetypes=('virtualbox disk image',), + mimetypes=('application/octet-stream',), + extensions=('.vdi',), + kind=file_system, + extractors=[extract_vm_image], + strict=True, +) + PatchHandler = Handler( name='Patch', filetypes=('diff', 'patch',), @@ -1062,5 +1188,8 @@ def try_to_extract(location, target_dir, extractor): AppleDmgHandler, IsoImageHandler, SquashfsHandler, - PatchHandler + QCOWHandler, + VMDKHandler, + VirtualBoxHandler, + PatchHandler, ] diff --git a/src/extractcode/cli.py b/src/extractcode/cli.py index 5591e5d..b4064b3 100644 --- a/src/extractcode/cli.py +++ b/src/extractcode/cli.py @@ -1,22 +1,10 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import os @@ -96,26 +84,85 @@ class ExtractCommand(cliutils.BaseCommand): @click.command(name='extractcode', epilog=epilog_text, cls=ExtractCommand) @click.pass_context -@click.argument('input', metavar='', type=click.Path(exists=True, readable=True)) - -@click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.') -@click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.') -@click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).') -@click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.') -@click.option('--ignore', default=[], multiple=True, help='Ignore files/directories following a glob-pattern.') +@click.argument( + 'input', + metavar='', + type=click.Path(exists=True, readable=True), +) + +@click.option( + '--verbose', + is_flag=True, + help='Print verbose file-by-file progress messages.', +) +@click.option( + '--quiet', + is_flag=True, + help='Do not print any summary or progress message.', +) +@click.option( + '--shallow', + is_flag=True, + help='Do not extract recursively nested archives in archives.', +) +@click.option( + '--replace-originals', + is_flag=True, + help='Replace extracted archives by the extracted content.', +) +@click.option( + '--ignore', + default=[], + multiple=True, + help='Ignore files/directories matching this glob pattern.', +) + +@click.option( + '--all-formats', + is_flag=True, + help='Extract archives from all known formats.', +) @click.help_option('-h', '--help') -@click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ExtractCode and licensing and exit.') -@click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.') -def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, ignore, *args, **kwargs): # NOQA - """extract archives and compressed files found in the file or directory tree. +@click.option( + '--about', + is_flag=True, + is_eager=True, + callback=print_about, + help='Show information about ExtractCode and its licensing and exit.', +) +@click.option( + '--version', + is_flag=True, + is_eager=True, + callback=print_version, + help='Show the version and exit.', +) +def extractcode( + ctx, + input, # NOQA + verbose, + quiet, + shallow, + replace_originals, + ignore, + all_formats, + *args, + **kwargs, +): + """extract archives and compressed files in the file or directory tree. Archives found inside an extracted archive are extracted recursively. + Use --shallow for a shallow extraction. Extraction for each archive is done in-place in a new directory named '-extract' created side-by-side with an archive. """ - abs_location = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input))) + abs_location = fileutils.as_posixpath( + os.path.abspath( + os.path.expanduser(input) + ) + ) def extract_event(item): """ @@ -125,17 +172,26 @@ def extract_event(item): return '' if not item: return '' + source = item.source if not isinstance(source, str): source = toascii(source, translit=True).decode('utf-8', 'replace') + if verbose: if item.done: return '' - line = source and get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or '' + line = source and get_relative_path( + path=source, + len_base_path=len_base_path, + base_is_dir=base_is_dir, + ) or '' + else: line = source and fileutils.file_name(source) or '' + if not isinstance(line, str): line = toascii(line, translit=True).decode('utf-8', 'replace') + return 'Extracting: %(line)s' % locals() def display_extract_summary(): @@ -149,13 +205,27 @@ def display_extract_summary(): has_errors = has_errors or bool(xev.errors) has_warnings = has_warnings or bool(xev.warnings) source = fileutils.as_posixpath(xev.source) + if not isinstance(source, str): source = toascii(source, translit=True).decode('utf-8', 'replace') - source = get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) + + source = get_relative_path( + path=source, + len_base_path=len_base_path, + base_is_dir=base_is_dir, + ) + for e in xev.errors: - echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red') + echo_stderr( + 'ERROR extracting: %(source)s: %(e)s' % locals(), + fg='red' + ) + for warn in xev.warnings: - echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow') + echo_stderr( + 'WARNING extracting: %(source)s: %(warn)s' % locals(), + fg='yellow' + ) summary_color = 'green' if has_warnings: @@ -174,10 +244,16 @@ def display_extract_summary(): has_extract_errors = False extractibles = extract_archives( - abs_location, recurse=not shallow, replace_originals=replace_originals, ignore_pattern=ignore) + abs_location, + recurse=not shallow, + replace_originals=replace_originals, + ignore_pattern=ignore, + all_formats=all_formats, + ) if not quiet: echo_stderr('Extracting archives...', fg='green') + with cliutils.progressmanager(extractibles, item_show_func=extract_event, verbose=verbose) as extraction_events: @@ -187,7 +263,9 @@ def display_extract_summary(): if repr(xev) not in unique_extract_events_with_errors: extract_result_with_errors.append(xev) unique_extract_events_with_errors.add(repr(xev)) + display_extract_summary() + else: for xev in extractibles: if xev.done and (xev.warnings or xev.errors): @@ -199,9 +277,9 @@ def display_extract_summary(): def get_relative_path(path, len_base_path, base_is_dir): """ - Return a posix relative path from the posix 'path' relative to a - base path of `len_base_path` length where the base is a directory if - `base_is_dir` True or a file otherwise. + Return a posix relative path from the posix 'path' relative to a base path + of `len_base_path` length where the base is a directory if `base_is_dir` + True or a file otherwise. """ path = os.fsdecode(path) if base_is_dir: diff --git a/src/extractcode/extract.py b/src/extractcode/extract.py index 4c4540b..41d4ed6 100644 --- a/src/extractcode/extract.py +++ b/src/extractcode/extract.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import logging @@ -30,7 +19,7 @@ from commoncode import fileutils from commoncode import ignore -import extractcode +import extractcode # NOQA import extractcode.archive logger = logging.getLogger(__name__) @@ -42,15 +31,15 @@ logger.setLevel(logging.DEBUG) """ -Extract archives and compressed files recursively to get the file content available for -further processing. This the high level extraction entry point. +Extract archives and compressed files recursively to get the file content +available for further processing. This the high level extraction entry point. This is NOT a general purpose un-archiver. The code tries hard to do the right thing, BUT the extracted files are not meant to be something that can be -faithfully re-archived to get an equivalent archive. The purpose instead is -to extract the content of the archives as faithfully and safely as possible to -make this content available for scanning: some paths may be altered. Some files -may be altered or skipped entirely. +faithfully re-archived to get an equivalent archive. The purpose instead is to +extract the content of the archives as faithfully and safely as possible to make +this content available for scanning: some paths may be altered. Some files may +be altered or skipped entirely. In particular: @@ -61,7 +50,7 @@ - Symlinks may be replaced by plain file copies as if they were regular files. Hardlinks may be recreated as regular files, not as hardlinks to the original - file. + files. - Files and directories may be renamed when their name is a duplicate. And a name may be considered a duplicate ignore upper and lower case mixes even @@ -123,21 +112,32 @@ def extract( if recurse and a nested archive is found, it is extracted to full depth first before resuming the file system walk. """ + + extract_events = extract_files( + location=location, + kinds=kinds, + recurse=recurse, + ignore_pattern=ignore_pattern, + ) + processed_events = [] processed_events_append = processed_events.append - for event in extract_files(location, kinds, recurse, ignore_pattern): + for event in extract_events: yield event if replace_originals: processed_events_append(event) - # move files around + # move files around when done if replace_originals: for xevent in reversed(processed_events): if xevent.done: source = xevent.source target = xevent.target if TRACE: - logger.debug('extract:replace_originals: replace %(source)r by %(target)r' % locals()) + logger.debug( + 'extract:replace_originals: replace ' + '%(source)r by %(target)r' % locals() + ) fileutils.delete(source) fileutils.copytree(target, source) fileutils.delete(target) @@ -155,7 +155,9 @@ def extract_files( Extract only archives of a kind listed in the `kinds` kind tuple. If `recurse` is True, extract recursively archives nested inside other - archives. If `recurse` is false, then do not extract further an already + archives. + + If `recurse` is false, then do not extract further an already extracted archive identified by the corresponding extract suffix location. """ ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={}) @@ -165,7 +167,8 @@ def extract_files( abs_location = abspath(expanduser(location)) for top, dirs, files in fileutils.walk(abs_location, ignored): if TRACE: - logger.debug('extract:walk: top: %(top)r dirs: %(dirs)r files: r(files)r' % locals()) + logger.debug( + 'extract:walk: top: %(top)r dirs: %(dirs)r files: r(files)r' % locals()) if not recurse: if TRACE: @@ -174,7 +177,10 @@ def extract_files( if extractcode.is_extraction_path(d): dirs.remove(d) if TRACE: - logger.debug('extract:walk: not recurse: removed dirs:' + repr(drs.symmetric_difference(set(dirs)))) + logger.debug( + 'extract:walk: not recurse: removed dirs:' + +repr(drs.symmetric_difference(set(dirs))) + ) for f in files: loc = join(top, f) @@ -193,7 +199,11 @@ def extract_files( logger.debug('extract:target: %(target)r' % locals()) # extract proper - for xevent in extract_file(loc, target, kinds): + for xevent in extract_file( + location=loc, + target=target, + kinds=kinds, + ): if TRACE: logger.debug('extract:walk:extraction event: %(xevent)r' % locals()) yield xevent @@ -217,10 +227,11 @@ def extract_file( target, kinds=extractcode.default_kinds, verbose=False, + all_formats=False, ): """ - Extract a single archive at `location` in the `target` directory if it is - of a kind supported in the `kinds` kind tuple. + Extract a single archive at `location` in the `target` directory if it is of + a kind supported in the `kinds` kind tuple. """ warnings = [] errors = [] @@ -228,10 +239,20 @@ def extract_file( if TRACE: emodule = getattr(extractor, '__module__', '') ename = getattr(extractor, '__name__', '') - logger.debug(f'extract_file: extractor: for: {location} with kinds: {kinds}: {emodule}.{ename}') + logger.debug( + f'extract_file: extractor: for: {location} with kinds: ' + f'{kinds}: {emodule}.{ename}' + ) if extractor: - yield ExtractEvent(location, target, done=False, warnings=[], errors=[]) + yield ExtractEvent( + source=location, + target=target, + done=False, + warnings=[], + errors=[], + ) + try: # extract first to a temp directory: if there is an error, the # extracted files will not be moved to target @@ -247,7 +268,14 @@ def extract_file( errors.append(traceback.format_exc()) if TRACE: tb = traceback.format_exc() - logger.debug('extract_file: ERROR: %(location)r: %(errors)r\n%(e)r\n%(tb)s' % locals()) + logger.debug( + 'extract_file: ERROR: %(location)r: %(errors)r\n%(e)r\n%(tb)s' % locals()) finally: - yield ExtractEvent(location, target, done=True, warnings=warnings, errors=errors) + yield ExtractEvent( + source=location, + target=target, + done=True, + warnings=warnings, + errors=errors, + ) diff --git a/src/extractcode/libarchive2.py b/src/extractcode/libarchive2.py index cedc4f2..6402e4d 100644 --- a/src/extractcode/libarchive2.py +++ b/src/extractcode/libarchive2.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # from functools import partial @@ -23,6 +12,7 @@ import logging import mmap import os +import warnings import ctypes.util from ctypes import c_char_p, c_wchar_p @@ -38,6 +28,7 @@ from commoncode import fileutils from commoncode import paths from commoncode import text +from commoncode.system import on_windows import extractcode from extractcode import ExtractError @@ -54,29 +45,29 @@ logger.setLevel(logging.DEBUG) """ -libarchive2 is a minimal and specialized wrapper around a vendored libarchive archive -extraction library. It only deals with archive extraction and does not know how to -create archives. +libarchive2 is a minimal and specialized wrapper around a vendored libarchive +archive extraction library. It only deals with archive extraction and does not +know how to create archives. -Its main purpose is to try hard to extract files from archives on multiple OSes and -makes some compromises in doing so: +Its main purpose is to try hard to extract files from archives on multiple OSes +and makes some compromises in doing so: - special files and links may be skipped entirely and not extracted at all. -- relative paths are resolved to ensure that files are always extracted under a root - extraction directory. +- relative paths are resolved to ensure that files are always extracted under a + root extraction directory. - files and directories may be renamed if they are not unique (ignoring case) in their extraction directory. -- files and directories are renamed by "transliterating" their names to plain ASCII - if their name contain non-ASCI characters. +- files and directories are renamed by "transliterating" their names to plain + ASCII if their name contain non-ASCI characters. -- files and directories are renamed if they contain characters or names that are not - portable on common OSes (e.g. COM1, ":", "*", etc) +- files and directories are renamed if they contain characters or names that are + not portable on common OSes (e.g. COM1, ":", "*", etc) -- permissions and modes are ignored entirely when extracting files to esnure that - extracted files are always readable. +- permissions and modes are ignored entirely when extracting files to ensure + that extracted files are always readable. It is inspired from several libarchive bindings such as libarchive_c and python-libarchive for Python and other similar wrappers for Ruby such as @@ -84,33 +75,61 @@ """ # keys for plugin-provided locations -EXTRACTCODE_LIBARCHIVE_LIBDIR = 'extractcode.libarchive.libdir' EXTRACTCODE_LIBARCHIVE_DLL = 'extractcode.libarchive.dll' + +EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR = 'EXTRACTCODE_LIBARCHIVE_PATH' + _LIBRARY_NAME = 'libarchive' def load_lib(): """ - Return the loaded libarchive shared library object from plugin-provided path. + Return the libarchive shared library object loaded from either: + - an environment variable ``EXTRACTCODE_LIBARCHIVE_PATH`` + - a plugin-provided path, + - the system PATH. + Raise an Exception if no libarchive can be found. """ from plugincode.location_provider import get_location - dll = get_location(EXTRACTCODE_LIBARCHIVE_DLL) - libdir = get_location(EXTRACTCODE_LIBARCHIVE_LIBDIR) - if not (dll and libdir) or not os.path.isfile(dll) or not os.path.isdir(libdir): - filepath = ctypes.util.find_library(_LIBRARY_NAME) - libarchive = ctypes.cdll.LoadLibrary(filepath) - if not libarchive: - raise ImportError( - 'CRITICAL: libarchive DLL is not installed. ' - 'Unable to continue: you need to install a valid extractcode-libarchive ' - 'plugin with a valid libarchive DLL available ' - 'or to have "libarchive" library installed in your system.' - ) - logger.warning('Cannot to use plugin for libarchive, defaulting to system library at ' + filepath) - return libarchive - return command.load_shared_library(dll, libdir) - + # try the environment first + dll_loc = os.environ.get(EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR) + + # try a plugin-provided path second + if not dll_loc: + dll_loc = get_location(EXTRACTCODE_LIBARCHIVE_DLL) + + # try the standard locations with find_library + if not dll_loc: + libarch_loc = ctypes.util.find_library(_LIBRARY_NAME) + libarchive = ctypes.cdll.LoadLibrary(libarch_loc) + if libarchive: + warnings.warn( + 'Using "libarchive" library found in a system location. ' + 'Install instead a extractcode-libarchive plugin for best support.' + ) + return libarchive + + # try the PATH + if not dll_loc: + dll = 'libarchive.dll' if on_windows else 'libarchive.so' + dll_loc = command.find_in_path(dll) + if dll_loc: + warnings.warn( + 'Using "libarchive" library found in the PATH. ' + 'Install instead a extractcode-libarchive plugin for best support.' + ) + + if not dll_loc or not os.path.isfile(dll_loc): + raise Exception( + 'CRITICAL: libarchive DLL is not installed. ' + 'Unable to continue: you need to install a valid extractcode-libarchive ' + 'plugin with a valid libarchive DLL available. ' + f'OR set the {EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR} environment variable. ' + 'OR install libarchive as a system package. ' + 'OR ensure libarchive is available in the system PATH.' + ) + return command.load_shared_library(dll_loc) def set_env_with_tz(): @@ -177,7 +196,8 @@ def extract(location, target_dir, skip_symlinks=True): logger.debug('skipping: {}'.format(entry)) if entry.issym and not skip_symlinks: - raise NotImplemented('extraction of symlinks with libarchive is not yet implemented.') + raise NotImplemented( + 'extraction of symlinks with libarchive is not yet implemented.') continue if TRACE: @@ -207,23 +227,24 @@ class Archive(object): Represent an iterable archive containing a list of Entry objects. Archive is designed to be used as a context manager with the "with" syntax: + with Archive('some.tgz') as archive: for entry in archive: - # dome something with entry + # do something with entry """ def __init__(self, location, uncompress=True, extract=True, block_size=10240): """ Build an Archive object from file at `location`. - If `uncompress` is True, the archive will be uncompressed first if compressed. - (e.g. a tar.gz will be ungzipped). + If `uncompress` is True, the archive will be uncompressed first if + compressed. (e.g. a tar.gz will be ungzipped). - If `extract` is True, the archive will be extracted if this is an archive. - (e.g. a cpio will be extracted). + If `extract` is True, the archive will be extracted if this is an + archive. (e.g. a cpio will be extracted). - If both are True, the archive will be uncompressed then extracted as needed. - (e.g. a tar.xz will be unxzed then untarred at once). + If both are True, the archive will be uncompressed then extracted as + needed. (e.g. a tar.xz will be unxzed then untarred at once). """ msg = 'At least one of `uncompress` or `extract` flag is required.' assert uncompress or extract, msg @@ -236,9 +257,9 @@ def __init__(self, location, uncompress=True, extract=True, block_size=10240): def open(self): """ - Open the archive for reading. - You must call close() when done to free up resources and avoid leaks. - Or use instead the Archive class as a context manager with the "with" keyword. + Open the archive for reading. You must call close() when done to free up + resources and avoid leaks. Or use instead the Archive class as a context + manager with the "with" keyword. """ # first close any existing opened struct for this file self.close() @@ -256,9 +277,9 @@ def open(self): def close(self): """ - Release any memory held by the underlying librachive for this archive. You - must call close() when done with an archive to free up resources and avoid - leaks. + Release any memory held by the underlying librachive for this archive. + You must call close() when done with an archive to free up resources and + avoid leaks. """ if self.archive_struct: free_archive(self.archive_struct) @@ -266,7 +287,7 @@ def close(self): def iter(self): """ - Yield Entry for this archive. + Yield Entry(ies) for this archive. """ assert self.archive_struct, 'Archive must be used as a context manager.' entry_struct = new_entry() @@ -314,7 +335,7 @@ class attributes. Some attributes are not handled on purpose because they by design to ensure extracted files are readable/writable and owned by the extracting user. """ - # TODO: re-check if users and groups may have some value for origin determination? + # TODO: re-check if users/groups may have some value for origin determination? # an archive object archive = attr.ib(repr=False) @@ -396,10 +417,10 @@ def get_path(self, func, func_w): def write(self, target_dir, transform_path=lambda x: x, skip_links=True): """ - Write entry to a file or directory saved relatively to the `target_dir` and - return the path where the file or directory was written or None if nothing - was written to disk. `transform_path` is a callable taking a path and - returning a transformed path such as resolving relative paths, + Write entry to a file or directory saved relatively to the `target_dir` + and return the path where the file or directory was written or None if + nothing was written to disk. `transform_path` is a callable taking a + path and returning a transformed path such as resolving relative paths, transliterating non-portable characters or other path transformations. The default is a no-op lambda. """ @@ -442,7 +463,10 @@ def write(self, target_dir, transform_path=lambda x: x, skip_links=True): # TODO: return some warning when original path has been renamed? unique_path = extractcode.new_name(target_path, is_dir=False) if TRACE: - logger.debug('path: \ntarget_path: {}\nunique_path: {}'.format(target_path, unique_path)) + logger.debug( + f'path: \ntarget_path: {target_path}\n' + f'unique_path: {unique_path}', + ) with open(unique_path, 'wb') as target: for content in self.get_content(): @@ -469,7 +493,13 @@ def get_content(self): class ArchiveException(ExtractError): - def __init__(self, rc=None, archive_struct=None, archive_func=None, root_ex=None): + def __init__( + self, + rc=None, + archive_struct=None, + archive_func=None, + root_ex=None, + ): self.root_ex = root_ex if root_ex and isinstance(root_ex, ArchiveException): self.rc = root_ex.rc @@ -488,8 +518,9 @@ def __init__(self, rc=None, archive_struct=None, archive_func=None, root_ex=None def __str__(self): if TRACE: - msg = (u'%(msg)r: in function %(func)r with rc=%(rc)r, errno=%(errno)r, ' - 'root_ex=%(root_ex)r') + msg = ( + '%(msg)r: in function %(func)r with rc=%(rc)r, ' + 'errno=%(errno)r, root_ex=%(root_ex)r') return msg % self.__dict__ return self.msg or '' @@ -514,7 +545,10 @@ class ArchiveErrorFailedToWriteEntry(ArchiveException): pass -class ArchiveErrorPasswordProtected(ArchiveException, ExtractErrorPasswordProtected): +class ArchiveErrorPasswordProtected( + ArchiveException, + ExtractErrorPasswordProtected, +): pass @@ -528,7 +562,8 @@ class ArchiveErrorIllegalOperationOnClosedArchive(ArchiveException): def errcheck(rc, archive_func, args, null=False): """ - ctypes error check handler for functions returning int, or null if null is True. + ctypes error check handler for functions returning int, or null if null is + True. """ if null: if rc is None: @@ -575,17 +610,19 @@ def errcheck(rc, archive_func, args, null=False): ##################################### # libarchive C functions declarations ##################################### -# NOTE: these declaration come with verbose doc to help with debugging and tracing -# lower level errors and issues. Some comments and the function signatures are -# copied from libarchve. + +# NOTE: these declaration come with verbose doc to help with debugging and +# tracing lower level errors and issues. Some comments and the function +# signatures are copied from libarchve. # -# NOTE: String data in librachive can be set or accessed as wide character strings or -# narrow char strings. The functions that use wide character strings are suffixed -# with _w. These are different representations of the same data: For example, if you -# store a narrow string and read the corresponding wide string, the object will -# transparently convert formats using the current locale. Similarly, if you store a -# wide string and then store a narrow string for the same data, the previously-set -# wide string will be discarded in favor of the new data. +# NOTE: String data in libarchive can be set or accessed as wide character +# strings or narrow char strings. The functions that use wide character strings +# are suffixed with _w. These are different representations of the same data: +# For example, if you store a narrow string and read the corresponding wide +# string, the object will transparently convert formats using the current +# locale. Similarly, if you store a wide string and then store a narrow string +# for the same data, the previously-set wide string will be discarded in favor +# of the new data. """ To read an archive, you must first obtain an initialized struct archive object diff --git a/src/extractcode/patch.py b/src/extractcode/patch.py index 8a197ca..47fbf28 100644 --- a/src/extractcode/patch.py +++ b/src/extractcode/patch.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import posixpath diff --git a/src/extractcode/sevenzip.py b/src/extractcode/sevenzip.py index 8639bdc..be58754 100644 --- a/src/extractcode/sevenzip.py +++ b/src/extractcode/sevenzip.py @@ -1,29 +1,21 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -from collections import defaultdict import io import logging import os import pprint import re +import warnings + +from collections import defaultdict +from shlex import quote as shlex_quote from shutil import which import attr @@ -40,8 +32,6 @@ from extractcode import ExtractErrorFailedToExtract from extractcode import ExtractWarningIncorrectEntry -from shlex import quote as shlex_quote - """ Low level support for p/7zip-based archive extraction. """ @@ -57,10 +47,11 @@ logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) -# keys for plugin-provided locations -EXTRACTCODE_7ZIP_LIBDIR = 'extractcode.sevenzip.libdir' +# key of a plugin-provided location EXTRACTCODE_7ZIP_EXE = 'extractcode.sevenzip.exe' +EXTRACTCODE_7ZIP_PATH_ENVVAR = 'EXTRACTCODE_7Z_PATH' + sevenzip_errors = [ ('unsupported method', 'Unsupported archive or broken archive'), ('wrong password', 'Password protected archive, unable to extract'), @@ -72,27 +63,51 @@ UNKNOWN_ERROR = 'Unknown extraction error' -def get_bin_locations(): +def get_command_location(_cache=[]): """ - Return a tuple of (lib_dir, cmd_loc) for 7zip loaded from plugin-provided path. + Return the location of a 7zip loaded from either: + - an environment variable ``EXTRACTCODE_7Z_PATH``, + - a plugin-provided path, + - the system PATH. + Raise an Exception if no 7Zip command can be found. """ + if _cache: + return _cache[0] + from plugincode.location_provider import get_location - cmd_loc = get_location(EXTRACTCODE_7ZIP_EXE) - libdir = get_location(EXTRACTCODE_7ZIP_LIBDIR) - if not (cmd_loc and libdir) or not os.path.isfile(cmd_loc) or not os.path.isdir(libdir): - sevenzip = which('7z') - if not sevenzip: - raise ImportError( - 'CRITICAL: 7zip executable is not installed. ' - 'Unable to continue: you need to install a valid extractcode-7z ' - 'plugin with a valid executable available ' - 'or install p7zip in your system, so that a "7z" program is available on your PATH' + # try the environment first + cmd_loc = os.environ.get(EXTRACTCODE_7ZIP_PATH_ENVVAR) + + # try a plugin-provided path second + if not cmd_loc: + cmd_loc = get_location(EXTRACTCODE_7ZIP_EXE) + + # try the PATH + if not cmd_loc: + cmd = '7z.exe' if on_windows else '7z' + cmd_loc = command.find_in_path(cmd) + + if not cmd_loc: + cmd_loc = which(cmd) + + if cmd_loc: + warnings.warn( + 'Using "7z" 7zip command found in the PATH. ' + 'Install instead a extractcode-7z plugin for best support.' ) - logger.warning('Cannot to use plugin for libarchive, defaulting to system binary at ' + sevenzip) - return '', sevenzip - return libdir, cmd_loc + if not cmd_loc or not os.path.isfile(cmd_loc): + raise Exception( + 'CRITICAL: 7zip executable is not installed. ' + 'Unable to continue: you need to install a valid extractcode-7z ' + 'plugin with a valid executable available. ' + f'OR set the {EXTRACTCODE_7ZIP_PATH_ENVVAR} environment variable. ' + 'OR install 7zip as a system package. ' + 'OR ensure 7zip is available in the system PATH.' + ) + _cache.append(cmd_loc) + return cmd_loc def get_7z_errors(stdout, stderr): @@ -133,8 +148,8 @@ def get_7z_errors(stdout, stderr): def get_7z_warnings(stdout): """ - Return a mapping of {path: warning_message} of 7zip warnings extracted from a - `stdout` text. + Return a mapping of {path: warning_message} of 7zip warnings extracted from + a `stdout` text. """ # FIXME: we should use only one pass over stdout for errors and warnings cannot_open = 'can not open output file' @@ -163,6 +178,7 @@ def convert_warnings_to_list(warnings): def list_extracted_7z_files(stdout): """ List all files extracted by 7zip based on the stdout analysis. + Based on 7zip Client7z.cpp: static const char *kExtractingString = "Extracting "; """ @@ -182,54 +198,69 @@ def is_rar(location): return T.filetype_file.lower().startswith('rar archive') -def extract(location, target_dir, arch_type='*', file_by_file=on_mac, skip_symlinks=True): +def extract( + location, + target_dir, + arch_type='*', + file_by_file=on_mac, + skip_symlinks=True, +): """ - Extract all files from a 7zip-supported archive file at location in the - target_dir directory. `skip_symlinks` by default. - Return a list of warning messages. - Raise exception on errors. + Extract all files from a 7zip-supported archive file at ``location`` in the + ``target_dir`` directory. ``skip_symlinks`` by default. + + Return a list of warning messages. Raise exception on errors. - The extraction will either be done all-files-at-once (default on most OSes) - or one-file-at-a-time after collecting a directory listing (for some - problematic OSes such as recent macOS) + ``arch_type`` is the type of 7zip archive passed to the -t 7zip option. Can + be None. - `arch_type` is the type of 7zip archive passed to the -t 7zip option. Can be - None. + Based on ``file_by_file`` the extraction will either be done all-files-at- + once (default on most OSes) or one-file-at-a-time after collecting a + directory listing (for some problematic OSes such as recent macOS) """ assert location abs_location = os.path.abspath(os.path.expanduser(location)) if not os.path.exists(abs_location): raise ExtractErrorFailedToExtract( - 'The system cannot find the path specified: {}'.format(repr(abs_location))) + f'The system cannot find the path specified: {abs_location}') if is_rar(location): raise ExtractErrorFailedToExtract( - 'RAR extraction disactivated: {}'.format(repr(location))) + f'RAR extraction deactivated: {location}') assert target_dir abs_target_dir = os.path.abspath(os.path.expanduser(target_dir)) if not os.path.exists(abs_target_dir): raise ExtractErrorFailedToExtract( - 'The system cannot find the target path specified: {}'.format(repr(target_dir))) + f'The system cannot find the target path specified: {target_dir}') + + if file_by_file: + extractor = extract_file_by_file + else: + extractor = extract_all_files_at_once - extractor = extract_file_by_file if file_by_file else extract_all_files_at_once return extractor( location=abs_location, target_dir=abs_target_dir, arch_type=arch_type, - skip_symlinks=skip_symlinks) + skip_symlinks=skip_symlinks, + ) -def extract_all_files_at_once(location, target_dir, arch_type='*', skip_symlinks=True): +def extract_all_files_at_once( + location, + target_dir, + arch_type='*', + skip_symlinks=True, +): """ - Extract all files from a 7zip-supported archive file at `location` in the - `target_dir` directory. + Extract all files from a 7zip-supported archive file at ``location`` in the + ``target_dir`` directory. - Return a list of warning messages. - Raise exception on errors. + Return a list of warning messages. Raise exception on errors. - `arch_type` is the type of 7zip archive passed to the -t 7zip option. Can be - None. + ``arch_type`` is the type of 7zip archive passed to the -t 7zip option. Can + be None. """ abs_location = os.path.abspath(os.path.expanduser(location)) abs_target_dir = os.path.abspath(os.path.expanduser(target_dir)) @@ -239,7 +270,7 @@ def extract_all_files_at_once(location, target_dir, arch_type='*', skip_symlinks ex_args = build_7z_extract_command( location=location, target_dir=target_dir, arch_type=arch_type) - rc, stdout, stderr = command.execute2(**ex_args) + rc, stdout, stderr = command.execute(**ex_args) if rc != 0: if TRACE: @@ -254,13 +285,18 @@ def extract_all_files_at_once(location, target_dir, arch_type='*', skip_symlinks return convert_warnings_to_list(get_7z_warnings(stdout)) -def build_7z_extract_command(location, target_dir, single_entry=None, arch_type='*'): +def build_7z_extract_command( + location, + target_dir, + single_entry=None, + arch_type='*', +): """ Return a mapping of 7z command line aguments to extract the archive at - `location` to `target_dir`. + ``location`` to ``target_dir``. - If `single_entry` contains an Entry, provide the command to extract only - that single entry "path" in the current directory without any leading path. + If ``single_entry`` contains an Entry, return the command to extract only + this single entry "path" in the current directory without any leading path. """ # 7z arguments @@ -281,7 +317,8 @@ def build_7z_extract_command(location, target_dir, single_entry=None, arch_type= # pass an empty password so that extraction with passwords WILL fail password = '-p' - # renaming may not behave the same way on all OSes in particular Mac and Windows + # renaming may not behave the same way on all OSes in particular Mac and + # Windows auto_rename_dupe_names = '-aou' # Ensure that we treat the FS as case insensitive if that's what it is @@ -300,13 +337,14 @@ def build_7z_extract_command(location, target_dir, single_entry=None, arch_type= # output_as_utf = '-sccUTF-8' # working_tmp_dir = '-w' - # NB: we force running in the GMT timezone, because 7z is unable to set - # the TZ correctly when the archive does not contain TZ info. This does - # not work on Windows, because 7z is not using the TZ env var there. + # NB: we force running in the GMT timezone, because 7z is unable to set the + # TZ correctly when the archive does not contain TZ info. This does not work + # on Windows, because 7z is not using the TZ env var there. timezone = dict(os.environ) timezone.update({u'TZ': u'GMT'}) timezone = command.get_env(timezone) - # Note: 7z does extract in the current directory so we cwd to the target dir first + # Note: 7z does extract in the current directory so we cwd to the target dir + # first args = [ extract, yes_to_all, @@ -321,12 +359,11 @@ def build_7z_extract_command(location, target_dir, single_entry=None, arch_type= if single_entry: args += [shlex_quote(single_entry.path)] - lib_dir, cmd_loc = get_bin_locations() + cmd_loc = get_command_location() ex_args = dict( cmd_loc=cmd_loc, args=args, - lib_dir=lib_dir, cwd=target_dir, env=timezone, ) @@ -338,15 +375,20 @@ def build_7z_extract_command(location, target_dir, single_entry=None, arch_type= return ex_args -def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True): +def extract_file_by_file( + location, + target_dir, + arch_type='*', + skip_symlinks=True, +): """ Extract all files using a one-by-one process from a 7zip-supported archive - file at location in the `target_dir` directory. + file at ``location`` in the ``target_dir`` directory. Return a list of warning messages if any or an empty list. Raise exception on errors. - `arch_type` is the type of 7zip archive passed to the -t 7zip option. + ``arch_type`` is the type of 7zip archive passed to the -t 7zip option. Can be None. """ abs_location = os.path.abspath(os.path.expanduser(location)) @@ -393,7 +435,7 @@ def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True single_entry=entry, arch_type=arch_type, ) - rc, stdout, stderr = command.execute2(**ex_args) + rc, stdout, stderr = command.execute(**ex_args) error = get_7z_errors(stdout, stderr) if error or rc != 0: @@ -414,7 +456,8 @@ def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True else: warnings[entry.path] = wmsg - # finally move that extracted file to its target location, possibly renamed + # finally move that extracted file to its target location, possibly + # renamed source_file_name = fileutils.file_name(entry.path) source_file_loc = os.path.join(tmp_extract_dir, source_file_name) if not os.path.exists(source_file_loc): @@ -449,10 +492,10 @@ def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True def list_entries(location, arch_type='*'): """ - Return a tuple of (iterator of Entry, error_messages). The generator contains - each entry found in a 7zip-supported archive file at `location`. Use the - provided 7zip `arch_type` CLI archive type code (e.g. with the "-t* 7z" cli - type option) (can be None). + Return a tuple of (iterator of Entry, error_messages). The generator + contains each entry found in a 7zip-supported archive file at `location`. + Use the provided 7zip `arch_type` CLI archive type code (e.g. with the "-t* + 7z" cli type option) (can be None). """ assert location abs_location = os.path.abspath(os.path.expanduser(location)) @@ -477,9 +520,9 @@ def list_entries(location, arch_type='*'): if on_windows: output_as_utf = '-sccUTF-8' - # NB: we force running in the GMT timezone, because 7z is unable to set - # the TZ correctly when the archive does not contain TZ info. This does - # not work on Windows, because 7z is not using the TZ env var there. + # NB: we force running in the GMT timezone, because 7z is unable to set the + # TZ correctly when the archive does not contain TZ info. This does not work + # on Windows, because 7z is not using the TZ env var there. timezone = dict(os.environ) timezone.update({u'TZ': u'GMT'}) timezone = command.get_env(timezone) @@ -494,12 +537,11 @@ def list_entries(location, arch_type='*'): abs_location, ] - lib_dir, cmd_loc = get_bin_locations() + cmd_loc = get_command_location() - rc, stdout, stderr = command.execute2( + rc, stdout, stderr = command.execute( cmd_loc=cmd_loc, args=args, - lib_dir=lib_dir, env=timezone, to_files=True) @@ -513,48 +555,46 @@ def list_entries(location, arch_type='*'): if rc != 0: error_messages = get_7z_errors(stdout, stderr) or UNKNOWN_ERROR - # the listing was produced as UTF on windows to avoid damaging binary - # paths in console outputs - utf = bool(output_as_utf) + return parse_7z_listing(stdout), error_messages - return parse_7z_listing(stdout, utf), error_messages - -def parse_7z_listing(location, utf=False): +def parse_7z_listing(location): """ Return a list Entry objects from parsing a long format 7zip listing from a file at `location`. - If `utf` is True or if on Python 3, the console output will treated as - utf-8-encoded text. Otherwise it is treated as bytes. - The 7zip -slt format looks like this: 1. a header with: + ----------------- - copyright and version details - '--' line - - archive header info, varying based on the archive types and subtype - - lines of key=value pairs - - ERRORS: followed by one or more message lines - - WARNINGS: followed by one or more message lines + - archive header info, varying based on the archive types and subtype + - lines of key=value pairs + - ERRORS: followed by one or more message lines + - WARNINGS: followed by one or more message lines - blank line 2. blocks of path aka. entry data, one for each path with: + ---------------------------------------------------------- - '----------' line once as the indicator of path blocks starting - for each archive member: - lines of either - - key = value pairs, with a possible twist that the Path may - contain a line return since a filename may. The first key is the Path. - - Errors: followed by one or more message lines - - Warnings: followed by one or more message lines - - Open Warning: : followed by one or more message lines + - key = value pairs, with a possible twist that the Path may + contain a line return since a filename may. The first key is the + Path. + - Errors: followed by one or more message lines + - Warnings: followed by one or more message lines + - Open Warning: : followed by one or more message lines - blank line 3. a footer + ----------- + - blank line - - footer sometimes with lines with summary stats - such as Warnings: 1 Errors: 1 + - footer sometimes with lines with summary stats such as: + Warnings: 1 Errors: 1 - a line with two or more dashes or an empty line We ignore the header and footer in a listing. @@ -588,7 +628,10 @@ def parse_7z_listing(location, utf=False): # then we have a global footer two_empty_lines = '\n\n' path_key = 'Path' - path_blocks = [pb for pb in paths.split(two_empty_lines) if pb and path_key in pb] + path_blocks = [ + pb for pb in paths.split(two_empty_lines) + if pb and path_key in pb + ] key_value_sep = '=' @@ -596,12 +639,19 @@ def parse_7z_listing(location, utf=False): for path_block in path_blocks: # we ignore empty lines as well as lines that do not contain a key - lines = [line.strip() for line in path_block.splitlines(False) if line.strip()] + lines = [ + line.strip() for line in path_block.splitlines(False) + if line.strip() + ] if not lines: continue # we have a weird case of path with line returns in the file name # we concatenate these in the first Path line - while len(lines) > 1 and lines[0].startswith(path_key) and key_value_sep not in lines[1]: + while ( + len(lines) > 1 + and lines[0].startswith(path_key) + and key_value_sep not in lines[1] + ): first_line = lines[0] second_line = lines.pop(1) first_line = '\n'.join([first_line, second_line]) @@ -610,7 +660,10 @@ def parse_7z_listing(location, utf=False): dangling_lines = [line for line in lines if key_value_sep not in line] entry_errors = [] if dangling_lines: - emsg = 'Invalid 7z listing path block missing "=" as key/value separator: {}'.format(repr(path_block)) + emsg = ( + 'Invalid 7z listing path block missing "=" as key/value ' + 'separator: {}'.format(repr(path_block)) + ) entry_errors.append(emsg) entry_attributes = {} @@ -621,7 +674,8 @@ def parse_7z_listing(location, utf=False): v = v.strip() entry_attributes[k] = v - entries.append(Entry.from_dict(infos=entry_attributes, errors=entry_errors)) + ntry = Entry.from_dict(infos=entry_attributes, errors=entry_errors) + entries.append(ntry) if TRACE_ENTRIES: logger.debug('parse_7z_listing: entries# {}\n'.format(len(entries))) @@ -670,7 +724,8 @@ def is_empty(self): @classmethod def from_dict(cls, infos, errors=None): """ - Return an Entry built from a 7zip path listing data in the `infos` mapping. + Return an Entry built from a 7zip path listing data in the `infos` + mapping. """ is_symlink = False is_hardlink = False diff --git a/src/extractcode/uncompress.py b/src/extractcode/uncompress.py index f584e84..143e336 100644 --- a/src/extractcode/uncompress.py +++ b/src/extractcode/uncompress.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import bz2 @@ -42,6 +31,7 @@ def uncompress(location, target_dir, decompressor, suffix=EXTRACT_SUFFIX): Uncompress a compressed file at location in the target_dir using the `decompressor` object. The uncompressed file is named after the original archive with a `suffix` added. + Return a list of warning messages. Raise Exceptions on errors. """ # FIXME: do not create a sub-directory and instead strip the "compression" @@ -122,5 +112,11 @@ def get_compressed_file_content(location, decompressor): return content, warnings -get_gz_compressed_file_content = partial(get_compressed_file_content, decompressor=gzip.GzipFile) -get_bz2_compressed_file_content = partial(get_compressed_file_content, decompressor=bz2.BZ2File) +get_gz_compressed_file_content = partial( + get_compressed_file_content, + decompressor=gzip.GzipFile, +) +get_bz2_compressed_file_content = partial( + get_compressed_file_content, + decompressor=bz2.BZ2File, +) diff --git a/src/extractcode/vmimage.py b/src/extractcode/vmimage.py new file mode 100644 index 0000000..2ce0ea7 --- /dev/null +++ b/src/extractcode/vmimage.py @@ -0,0 +1,379 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import os +import pathlib +import shutil +import warnings + +import attr + +from commoncode import fileutils +from commoncode.system import on_linux +from commoncode.text import as_unicode + +from extractcode import ExtractErrorFailedToExtract + +""" +Support to extract Virtual Machine image formats and the filesystem(s) they +contain. This is based on libguestfs-tools and is tested only on Linux. Works +only if libguestfs tool guestfish: +- has its path in the "EXTRACTCODE_GUESTFISH_PATH" environment variable. +- or is in the system PATH. + +See https://libguestfs.org/ +""" + +logger = logging.getLogger(__name__) + +TRACE = False + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + +GUESTFISH_NOT_FOUND = ( + 'WARNING: guestfish executable is not installed. ' + 'Unable to extract virtual machine image: you need to install the ' + 'guestfish tool from libguestfs and extra FS drivers as needed. ' + 'See the ExtractCode README.rst at ' + 'https://github.com/nexB/extractcode/blob/main/README.rst ' + 'and https://libguestfs.org/ for details.' +) + +GUESTFISH_KERNEL_NOT_READABLE = ( +'''libguestfs requires the kernel executable to be readable. +This is the case by default on most Linux distributions except on Ubuntu. +Please follow the ExtractCode installation instructions in the README.rst at: +https://github.com/nexB/extractcode/blob/main/README.rst ' +''') + +EXTRACTCODE_GUESTFISH_PATH_ENVVAR = 'EXTRACTCODE_GUESTFISH_PATH' + + +def get_command(env_var=EXTRACTCODE_GUESTFISH_PATH_ENVVAR, command='guestfish'): + """ + Return the location to the guestfish command or None. + """ + cmd_loc = os.environ.get(env_var, None) + if cmd_loc and os.path.exists(cmd_loc): + return cmd_loc + + cmd_loc = shutil.which(command) or None + if not cmd_loc: + warnings.warn(GUESTFISH_NOT_FOUND) + + return cmd_loc + + +def check_linux_kernel_is_readable(): + """ + Return True if the kernel executable file can be read. This is required by + guestfish and libguestfs and this is an oddity mostly on Ubuntu. + + See: + - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725 + - https://bugzilla.redhat.com/show_bug.cgi?id=1670790 + - https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662 + """ + + if on_linux: + kernels = list(pathlib.Path('/boot').glob('vmlinuz-*')) + if not kernels: + raise ExtractErrorFailedToExtract(GUESTFISH_KERNEL_NOT_READABLE) + for kern in kernels: + if not os.access(kern, os.R_OK): + raise ExtractErrorFailedToExtract( + f'Unable to read kernel at: {kern}.\n' + f'{GUESTFISH_KERNEL_NOT_READABLE}' + ) + + +@attr.s +class VmImage: + location = attr.ib() + image_format = attr.ib() + guestfish_command = attr.ib() + + @classmethod + def from_file(cls, location): + """ + Build a new VMImage from the file at location. + Raise excptions on errors. + """ + if not on_linux: + raise ExtractErrorFailedToExtract( + 'VM Image extraction only supported on Linux.') + + check_linux_kernel_is_readable() + + assert location + abs_location = os.path.abspath(os.path.expanduser(location)) + + if not os.path.exists(abs_location): + raise ExtractErrorFailedToExtract( + f'The system cannot find the path specified: {abs_location}') + + supported_gfs_formats_by_extension = { + '.qcow2': 'qcow2', + '.qcow2c': 'qcow2', + '.qcow': 'qcow2', + '.img': 'qcow2', + '.vmdk': 'vmdk', + '.vdi': 'vdi', + } + + extension = fileutils.file_extension(location) + image_format = supported_gfs_formats_by_extension.get(extension) + + if not image_format: + raise ExtractErrorFailedToExtract( + f'Unsupported VM image format: {location}') + + cmd_loc = get_command() + if not cmd_loc: + raise ExtractErrorFailedToExtract(GUESTFISH_NOT_FOUND) + + return cls( + location=location, + image_format=image_format, + guestfish_command=cmd_loc, + ) + + def listfs(self, skip_partitions=('swap',)): + """ + Return a list of (filesystem /partition/ device path, filesystem type) + for each filesystem found in this image. + + Skip the partitions names list ``skip_partitions``. + + We run guestfish for this: + $ guestfish --ro add foo.qcow2 : run : list-filesystems + /partition/sda1: ext4 + """ + args = [ + '--ro', + f'--format={self.image_format}', + '--add' , self.location, + 'run', + ':', 'list-filesystems', + ] + stdout = self.run_guestfish(args) + + filesystems = [] + entries = stdout.strip().splitlines(False) + for entry in entries: + entry = entry.strip() + if not entry: + continue + if ':' in entry: + partition, _, fstype = entry.partition(':') + fstype = fstype.strip() + else: + partition = entry + fstype = None + + if any(s in partition for s in skip_partitions): + continue + + filesystems.append((partition, fstype,)) + + return filesystems + + def extract_image(self, target_tarball): + """ + Extract all files from this VM image to the ``target_tarball`` file as a + gzipped-compressed tarball (.tar.gz). Raise Exception on errors. + """ + args = [ + '--ro', + '--inspector', + f'--format={self.image_format}', + '--add', self.location, + 'tar-out', '/', target_tarball, 'compress:gzip', + ] + + self.run_guestfish(args) + + def extract_partition(self, partition, target_tarball): + """ + Extract all files from a single ``partition`` of this VM image to the + ``target_tarball`` file as a gzipped-compressed tarball (.tar.gz). Raise + exception on errors. + """ + # TODO: there could be devices/partitions we do not want to extract? + # guestfish --ro add foo.qcow2 : run : mount /dev/sda1 / : tar-out /etc foo.tgz compress:gzip + + args = [ + '--ro', + f'--format={self.image_format}', + '--add', self.location, + 'run', + ':', 'mount', partition, '/', + ':', 'tar-out', '/', target_tarball, 'compress:gzip', + ] + self.run_guestfish(args) + + def run_guestfish(self, args, timeout=None): + """ + Run guestfish with ``args`` arguments and am optional ``timeout`` in + seconds. Return stdout as a unicode string. Raise Exception on error. + """ + import subprocess + full_args = [self.guestfish_command] + args + try: + stdout = subprocess.check_output( + full_args, + timeout=timeout, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as cpe: + args = ' '.join([self.guestfish_command] + args) + output = as_unicode(cpe.output) + error = ( + f'Failed to run guestfish to extract VM image: {args}\n' + f'output: {output}' + ) + raise ExtractErrorFailedToExtract(error) + + return as_unicode(stdout) + + +def extract(location, target_dir, as_tarballs=False, skip_symlinks=True): + """ + Extract all files from a guestfish-supported VM image archive file at + ``location`` in the ``target_dir`` directory. + Return a list of warning messages if any or an empty list. + Raise Exception on errors. + + Optionally only extract the intermediate tarballs if ``as_tarball`` is True. + Otherwise, extract to intermediate tarballs and then extract each tarballs + to the final directory. + + Optionally skip extracting symlinks with ``skip_symlinks``. + + This works only on Linux. + """ + assert target_dir + abs_target_dir = os.path.abspath(os.path.expanduser(target_dir)) + if not os.path.exists(abs_target_dir) or not os.path.isdir(abs_target_dir): + raise ExtractErrorFailedToExtract( + f'The system cannot find the target directory path ' + f'specified: {target_dir}') + + vmimage = VmImage.from_file(location) + + warnings = [] + + filename = fileutils.file_name(vmimage.location) + + # try a plain extract first + try: + + if not as_tarballs: + intermediate_dir = fileutils.get_temp_dir( + prefix='extractcode-vmimage') + tdir = intermediate_dir + else: + tdir = target_dir + + target_tarball = os.path.join(tdir, f'{filename}.tar.gz') + vmimage.extract_image(target_tarball=target_tarball) + + if not as_tarballs: + # extract the temp tarball to the final location + warns = extract_image_tarball( + tarball=target_tarball, + target_dir=target_dir, + skip_symlinks=skip_symlinks, + ) + warnings.extend(warns) + + except ExtractErrorFailedToExtract as e: + print('Cannot extract VM Image filesystems as a single file tree.') + + warnings.append( + f'Cannot extract VM Image filesystems as a single file tree:\n{e}') + # fall back to file system extraction, one partition at a time + partitions = vmimage.listfs() + if not partitions: + raise + + if len(partitions) == 1: + # we can safely extract this to a root / dir as we have only one partition + partition, _parttype = partitions[0] + if not as_tarballs: + tdir = fileutils.get_temp_dir(prefix='extractcode-vmimage') + else: + tdir = target_dir + + target_tarball = os.path.join(tdir, f'{filename}.tar.gz') + vmimage.extract_partition( + partition=partition, + target_tarball=target_tarball, + ) + + if not as_tarballs: + # extract the temp tarball to the final location + warns = extract_image_tarball( + tarball=target_tarball, + target_dir=target_dir, + skip_symlinks=skip_symlinks, + ) + warnings.extend(warns) + else: + # with multiple partitions, we extract each partition to a unique + # base name based after the partition device name + + for partition, _parttype in partitions: + base_name = partition.replace('/', '-') + + if not as_tarballs: + tdir = fileutils.get_temp_dir(prefix='extractcode-vmimage') + else: + tdir = target_dir + + partition_tarball = os.path.join( + tdir, + f'{filename}-{base_name}.tar.gz', + ) + vmimage.extract_partition( + partition=partition, + target_tarball=partition_tarball, + ) + + if not as_tarballs: + # extract the temp tarball to the final location + # which is a new subdirectory + partition_target_dir = os.path.join(target_dir, base_name) + fileutils.create_dir(partition_target_dir) + warns = extract_image_tarball( + tarball=target_tarball, + target_dir=partition_target_dir, + skip_symlinks=skip_symlinks, + ) + warnings.extend(warns) + + return warnings + + +def extract_image_tarball(tarball, target_dir, skip_symlinks=True): + """ + Extract an intermediate image ``tarball`` to its final ``target_dir`` + directory. Optionally skip extracting symlinks with ``skip_symlinks``. + Return a list of warning messages. Raise Exception on errors. + """ + from extractcode.libarchive2 import extract + return extract( + location=tarball, + target_dir=target_dir, + skip_symlinks=skip_symlinks, + ) diff --git a/tests/README.rst b/tests/README.rst deleted file mode 100644 index d94783e..0000000 --- a/tests/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python test modules in this directory. - diff --git a/tests/data/vmimage/CHANGELOG.rst b/tests/data/vmimage/CHANGELOG.rst new file mode 100644 index 0000000..39a0096 --- /dev/null +++ b/tests/data/vmimage/CHANGELOG.rst @@ -0,0 +1,36 @@ +Release notes +============= + +vNext +----- + + +Version 21.1.21 +--------------- + +- Bump dependencies and use latest typecode and binaries. This is to fix + installation problems on multiple OSes. + + +Version 21.1.21 +--------------- + +- Add new [full] extra requires that install all the dependencies +- Fix bug related to commoncode libraries loading +- Improve the extra requirements +- Set minimum version for dependencies +- Improve documentation + + +Version 21.1.15 +--------------- + +- Drop support for Python 2 +- Use the latest CommonCode and TypeCode libraries +- Add azure-pipelines CI support + + +Version 20.10 +------------- + +- Initial release. diff --git a/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz b/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz new file mode 100644 index 0000000..bc946cb Binary files /dev/null and b/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz differ diff --git a/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz.ABOUT b/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz.ABOUT new file mode 100644 index 0000000..3f9d3c1 --- /dev/null +++ b/tests/data/vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz.ABOUT @@ -0,0 +1,2 @@ +date: 2021-04-08 +download_url: https://github.com/jc-lab/pve-mirror_qemu/blob/51db2d7cf26d05a961ec0ee0eb773594b32cc4a1/tests/data/uefi-boot-images/bios-tables-test.x86_64.iso.qcow2?raw=true diff --git a/tests/data/vmimage/foobar.qcow2.tar.gz b/tests/data/vmimage/foobar.qcow2.tar.gz new file mode 100644 index 0000000..8bf6215 Binary files /dev/null and b/tests/data/vmimage/foobar.qcow2.tar.gz differ diff --git a/tests/extractcode_assert_utils.py b/tests/extractcode_assert_utils.py index 537cf1d..d930d2e 100644 --- a/tests/extractcode_assert_utils.py +++ b/tests/extractcode_assert_utils.py @@ -1,22 +1,10 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import json @@ -116,11 +104,11 @@ def check_no_error(result): def is_posixpath(location): """ - Return True if the `location` path is likely a POSIX-like path using POSIX path - separators (slash or "/")or has no path separator. + Return True if the `location` path is likely a POSIX-like path using POSIX + path separators (slash or "/")or has no path separator. - Return False if the `location` path is likely a Windows-like path using backslash - as path separators (e.g. "\"). + Return False if the `location` path is likely a Windows-like path using + backslash as path separators (e.g. "\"). """ has_slashes = '/' in location has_backslashes = '\\' in location @@ -140,10 +128,11 @@ def is_posixpath(location): def to_posix(path): """ - Return a path using the posix path separator given a path that may contain posix - or windows separators, converting \\ to /. NB: this path will still be valid in - the windows explorer (except as a UNC or share name). It will be a valid path - everywhere in Python. It will not be valid for windows command line operations. + Return a path using the posix path separator given a path that may contain + posix or windows separators, converting \\ to /. NB: this path will still be + valid in the windows explorer (except as a UNC or share name). It will be a + valid path everywhere in Python. It will not be valid for windows command + line operations. """ is_unicode = isinstance(path, str) ntpath_sep = is_unicode and u'\\' or '\\' @@ -193,7 +182,14 @@ def assertRaisesInstance(self, excInstance, callableObj, *args, **kwargs): excName = str(excClass) raise self.failureException('%s not raised' % excName) - def check_extract(self, test_function, test_file, expected, expected_warnings=None, check_all=False): + def check_extract( + self, + test_function, + test_file, + expected, + expected_warnings=None, + check_all=False, + ): """ Run the extraction `test_function` on `test_file` checking that a map of expected paths --> size exist in the extracted target directory. @@ -209,8 +205,14 @@ def check_extract(self, test_function, test_file, expected, expected_warnings=No if check_all: len_test_dir = len(test_dir) - extracted = {path[len_test_dir:]: filetype.get_size(path) for path in fileutils.resource_iter(test_dir, with_dirs=False)} - expected = {os.path.join(test_dir, exp_path): exp_size for exp_path, exp_size in expected.items()} + extracted = { + path[len_test_dir:]: filetype.get_size(path) + for path in fileutils.resource_iter(test_dir, with_dirs=False) + } + expected = { + os.path.join(test_dir, exp_path): exp_size + for exp_path, exp_size in expected.items() + } assert sorted(expected.items()) == sorted(extracted.items()) else: for exp_path, exp_size in expected.items(): @@ -248,9 +250,10 @@ def assertExceptionContains(self, text, callableObj, *args, **kwargs): except Exception as e: if text not in str(e): raise self.failureException( - 'Exception %(e)r raised, ' - 'it should contain the text %(text)r ' - 'and does not' % locals()) + 'Exception %(e)r raised, ' + 'it should contain the text %(text)r ' + 'and does not' % locals() + ) else: raise self.failureException( 'Exception containing %(text)r not raised' % locals()) diff --git a/tests/test_archive.py b/tests/test_archive.py index 71b1da2..3d2eaca 100644 --- a/tests/test_archive.py +++ b/tests/test_archive.py @@ -1,25 +1,15 @@ # -*- coding: utf-8 -*- # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import os +from pathlib import Path import pytest @@ -59,6 +49,7 @@ project_root = os.path.dirname(os.path.dirname(__file__)) + class TestGetExtractorTest(BaseArchiveTestCase): def test_get_extractors_1(self): @@ -215,6 +206,18 @@ def test_get_extractor_for_compressed_svgz_docs(self): expected = [] self.check_get_extractors(test_file, expected, kinds=extractcode.default_kinds) + def test_get_extractor_qcow2(self): + test_file = self.extract_test_tar('vmimage/foobar.qcow2.tar.gz') + test_file = str(Path(test_file) / 'foobar.qcow2') + + expected = [] + self.check_get_extractors(test_file, expected, kinds=extractcode.default_kinds) + + expected = [archive.extract_vm_image] + self.check_get_extractors(test_file, expected, kinds=()) + self.check_get_extractors(test_file, expected, kinds=(extractcode.file_system,)) + self.check_get_extractors(test_file, expected, kinds=extractcode.all_kinds) + def test_get_extractor_for_dia(self): test_file = self.get_test_loc('archive/dia/dia.dia', copy=True) @@ -294,7 +297,7 @@ def test_7zip_extract_can_extract_to_relative_paths(self): from extractcode.sevenzip import extract test_file = self.get_test_loc('archive/relative_path/basic.zip', copy=True) - + project_tmp = join(project_root, 'tmp') fileutils.create_dir(project_tmp) project_root_abs = abspath(project_root) @@ -2209,12 +2212,16 @@ def test_uncompress_lzip_basic(self): class ExtractArchiveWithIllegalFilenamesTestCase(BaseArchiveTestCase): def check_extract_weird_names( - self, test_function, test_file, expected_suffix, - expected_warnings=None, - expected_exception=None, - check_warnings=True, check_only_warnings=False, - regen=False, - ): + self, + test_function, + test_file, + expected_suffix, + expected_warnings=None, + expected_exception=None, + check_warnings=True, + check_only_warnings=False, + regen=False, + ): """ Run the extraction `test_function` on `test_file` checking that the paths listed in the `test_file.excepted` file exist in the extracted target diff --git a/tests/test_extract.py b/tests/test_extract.py index 58842cc..facba67 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,22 +1,11 @@ # -*- coding: utf-8 -*- # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import io @@ -29,7 +18,6 @@ from commoncode.fileutils import as_posixpath from commoncode.system import on_linux from commoncode.system import on_windows -from commoncode.testcase import FileBasedTesting import extractcode from extractcode import extract diff --git a/tests/test_extractcode.py b/tests/test_extractcode.py index 0a7dafb..cf06d44 100644 --- a/tests/test_extractcode.py +++ b/tests/test_extractcode.py @@ -1,22 +1,10 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # from os.path import dirname diff --git a/tests/test_extractcode_cli.py b/tests/test_extractcode_cli.py index a793959..6ef6b74 100644 --- a/tests/test_extractcode_cli.py +++ b/tests/test_extractcode_cli.py @@ -1,22 +1,10 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import os @@ -44,6 +32,8 @@ def run_extract(options, expected_rc=None, cwd=None): Run extractcode as a plain subprocess. Return rc, stdout, stderr. """ bin_dir = 'Scripts' if on_windows else 'bin' + # note: this assumes that we are using a standard directory layout as set + # with the configure script cmd_loc = os.path.join(project_root, 'tmp', bin_dir, 'extractcode') assert os.path.exists(cmd_loc + ('.exe' if on_windows else '')) args = [cmd_loc] + options @@ -82,13 +72,17 @@ def test_extractcode_command_does_extract_verbose(): result = run_extract(['--verbose', test_dir], expected_rc=1) assert os.path.exists(os.path.join(test_dir, 'some.tar.gz-extract')) - assert 'Extracting archives...' in result.stderr - assert 'some.tar.gz' in result.stdout - assert 'broken.tar.gz' in result.stderr - assert 'tarred_gzipped.tgz' in result.stdout - assert 'ERROR extracting' in result.stderr - assert "broken.tar.gz: Unrecognized archive format" in result.stderr - assert 'Extracting done.' in result.stderr + try: + assert 'some.tar.gz' in result.stdout + assert 'tarred_gzipped.tgz' in result.stdout + + assert 'Extracting archives...' in result.stderr + assert 'ERROR extracting' in result.stderr + assert 'broken.tar.gz' in result.stderr + assert "broken.tar.gz: Unrecognized archive format" in result.stderr + assert 'Extracting done.' in result.stderr + except: + assert [result.stderr, result.stdout] == [] def test_extractcode_command_always_shows_something_if_not_using_a_tty_verbose_or_not(): diff --git a/tests/test_libarchive2.py b/tests/test_libarchive2.py index 70ce34b..0ea6cb3 100644 --- a/tests/test_libarchive2.py +++ b/tests/test_libarchive2.py @@ -1,23 +1,12 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# + import os from commoncode import fileutils @@ -47,12 +36,13 @@ def test_libarchive_extract_can_extract_to_relative_paths(self): project_tmp = join(project_root, 'tmp') fileutils.create_dir(project_tmp) project_root_abs = abspath(project_root) - test_src_dir = tempfile.mkdtemp(dir=project_tmp).replace(project_root_abs, '').strip('\\/') - test_tgt_dir = tempfile.mkdtemp(dir=project_tmp).replace(project_root_abs, '').strip('\\/') + test_src_dir = tempfile.mkdtemp( + dir=project_tmp).replace(project_root_abs, '').strip('\\/') + test_tgt_dir = tempfile.mkdtemp( + dir=project_tmp).replace(project_root_abs, '').strip('\\/') shutil.copy(test_file, test_src_dir) test_src_file = join(test_src_dir, 'basic.zip') result = list(extract(test_src_file, test_tgt_dir)) assert [] == result expected = ['c/a/a.txt', 'c/b/a.txt', 'c/c/a.txt'] check_files(test_tgt_dir, expected) - diff --git a/tests/test_patch.py b/tests/test_patch.py index 6ecb3cc..5a70951 100644 --- a/tests/test_patch.py +++ b/tests/test_patch.py @@ -1,22 +1,10 @@ - -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import io diff --git a/tests/test_sevenzip.py b/tests/test_sevenzip.py index ee32439..4a7637e 100644 --- a/tests/test_sevenzip.py +++ b/tests/test_sevenzip.py @@ -1,21 +1,10 @@ # -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # import os @@ -35,7 +24,13 @@ class TestSevenZip(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - def check_results_with_expected_json(self, results, expected_loc, clean_dates=False, regen=False): + def check_results_with_expected_json( + self, + results, + expected_loc, + clean_dates=False, + regen=False, + ): if regen: with open(expected_loc, 'w') as ex: json.dump(results, ex, indent=2, separators=(',', ':')) @@ -181,7 +176,7 @@ class TestSevenParseListing(TestSevenZip): def check_parse_7z_listing(self, test_loc, regen=False): test_loc = self.get_test_loc(test_loc) - results = [e.to_dict(full=True) for e in sevenzip.parse_7z_listing(location=test_loc, utf=True)] + results = [e.to_dict(full=True) for e in sevenzip.parse_7z_listing(location=test_loc)] expected_loc = test_loc + '-expected.json' self.check_results_with_expected_json( results=results, expected_loc=expected_loc, regen=regen) diff --git a/tests/test_vmimage.py b/tests/test_vmimage.py new file mode 100644 index 0000000..6653cdd --- /dev/null +++ b/tests/test_vmimage.py @@ -0,0 +1,47 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/extractcode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from pathlib import Path + +import pytest + +from commoncode.system import on_linux + +from extractcode_assert_utils import BaseArchiveTestCase +from extractcode_assert_utils import check_files + +from extractcode import vmimage + + +@pytest.mark.skipif(not on_linux, reason='Only linux supports image extraction') +class TestExtractVmImage(BaseArchiveTestCase): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_can_listfs_from_qcow2_image(self): + test_file = self.extract_test_tar('vmimage/foobar.qcow2.tar.gz') + test_file = str(Path(test_file) / 'foobar.qcow2') + vmi = vmimage.VmImage.from_file(test_file) + assert [('/dev/sda', 'ext2')] == vmi.listfs() + + def test_can_extract_qcow2_vm_image_as_tarball(self): + test_file = self.extract_test_tar('vmimage/foobar.qcow2.tar.gz') + test_file = str(Path(test_file) / 'foobar.qcow2') + target_dir = self.get_temp_dir('vmimage') + vmimage.extract(location=test_file, target_dir=target_dir, as_tarballs=True) + expected = ['foobar.qcow2.tar.gz'] + check_files(target_dir, expected) + + def test_can_extract_qcow2_vm_image_not_as_tarball(self): + test_file = self.extract_test_tar('vmimage/bios-tables-test.x86_64.iso.qcow2.tar.gz') + test_file = str(Path(test_file) / 'bios-tables-test.x86_64.iso.qcow2') + target_dir = self.get_temp_dir('vmimage') + vmimage.extract(location=test_file, target_dir=target_dir, as_tarballs=False) + expected = ['bios_tab.fat', 'boot.cat'] + check_files(target_dir, expected)