From 083bd0483b5869219db51623eed44a9cd711989b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 26 Oct 2020 15:30:56 -0700 Subject: [PATCH 01/34] Add azure pipeline config files and templates * Create configure.bat so we can use our skeleton for Windows projects Signed-off-by: Jono Yang --- .travis.yml | 4 +- azure-pipelines.yml | 45 ++++++++++++ configure.bat | 160 +++++++++++++++++++++++++++++++++++++++++ etc/ci/azure-linux.yml | 37 ++++++++++ etc/ci/azure-mac.yml | 36 ++++++++++ etc/ci/azure-win.yml | 36 ++++++++++ 6 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 azure-pipelines.yml create mode 100644 configure.bat create mode 100644 etc/ci/azure-linux.yml create mode 100644 etc/ci/azure-mac.yml create mode 100644 etc/ci/azure-win.yml diff --git a/.travis.yml b/.travis.yml index bcc3be8..7a342df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # This is a skeleton Travis CI config file that provides a starting point for adding CI # to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. +# will be specific to that language. # # See https://config.travis-ci.com/ for a full list of configuration options. @@ -18,4 +18,4 @@ python: install: ./configure # Scripts to run at script stage -script: bin/pytest +script: tmp/bin/pytest diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..904ac90 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,45 @@ + +################################################################################ +# We use Azure to run the full tests suites on Python 3.6 +# on Windows (32 and 64), macOS and Linux (64 various distro) +################################################################################ + +jobs: + +################################################################################ +# These jobs are using VMs and Azure-provided Python 3.6 +################################################################################ + + - template: etc/ci/azure-linux.yml + parameters: + job_name: vm_ubuntu16_py36 + image_name: ubuntu-16.04 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-mac.yml + parameters: + job_name: macos1015_py36 + image_name: macos-10.15 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_32_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x86 + test_suites: + all: Scripts\py.test -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_64_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x64 + test_suites: + misc: Scripts\py.test -vvs --reruns=3 diff --git a/configure.bat b/configure.bat new file mode 100644 index 0000000..0a2ca00 --- /dev/null +++ b/configure.bat @@ -0,0 +1,160 @@ +@echo OFF +@setlocal +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem ################################ +@rem # A configuration script for Windows +@rem # +@rem # The options and (optional) arguments are: +@rem # --clean : this is exclusive of anything else and cleans the environment +@rem # from built and installed files +@rem # +@rem # --python < path to python.exe> : this must be the first argument and set +@rem # the path to the Python executable to use. If < path to python.exe> is +@rem # set to "path", then the executable will be the python.exe available +@rem # in the PATH. +@rem # +@rem # : this must be the last argument and sets the path to a +@rem # configuration directory to use. +@rem ################################ + +@rem ################################ +@rem # Defaults. Change these variables to customize this script locally +@rem ################################ +@rem # you can define one or more thirdparty dirs, each where the varibale name +@rem # is prefixed with TPP_DIR +set "TPP_DIR=thirdparty" + +@rem # default configurations for dev +set "CONF_DEFAULT=etc/conf/dev" + +@rem # default thirdparty dist for dev +if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( + set "TPP_DIR_DEV=thirdparty/dev" +) + +@rem # default supported version for Python 3 +set SUPPORTED_PYTHON3=3.6 + +@rem ################################# + +@rem python --version +@rem python -c "import sys;print(sys.executable)" + + +@rem Current directory where this .bat files lives +set CFG_ROOT_DIR=%~dp0 + +@rem path where a configured Python should live in the current virtualenv if installed +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe + +set PYTHON_EXECUTABLE= + +@rem parse command line options and arguments +:collectopts +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b +if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts + +@rem We are not cleaning: Either we have a provided configure config path or we use a default. +if ""%1""=="""" ( + set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% +) else ( + set CFG_CMD_LINE_ARGS=%1 +) + +@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run +if exist ""%CONFIGURED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% + goto run +) + +@rem If we have a command arg for Python use this as-is +if ""%PROVIDED_PYTHON%""==""path"" ( + @rem use a bare python available in the PATH + set PYTHON_EXECUTABLE=python + goto run +) +if exist ""%PROVIDED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% + goto run +) + + +@rem otherwise we search for a suitable Python interpreter +:find_python + +@rem First check the existence of the "py" launcher (available in Python 3) +@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 +@rem if not, check if we have an old py 2.7 +@rem exist if all fails + +where py >nul 2>nul +if %ERRORLEVEL% == 0 ( + @rem we have a py launcher, check for the availability of our required Python 3 version + py -3.6 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -3.6 + ) else ( + @rem we have no required python 3, let's try python 2: + py -2 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -2 + ) else ( + @rem we have py and no python 3 and 2, exit + echo * Unable to find an installation of Python. + exit /b 1 + ) + ) +) else ( + @rem we have no py launcher, check for a default Python 2 installation + if not exist ""%DEFAULT_PYTHON2%"" ( + echo * Unable to find an installation of Python. + exit /b 1 + ) else ( + set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + ) +) + +:run + +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +call mkdir "%CFG_ROOT_DIR%tmp" +call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call "%CFG_ROOT_DIR%tmp\Scripts\activate" +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel + + +@rem Return a proper return code on failure +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) +endlocal +goto activate + + +:cli_help +echo A configuration script for Windows +echo usage: configure [options] [path/to/config/directory] +echo. +echo The options and arguments are: +echo [path/to/config/directory] : this optionally sets the path to a +echo configuration directory to use. Defaults to etc/conf/dev if not set +echo. +echo --clean : this is exclusive of anything else and cleans the environment +echo from built and installed files +echo. +echo --python path/to/python.exe : this is set to the path of an alternative +echo Python executable to use. If path/to/python.exe is set to "path", +echo then the executable will be the python.exe available in the PATH. +echo. + + +:activate +@rem Activate the virtualenv +if exist "%CFG_ROOT_DIR%Scripts\activate" ( + "%CFG_ROOT_DIR%Scripts\activate" +) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml new file mode 100644 index 0000000..2e12e5b --- /dev/null +++ b/etc/ci/azure-linux.yml @@ -0,0 +1,37 @@ +parameters: + job_name: '' + image_name: 'ubuntu-16.04' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-mac.yml new file mode 100644 index 0000000..752ae2e --- /dev/null +++ b/etc/ci/azure-mac.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml new file mode 100644 index 0000000..6220857 --- /dev/null +++ b/etc/ci/azure-win.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x86 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: configure --python path + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' From 847564ead159cfcebe0f04066daa6f23e1a5a123 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:40:38 -0700 Subject: [PATCH 02/34] Fix path to Scripts directory * Remove unused variables and options Signed-off-by: Jono Yang --- configure.bat | 48 ++++-------------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/configure.bat b/configure.bat index 0a2ca00..cbb4244 100644 --- a/configure.bat +++ b/configure.bat @@ -13,41 +13,12 @@ @rem # the path to the Python executable to use. If < path to python.exe> is @rem # set to "path", then the executable will be the python.exe available @rem # in the PATH. -@rem # -@rem # : this must be the last argument and sets the path to a -@rem # configuration directory to use. -@rem ################################ - @rem ################################ -@rem # Defaults. Change these variables to customize this script locally -@rem ################################ -@rem # you can define one or more thirdparty dirs, each where the varibale name -@rem # is prefixed with TPP_DIR -set "TPP_DIR=thirdparty" - -@rem # default configurations for dev -set "CONF_DEFAULT=etc/conf/dev" - -@rem # default thirdparty dist for dev -if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( - set "TPP_DIR_DEV=thirdparty/dev" -) - -@rem # default supported version for Python 3 -set SUPPORTED_PYTHON3=3.6 - -@rem ################################# - -@rem python --version -@rem python -c "import sys;print(sys.executable)" - @rem Current directory where this .bat files lives set CFG_ROOT_DIR=%~dp0 - @rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe - +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= @rem parse command line options and arguments @@ -56,13 +27,6 @@ if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts -@rem We are not cleaning: Either we have a provided configure config path or we use a default. -if ""%1""=="""" ( - set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% -) else ( - set CFG_CMD_LINE_ARGS=%1 -) - @rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run if exist ""%CONFIGURED_PYTHON%"" ( set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% @@ -83,7 +47,6 @@ if exist ""%PROVIDED_PYTHON%"" ( @rem otherwise we search for a suitable Python interpreter :find_python - @rem First check the existence of the "py" launcher (available in Python 3) @rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 @rem if not, check if we have an old py 2.7 @@ -116,8 +79,8 @@ if %ERRORLEVEL% == 0 ( ) ) -:run +:run @rem without this things may not always work on Windows 10, but this makes things slower set PYTHONDONTWRITEBYTECODE=1 @@ -141,9 +104,6 @@ echo A configuration script for Windows echo usage: configure [options] [path/to/config/directory] echo. echo The options and arguments are: -echo [path/to/config/directory] : this optionally sets the path to a -echo configuration directory to use. Defaults to etc/conf/dev if not set -echo. echo --clean : this is exclusive of anything else and cleans the environment echo from built and installed files echo. @@ -155,6 +115,6 @@ echo. :activate @rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%Scripts\activate" ( - "%CFG_ROOT_DIR%Scripts\activate" +if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( + "%CFG_ROOT_DIR%tmp\Scripts\activate" ) From 0f293cb11374d96df99757ce8dc6bed4730c9751 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:45:16 -0700 Subject: [PATCH 03/34] Install current project in configure.bat Signed-off-by: Jono Yang --- configure.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index cbb4244..958f5bf 100644 --- a/configure.bat +++ b/configure.bat @@ -21,6 +21,7 @@ set CFG_ROOT_DIR=%~dp0 set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= + @rem parse command line options and arguments :collectopts if "%1" EQU "--help" (goto cli_help) @@ -89,7 +90,7 @@ call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtua call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" call "%CFG_ROOT_DIR%tmp\Scripts\activate" call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel - +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] @rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( From 63f6946e1b3b070924b156a86b0ed0c3da6b7a48 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 11:41:51 -0700 Subject: [PATCH 04/34] Call pytest from proper location * Remove rerun option from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 904ac90..cf84da2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ jobs: image_name: ubuntu-16.04 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: @@ -24,7 +24,7 @@ jobs: image_name: macos-10.15 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -33,7 +33,7 @@ jobs: python_versions: ['3.6'] python_architecture: x86 test_suites: - all: Scripts\py.test -vvs --reruns=3 + all: tmp\Scripts\pytest -vvs - template: etc/ci/azure-win.yml parameters: @@ -42,4 +42,4 @@ jobs: python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: Scripts\py.test -vvs --reruns=3 + misc: tmp\Scripts\pytest -vvs From 7fd250691c0f9db772e289d3cfb2224d70f3dcd0 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 12:43:10 -0700 Subject: [PATCH 05/34] Use newer VM images on Azure Signed-off-by: Jono Yang --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf84da2..fad6928 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,8 +12,8 @@ jobs: - template: etc/ci/azure-linux.yml parameters: - job_name: vm_ubuntu16_py36 - image_name: ubuntu-16.04 + job_name: ubuntu18_py36 + image_name: ubuntu-18.04 python_versions: ['3.6'] test_suites: all: tmp/bin/pytest -n 2 -vvs @@ -28,8 +28,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_32_py36 - image_name: vs2017-win2016 + job_name: win2019_32_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x86 test_suites: @@ -37,9 +37,9 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_64_py36 - image_name: vs2017-win2016 + job_name: win2019_64_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -vvs From bceb8f98633bda15982c667848dc86be19ee6f97 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 28 Oct 2020 17:03:27 -0700 Subject: [PATCH 06/34] Add .gitattributes * We have this to ensure the line ending of configure.bat is always CRLF Signed-off-by: Jono Yang --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2d555b2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Set configure.bat's line ending to CRLF. Sometimes batch scripts don't work +# properly on Windows if the line ending is LF and not CRLF +configure.bat eol=crlf From e772fe670da017a9ff51e3b7119996f12169e79c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 3 Nov 2020 19:32:39 -0800 Subject: [PATCH 07/34] Clean template Signed-off-by: Jono Yang --- etc/ci/azure-linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml index 2e12e5b..752ae2e 100644 --- a/etc/ci/azure-linux.yml +++ b/etc/ci/azure-linux.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: '' python_versions: [] test_suites: {} python_architecture: x64 @@ -19,7 +19,6 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} - steps: - checkout: self fetchDepth: 10 From 629abedc035290810b4efa30b3e2cc47951f2344 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 5 Nov 2020 16:36:20 +0530 Subject: [PATCH 08/34] Update .gitignore to ignore Jupyter temp files Signed-off-by: Ayan Sinha Mahapatra --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0abbef1..68de2d2 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ pyvenv.cfg /.pytest_cache/ lib64 tcl + +# Ignore Jupyter Notebook related temp files +.ipynb_checkpoints/ From ef210cd813de2961fe8ca4a5ec7f14532ea6e9f8 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Mon, 16 Nov 2020 15:29:48 -0500 Subject: [PATCH 09/34] Quick doc update Signed-off-by: Steven Esser --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 0f1585b..a0e682f 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,9 @@ A brand new project cd my-new-repo git pull git@github.com:nexB/skeleton + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + From here, you can make the appropriate changes to the files for your specific project. Update an existing project From 0b6caf92fe394bec7f9f43a338dc0e2d3a32d5df Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:17:12 +0530 Subject: [PATCH 10/34] Add RTD docs configuration file Adds a RTD configuration file (v2) to customize builds. Signed-off-by: Ayan Sinha Mahapatra --- .readthedocs.yml | 18 ++++++++++++++++++ setup.cfg | 4 ++++ 2 files changed, 22 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..1b71cd9 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Where the Sphinx conf.py file is located +sphinx: + configuration: docs/source/conf.py + +# Setting the python version and doc build requirements +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/setup.cfg b/setup.cfg index a7ab2fe..e4274bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,3 +35,7 @@ testing = # upstream pytest >= 6 pytest-xdist >= 2 +docs= + Sphinx>=3.3.1 + sphinx-rtd-theme>=0.5.0 + doc8>=0.8.1 \ No newline at end of file From f2c1400e39aa99b2a53392910d01509a0a8114f7 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:19:15 +0530 Subject: [PATCH 11/34] Add basic RTD documentaion #4 Signed-off-by: Ayan Sinha Mahapatra --- docs/Makefile | 20 +++++++++++ docs/make.bat | 35 +++++++++++++++++++ docs/source/conf.py | 63 ++++++++++++++++++++++++++++++++++ docs/source/index.rst | 15 ++++++++ docs/source/skeleton/index.rst | 15 ++++++++ 5 files changed, 148 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/skeleton/index.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..6247f7e --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..529cae3 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,63 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'nexb-skeleton' +copyright = 'nexb Inc.' +author = 'nexb Inc.' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # override wide tables in RTD theme + ], + "display_github": True, + "github_user": "nexB", + "github_repo": "nexb-skeleton", + "github_version": "develop", # branch + "conf_py_path": "/docs/source/", # path in the checkout to the docs root + } \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..67fcf21 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,15 @@ +Welcome to nexb-skeleton's documentation! +========================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + skeleton/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst new file mode 100644 index 0000000..7dfc6cb --- /dev/null +++ b/docs/source/skeleton/index.rst @@ -0,0 +1,15 @@ +# Docs Structure Guide +# Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html +# +# 1. Place docs in folders under source for different sections +# 2. Link them by adding individual index files in each section +# to the main index, and then files for each section to their +# respective index files. +# 3. Use `.. include` statements to include other .rst files +# or part of them, or use hyperlinks to a section of the docs, +# to get rid of repetition. +# https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment +# +# Note: Replace these guide/placeholder docs + +.. include:: ../../../README.rst From e7d19903edae48c176baab11a9a5b7393ae74854 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:20:49 +0530 Subject: [PATCH 12/34] Add RTD requirements file and test scripts Signed-off-by: Ayan Sinha Mahapatra --- docs/scripts/doc8_style_check.sh | 5 +++++ docs/scripts/sphinx_build_link_check.sh | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 docs/scripts/doc8_style_check.sh create mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh new file mode 100644 index 0000000..9416323 --- /dev/null +++ b/docs/scripts/doc8_style_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Check for Style Code Violations +doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh new file mode 100644 index 0000000..c542686 --- /dev/null +++ b/docs/scripts/sphinx_build_link_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Build locally, and then check links +sphinx-build -E -W -b linkcheck source build \ No newline at end of file From 7d37af0b6636ab23780503a0e8de8287ab282b66 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 16 Dec 2020 19:45:34 +0530 Subject: [PATCH 13/34] Add `src` folder to pass CI tests and RTD builds Signed-off-by: Ayan Sinha Mahapatra --- src/README.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/README.rst diff --git a/src/README.rst b/src/README.rst new file mode 100644 index 0000000..efb1a14 --- /dev/null +++ b/src/README.rst @@ -0,0 +1,5 @@ +Package Module +-------------- + +Put your python modules in this directory. + From 03ffc8a23606b22ad728bb093e52264bfe1af658 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 6 Jan 2021 18:54:31 +0100 Subject: [PATCH 14/34] Ensure we use official full text of Apache 2.0 Taken from https://www.apache.org/licenses/LICENSE-2.0.txt Signed-off-by: Philippe Ombredanne --- apache-2.0.LICENSE | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apache-2.0.LICENSE b/apache-2.0.LICENSE index d9a10c0..261eeb9 100644 --- a/apache-2.0.LICENSE +++ b/apache-2.0.LICENSE @@ -174,3 +174,28 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 5e386d9ff0bfe77c170e37abde84313aa45b3a3f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:04:37 +0100 Subject: [PATCH 15/34] Never ever let Git convert line delimiters Signed-off-by: Philippe Ombredanne --- .gitattributes | 5 +- configure.bat | 242 ++++++++++++++++++++++++------------------------- 2 files changed, 123 insertions(+), 124 deletions(-) diff --git a/.gitattributes b/.gitattributes index 2d555b2..c446d38 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,2 @@ -# Set configure.bat's line ending to CRLF. Sometimes batch scripts don't work -# properly on Windows if the line ending is LF and not CRLF -configure.bat eol=crlf +# Ignore all Git auto CR/LF line endings conversions +* binary diff --git a/configure.bat b/configure.bat index 958f5bf..f03ea07 100644 --- a/configure.bat +++ b/configure.bat @@ -1,121 +1,121 @@ -@echo OFF -@setlocal -@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. - -@rem ################################ -@rem # A configuration script for Windows -@rem # -@rem # The options and (optional) arguments are: -@rem # --clean : this is exclusive of anything else and cleans the environment -@rem # from built and installed files -@rem # -@rem # --python < path to python.exe> : this must be the first argument and set -@rem # the path to the Python executable to use. If < path to python.exe> is -@rem # set to "path", then the executable will be the python.exe available -@rem # in the PATH. -@rem ################################ - -@rem Current directory where this .bat files lives -set CFG_ROOT_DIR=%~dp0 -@rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe -set PYTHON_EXECUTABLE= - - -@rem parse command line options and arguments -:collectopts -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b -if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts - -@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run -if exist ""%CONFIGURED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% - goto run -) - -@rem If we have a command arg for Python use this as-is -if ""%PROVIDED_PYTHON%""==""path"" ( - @rem use a bare python available in the PATH - set PYTHON_EXECUTABLE=python - goto run -) -if exist ""%PROVIDED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% - goto run -) - - -@rem otherwise we search for a suitable Python interpreter -:find_python -@rem First check the existence of the "py" launcher (available in Python 3) -@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 -@rem if not, check if we have an old py 2.7 -@rem exist if all fails - -where py >nul 2>nul -if %ERRORLEVEL% == 0 ( - @rem we have a py launcher, check for the availability of our required Python 3 version - py -3.6 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -3.6 - ) else ( - @rem we have no required python 3, let's try python 2: - py -2 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -2 - ) else ( - @rem we have py and no python 3 and 2, exit - echo * Unable to find an installation of Python. - exit /b 1 - ) - ) -) else ( - @rem we have no py launcher, check for a default Python 2 installation - if not exist ""%DEFAULT_PYTHON2%"" ( - echo * Unable to find an installation of Python. - exit /b 1 - ) else ( - set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% - ) -) - - -:run -@rem without this things may not always work on Windows 10, but this makes things slower -set PYTHONDONTWRITEBYTECODE=1 - -call mkdir "%CFG_ROOT_DIR%tmp" -call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz -call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" -call "%CFG_ROOT_DIR%tmp\Scripts\activate" -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] - -@rem Return a proper return code on failure -if %ERRORLEVEL% neq 0 ( - exit /b %ERRORLEVEL% -) -endlocal -goto activate - - -:cli_help -echo A configuration script for Windows -echo usage: configure [options] [path/to/config/directory] -echo. -echo The options and arguments are: -echo --clean : this is exclusive of anything else and cleans the environment -echo from built and installed files -echo. -echo --python path/to/python.exe : this is set to the path of an alternative -echo Python executable to use. If path/to/python.exe is set to "path", -echo then the executable will be the python.exe available in the PATH. -echo. - - -:activate -@rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( - "%CFG_ROOT_DIR%tmp\Scripts\activate" -) +@echo OFF +@setlocal +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem ################################ +@rem # A configuration script for Windows +@rem # +@rem # The options and (optional) arguments are: +@rem # --clean : this is exclusive of anything else and cleans the environment +@rem # from built and installed files +@rem # +@rem # --python < path to python.exe> : this must be the first argument and set +@rem # the path to the Python executable to use. If < path to python.exe> is +@rem # set to "path", then the executable will be the python.exe available +@rem # in the PATH. +@rem ################################ + +@rem Current directory where this .bat files lives +set CFG_ROOT_DIR=%~dp0 +@rem path where a configured Python should live in the current virtualenv if installed +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe +set PYTHON_EXECUTABLE= + + +@rem parse command line options and arguments +:collectopts +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b +if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts + +@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run +if exist ""%CONFIGURED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% + goto run +) + +@rem If we have a command arg for Python use this as-is +if ""%PROVIDED_PYTHON%""==""path"" ( + @rem use a bare python available in the PATH + set PYTHON_EXECUTABLE=python + goto run +) +if exist ""%PROVIDED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% + goto run +) + + +@rem otherwise we search for a suitable Python interpreter +:find_python +@rem First check the existence of the "py" launcher (available in Python 3) +@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 +@rem if not, check if we have an old py 2.7 +@rem exist if all fails + +where py >nul 2>nul +if %ERRORLEVEL% == 0 ( + @rem we have a py launcher, check for the availability of our required Python 3 version + py -3.6 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -3.6 + ) else ( + @rem we have no required python 3, let's try python 2: + py -2 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -2 + ) else ( + @rem we have py and no python 3 and 2, exit + echo * Unable to find an installation of Python. + exit /b 1 + ) + ) +) else ( + @rem we have no py launcher, check for a default Python 2 installation + if not exist ""%DEFAULT_PYTHON2%"" ( + echo * Unable to find an installation of Python. + exit /b 1 + ) else ( + set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + ) +) + + +:run +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +call mkdir "%CFG_ROOT_DIR%tmp" +call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call "%CFG_ROOT_DIR%tmp\Scripts\activate" +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] + +@rem Return a proper return code on failure +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) +endlocal +goto activate + + +:cli_help +echo A configuration script for Windows +echo usage: configure [options] [path/to/config/directory] +echo. +echo The options and arguments are: +echo --clean : this is exclusive of anything else and cleans the environment +echo from built and installed files +echo. +echo --python path/to/python.exe : this is set to the path of an alternative +echo Python executable to use. If path/to/python.exe is set to "path", +echo then the executable will be the python.exe available in the PATH. +echo. + + +:activate +@rem Activate the virtualenv +if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( + "%CFG_ROOT_DIR%tmp\Scripts\activate" +) From 2e48dca222b6f5889a3874ed5cda5476a7e9ff9b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:05:30 +0100 Subject: [PATCH 16/34] Run tests on more OSes and Python versions Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 55 ++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fad6928..9a4c950 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,45 +1,64 @@ ################################################################################ -# We use Azure to run the full tests suites on Python 3.6 -# on Windows (32 and 64), macOS and Linux (64 various distro) +# We use Azure to run the full tests suites on multiple Python 3.x +# on multiple Windows, macOS and Linux versions all on 64 bits +# These jobs are using VMs with Azure-provided Python builds ################################################################################ jobs: -################################################################################ -# These jobs are using VMs and Azure-provided Python 3.6 -################################################################################ + - template: etc/ci/azure-linux.yml + parameters: + job_name: ubuntu16_cpython + image_name: ubuntu-16.04 + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -vvs - template: etc/ci/azure-linux.yml parameters: - job_name: ubuntu18_py36 + job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-linux.yml + parameters: + job_name: ubuntu20_cpython + image_name: ubuntu-20.04 + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-mac.yml + parameters: + job_name: macos1014_cpython + image_name: macos-10.14 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: - job_name: macos1015_py36 + job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_32_py36 - image_name: windows-2019 - python_versions: ['3.6'] - python_architecture: x86 + job_name: win2016_cpython + image_name: vs2017-win2016 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_64_py36 + job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6'] - python_architecture: x64 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -n 2 -vvs From b959539450069bb573653a3d1265a95cb6c6f563 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:11:01 +0100 Subject: [PATCH 17/34] Do not make wheel universal Also include more license files Signed-off-by: Philippe Ombredanne --- setup.cfg | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index e4274bb..f791084 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,15 @@ -[wheel] -universal=1 - [metadata] -license_file = apache-2.0.LICENSE +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst name = skeleton author = nexB. Inc. and others author_email = info@aboutcode.org +license = Apache-2.0 + +# description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst url = https://github.com/nexB/skeleton @@ -17,6 +21,7 @@ classifiers = Topic :: Software Development Topic :: Utilities keywords = + utilities [options] package_dir= From 98641a067009e2b1c08682e99cbae84d30f59d15 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:00 +0100 Subject: [PATCH 18/34] Format and add license Signed-off-by: Philippe Ombredanne --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index a35c8c9..8f3a68e 100755 --- a/configure +++ b/configure @@ -1,6 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # set -e From 9fa3f315b7cc1495889b95d50a15cc738d48887f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:13 +0100 Subject: [PATCH 19/34] Format Signed-off-by: Philippe Ombredanne --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 45f160d..bac24a4 100644 --- a/setup.py +++ b/setup.py @@ -3,4 +3,4 @@ import setuptools if __name__ == "__main__": - setuptools.setup() \ No newline at end of file + setuptools.setup() From 0b8cd65580db4ad56b31fc67dd438a006cc66a6b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:27 +0100 Subject: [PATCH 20/34] Improve documentation Signed-off-by: Philippe Ombredanne --- src/README.rst | 5 +---- tests/README.rst | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 tests/README.rst diff --git a/src/README.rst b/src/README.rst index efb1a14..ec651fc 100644 --- a/src/README.rst +++ b/src/README.rst @@ -1,5 +1,2 @@ -Package Module --------------- - -Put your python modules in this directory. +Put your Python source code (and installable data) in this directory. diff --git a/tests/README.rst b/tests/README.rst new file mode 100644 index 0000000..d94783e --- /dev/null +++ b/tests/README.rst @@ -0,0 +1,2 @@ +Put your Python test modules in this directory. + From 300769149b530e283a4e4a6aa7f338ca72e5df5e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:48 +0100 Subject: [PATCH 21/34] Format for spaces Signed-off-by: Philippe Ombredanne --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e75f1ce..55fb92c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,11 +36,11 @@ norecursedirs = [ python_files = "*.py" -python_classes="Test" -python_functions="test" +python_classes = "Test" +python_functions = "test" addopts = [ "-rfExXw", "--strict", "--doctest-modules" -] \ No newline at end of file +] From b49895c69abd9713461de61032645cc2c492e73e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:14:47 +0100 Subject: [PATCH 22/34] Add manifest for source distributions Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ef3721e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +graft src + +include *.LICENSE +include NOTICE +include *.ABOUT +include *.toml +include *.yml +include *.rst +include setup.* +include configure* +include requirements* +include .git* + +global-exclude *.py[co] __pycache__ *.*~ + From d3e2d28d9f6cde0ded3d8450107d14fb4da05c4e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:15:06 +0100 Subject: [PATCH 23/34] Add Apache license NOTICE Signed-off-by: Philippe Ombredanne --- NOTICE | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..65936b2 --- /dev/null +++ b/NOTICE @@ -0,0 +1,19 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From 6288928b35fa4f61040fd0a40bf5dbf6b32324df Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:02:32 +0100 Subject: [PATCH 24/34] Update license text and notice Signed-off-by: Philippe Ombredanne --- NOTICE | 19 +++++++++++++++++++ apache-2.0.LICENSE | 25 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..16325da --- /dev/null +++ b/NOTICE @@ -0,0 +1,19 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/apache-2.0.LICENSE b/apache-2.0.LICENSE index d9a10c0..261eeb9 100644 --- a/apache-2.0.LICENSE +++ b/apache-2.0.LICENSE @@ -174,3 +174,28 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From ae42157b2a45d6ed87a2b4ffd5d1b13965e1fb6f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:02:43 +0100 Subject: [PATCH 25/34] Add correct authors Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc..00bd7dc 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,11 @@ The following organizations or individuals have contributed to this repo: -- +- Abhishek Kumar @Abhishek-Dev09 +- AlexB @a-tinsmith +- Maximilian Huber @maxhbr +- Michael Rupprecht @michaelrup +- Philippe Ombredanne @pombredanne +- Qingmin Duanmu @qduanmu +- Rakesh Balusa @balusarakesh +- Ravi Jain @JRavi2 +- Steven Esser @majurg From 74a7468a6d26ec900b76bfb92a7ce8fb71e9b3c8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:02:58 +0100 Subject: [PATCH 26/34] Populate changelog Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5f8bc8d..11413dc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,11 @@ Release notes +============= + +vNext +----- + +Version 20.10 ------------- -### Version 0.0.0 -*xxxx-xx-xx* -- Initial release. +*2020-10-06* +- Initial release. From 51ad1df5392f01a792d25d0cb0875de624364048 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:03:19 +0100 Subject: [PATCH 27/34] Ignore all Git auto CR/LF line endings conversions Signed-off-by: Philippe Ombredanne --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c446d38 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Ignore all Git auto CR/LF line endings conversions +* binary From bd323418c8cd5038212c27d32d671c677a75c93e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:04:04 +0100 Subject: [PATCH 28/34] Add manifest for sdist Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a3342b3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +graft src + +include *.LICENSE +include NOTICE +include *.ABOUT +include *.toml +include *.rst +include setup.* +include configure* +include requirements* + +global-exclude *.py[co] __pycache__ *.*~ + From 6ef97c1b590377fb0f9b3b02f94c7575a5a8c98d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 11:16:04 +0100 Subject: [PATCH 29/34] Add correct notice Signed-off-by: Philippe Ombredanne --- NOTICE | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/NOTICE b/NOTICE index 99eb76b..65936b2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,14 +1,19 @@ -graft src - -include *.LICENSE -include NOTICE -include *.ABOUT -include *.toml -include *.yml -include *.rst -include setup.* -include configure* -include requirements* -include .git* - -global-exclude *.py[co] __pycache__ *.*~ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From 9f0e9d67df38915760714fd0c900ac8dd8c45105 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 12:51:26 +0100 Subject: [PATCH 30/34] Improve README and command line notice Signed-off-by: Philippe Ombredanne --- README.rst | 20 +- src/extractcode/NOTICE | 75 ++--- src/extractcode/PSF.LICENSE | 635 ------------------------------------ src/extractcode/README.rst | 11 - 4 files changed, 32 insertions(+), 709 deletions(-) delete mode 100644 src/extractcode/PSF.LICENSE delete mode 100644 src/extractcode/README.rst diff --git a/README.rst b/README.rst index e80f191..b0f2655 100644 --- a/README.rst +++ b/README.rst @@ -1,18 +1,24 @@ ExtractCode =========== -CommonCode -========== - - license: Apache-2.0 - copyright: copyright (c) nexB. Inc. and others - homepage_url: https://github.com/nexB/extractode - keywords: archiev, extraction, libarchive, 7zip, scancode-toolkit -A set of functions and utilities used to extract archives in a mostly universal way. -This libraries uses multiple techniques to extract archives reliably including -using the Python standard library, and bundled 7zip and libarchive to use the -best tool to extract evebtually any archive and compressed file. + +ExtractCode is a universal archive extractor. It uses behind the scenes +the Python standard library, a custom ctypes binding to libarchive and +the 7zip command line to extract a large number of common and +less common archives and compressed files. It tries to extract things +in the same way on all OSes, including auto-renaming files that would +not have valid names on certain filesystems or when there are multiple +copies of the same path in a given archive. +The extraction is driven from a "voting" system that considers the +file extension(s) and name, the file type and mime type (using a ctypes +binding to libmagic) to select the most appropriate extractor or +uncompressor function. It can handle multi-level archives such as tar.gz. + Visit https://aboutcode.org and https://github.com/nexB/ for support and download. diff --git a/src/extractcode/NOTICE b/src/extractcode/NOTICE index e7eb6d3..65936b2 100644 --- a/src/extractcode/NOTICE +++ b/src/extractcode/NOTICE @@ -1,56 +1,19 @@ -Software license -================ - -Copyright (c) 2017 nexB Inc. and others. All rights reserved. -http://nexb.com and https://github.com/nexB/scancode-toolkit/ -The ScanCode software is licensed under the Apache License version 2.0. -Data generated with ScanCode require an acknowledgment. -ScanCode is a trademark of nexB Inc. - -You may not use this software except in compliance with the License. -You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - -When you publish or redistribute any data created with ScanCode or any ScanCode -derivative work, you must accompany this data with the following acknowledgment: - - Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES - OR CONDITIONS OF ANY KIND, either express or implied. No content created from - ScanCode should be considered or used as legal advice. Consult an Attorney - for any legal advice. - ScanCode is a free software code scanning tool from nexB Inc. and others. - Visit https://github.com/nexB/scancode-toolkit/ for support and download. - - -Third-party software licenses -============================= - -ScanCode embeds third-party free and open source software packages under various -licenses including copyleft licenses. Some of the third-party software packages -are delivered as pre-built binaries. The origin and license of these packages is -documented by .ABOUT files. - -The corresponding source code for pre-compiled third-party software is available -for immediate download from the same release page where you obtained ScanCode at: -https://github.com/nexB/scancode-toolkit/ -or https://github.com/nexB/scancode-thirdparty-src/ - -You may also contact us to request the source code by email at info@nexb.com or -by postal mail at: - - nexB Inc., ScanCode open source code request - 735 Industrial Road, Suite #101, 94070 San Carlos, CA, USA - -Please indicate in your communication the ScanCode version for which you are -requesting source code. - - -License for ScanCode datasets -============================= - -ScanCode includes datasets (e.g. for license detection) that are dedicated -to the Public Domain using the Creative Commons CC0 1.0 Universal (CC0 1.0) -Public Domain Dedication: http://creativecommons.org/publicdomain/zero/1.0/ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/src/extractcode/PSF.LICENSE b/src/extractcode/PSF.LICENSE deleted file mode 100644 index f472924..0000000 --- a/src/extractcode/PSF.LICENSE +++ /dev/null @@ -1,635 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.2 2.1.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2.1 2.2 2002 PSF yes - 2.2.2 2.2.1 2002 PSF yes - 2.2.3 2.2.2 2003 PSF yes - 2.3 2.2.2 2002-2003 PSF yes - 2.3.1 2.3 2002-2003 PSF yes - 2.3.2 2.3.1 2002-2003 PSF yes - 2.3.3 2.3.2 2002-2003 PSF yes - 2.3.4 2.3.3 2004 PSF yes - 2.3.5 2.3.4 2005 PSF yes - 2.4 2.3 2004 PSF yes - 2.4.1 2.4 2005 PSF yes - 2.4.2 2.4.1 2005 PSF yes - 2.4.3 2.4.2 2006 PSF yes - 2.4.4 2.4.3 2006 PSF yes - 2.5 2.4 2006 PSF yes - 2.5.1 2.5 2007 PSF yes - 2.5.2 2.5.2 2008 PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python -alone or in any derivative version, provided, however, that PSF's -License Agreement and PSF's notice of copyright, i.e., "Copyright (c) -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative -version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -This copy of Python includes a copy of bzip2, which is licensed under the following terms: - - -This program, "bzip2", the associated library "libbzip2", and all -documentation, are copyright (C) 1996-2005 Julian R Seward. All -rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. The origin of this software must not be misrepresented; you must - not claim that you wrote the original software. If you use this - software in a product, an acknowledgment in the product - documentation would be appreciated but is not required. - -3. Altered source versions must be plainly marked as such, and must - not be misrepresented as being the original software. - -4. The name of the author may not be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Julian Seward, Cambridge, UK. -jseward@acm.org -bzip2/libbzip2 version 1.0.3 of 15 February 2005 - - -This copy of Python includes a copy of db, which is licensed under the following terms: - -/*- - * $Id: LICENSE,v 12.1 2005/06/16 20:20:10 bostic Exp $ - */ - -The following is the license that applies to this copy of the Berkeley DB -software. For a license to use the Berkeley DB software under conditions -other than those described here, or to purchase support for this software, -please contact Sleepycat Software by email at info@sleepycat.com, or on -the Web at http://www.sleepycat.com. - -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= -/* - * Copyright (c) 1990-2005 - * Sleepycat Software. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Redistributions in any form must be accompanied by information on - * how to obtain complete source code for the DB software and any - * accompanying software that uses the DB software. The source code - * must either be included in the distribution or be available for no - * more than the cost of distribution plus a nominal fee, and must be - * freely redistributable under reasonable conditions. For an - * executable file, complete source code means the source code for all - * modules it contains. It does not include source code for modules or - * files that typically accompany the major components of the operating - * system on which the executable file runs. - * - * THIS SOFTWARE IS PROVIDED BY SLEEPYCAT SOFTWARE ``AS IS'' AND ANY EXPRESS - * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR - * NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL SLEEPYCAT SOFTWARE - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ -/* - * Copyright (c) 1990, 1993, 1994, 1995 - * The Regents of the University of California. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * Copyright (c) 1995, 1996 - * The President and Fellows of Harvard University. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY HARVARD AND ITS CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL HARVARD OR ITS CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -This copy of Python includes a copy of openssl, which is licensed under the following terms: - - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a dual license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. Actually both licenses are BSD-style - Open Source licenses. In case of any license issues related to OpenSSL - please contact openssl-core@openssl.org. - - OpenSSL License - --------------- - -/* ==================================================================== - * Copyright (c) 1998-2005 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - -/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ - - -This copy of Python includes a copy of tcl, which is licensed under the following terms: - -This software is copyrighted by the Regents of the University of -California, Sun Microsystems, Inc., Scriptics Corporation, ActiveState -Corporation and other parties. The following terms apply to all files -associated with the software unless explicitly disclaimed in -individual files. - -The authors hereby grant permission to use, copy, modify, distribute, -and license this software and its documentation for any purpose, provided -that existing copyright notices are retained in all copies and that this -notice is included verbatim in any distributions. No written agreement, -license, or royalty fee is required for any of the authorized uses. -Modifications to this software may be copyrighted by their authors -and need not follow the licensing terms described here, provided that -the new terms are clearly indicated on the first page of each file where -they apply. - -IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY -FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES -ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY -DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE -IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE -NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR -MODIFICATIONS. - -GOVERNMENT USE: If you are acquiring this software on behalf of the -U.S. government, the Government shall have only "Restricted Rights" -in the software and related documentation as defined in the Federal -Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you -are acquiring the software on behalf of the Department of Defense, the -software shall be classified as "Commercial Computer Software" and the -Government shall have only "Restricted Rights" as defined in Clause -252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the -authors grant the U.S. Government and others acting in its behalf -permission to use and distribute the software in accordance with the -terms specified in this license. - -This copy of Python includes a copy of tk, which is licensed under the following terms: - -This software is copyrighted by the Regents of the University of -California, Sun Microsystems, Inc., and other parties. The following -terms apply to all files associated with the software unless explicitly -disclaimed in individual files. - -The authors hereby grant permission to use, copy, modify, distribute, -and license this software and its documentation for any purpose, provided -that existing copyright notices are retained in all copies and that this -notice is included verbatim in any distributions. No written agreement, -license, or royalty fee is required for any of the authorized uses. -Modifications to this software may be copyrighted by their authors -and need not follow the licensing terms described here, provided that -the new terms are clearly indicated on the first page of each file where -they apply. - -IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY -FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES -ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY -DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE -IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE -NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR -MODIFICATIONS. - -GOVERNMENT USE: If you are acquiring this software on behalf of the -U.S. government, the Government shall have only "Restricted Rights" -in the software and related documentation as defined in the Federal -Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you -are acquiring the software on behalf of the Department of Defense, the -software shall be classified as "Commercial Computer Software" and the -Government shall have only "Restricted Rights" as defined in Clause -252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the -authors grant the U.S. Government and others acting in its behalf -permission to use and distribute the software in accordance with the -terms specified in this license. diff --git a/src/extractcode/README.rst b/src/extractcode/README.rst deleted file mode 100644 index bc5bfeb..0000000 --- a/src/extractcode/README.rst +++ /dev/null @@ -1,11 +0,0 @@ -extractcode is a universal archive extractor. It uses behind the scenes -the Python standard library, a custom ctypes binding to libarchive and -the 7zip command line to extract a large number of common and -less common archives and compressed files. It tries to extract things -in the same way on all OSes, including auto-renaming files that would -not have valid names on certain filesystems or when there are multiple -copies of the same path in a given archive. -The extraction is driven from a "voting" system that considers the -file extension(s) and name, the file type and mime type (using a ctypes -binding to libmagic) to select the most appropriate extractor or -uncompressor function. It can handle multi-level archives such as tar.gz. From e872181106dcfeec4f4db7dbfb2a2a5da15683c5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 12:51:39 +0100 Subject: [PATCH 31/34] Remove Python2 support Signed-off-by: Philippe Ombredanne --- src/extractcode/__init__.py | 101 +++------ src/extractcode/api.py | 37 ++-- src/extractcode/archive.py | 62 ++---- src/extractcode/cli.py | 83 +++---- src/extractcode/extract.py | 54 ++--- src/extractcode/libarchive2.py | 50 ++--- src/extractcode/patch.py | 60 ++--- src/extractcode/sevenzip.py | 107 ++++----- src/extractcode/uncompress.py | 97 +++----- tests/extractcode/extractcode_assert_utils.py | 59 ++--- tests/extractcode/test_archive.py | 208 +++++------------- tests/extractcode/test_extract.py | 47 ++-- tests/extractcode/test_extractcode.py | 36 ++- tests/extractcode/test_extractcode_cli.py | 82 +++---- tests/extractcode/test_libarchive2.py | 37 ++-- tests/extractcode/test_patch.py | 49 ++--- tests/extractcode/test_sevenzip.py | 50 ++--- 17 files changed, 420 insertions(+), 799 deletions(-) diff --git a/src/extractcode/__init__.py b/src/extractcode/__init__.py index b9900b4..b41efdc 100644 --- a/src/extractcode/__init__.py +++ b/src/extractcode/__init__.py @@ -1,30 +1,22 @@ # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals import logging import os @@ -33,17 +25,16 @@ import shutil import sys +from os.path import dirname +from os.path import join +from os.path import exists + from commoncode.fileutils import as_posixpath from commoncode.fileutils import create_dir from commoncode.fileutils import file_name -from commoncode.fileutils import fsencode from commoncode.fileutils import parent_directory from commoncode.text import toascii from commoncode.system import on_linux -from commoncode.system import py2 -from os.path import dirname -from os.path import join -from os.path import exists logger = logging.getLogger(__name__) DEBUG = False @@ -53,16 +44,8 @@ root_dir = join(dirname(__file__), 'bin') -POSIX_PATH_SEP = b'/' if (on_linux and py2) else '/' -WIN_PATH_SEP = b'\\' if (on_linux and py2) else '\\' -PATHS_SEPS = POSIX_PATH_SEP + WIN_PATH_SEP -EMPTY_STRING = b'' if (on_linux and py2) else '' -DOT = b'.' if (on_linux and py2) else '.' -DOTDOT = DOT + DOT -UNDERSCORE = b'_' if (on_linux and py2) else '_' - # Suffix added to extracted target_dir paths -EXTRACT_SUFFIX = b'-extract' if (on_linux and py2) else r'-extract' +EXTRACT_SUFFIX = '-extract' # high level archive "kinds" docs = 1 @@ -103,10 +86,7 @@ def is_extraction_path(path): """ Return True is the path points to an extraction path. """ - if on_linux and py2: - path = fsencode(path) - - return path and path.rstrip(PATHS_SEPS).endswith(EXTRACT_SUFFIX) + return path and path.rstrip('\\/').endswith(EXTRACT_SUFFIX) def is_extracted(location): @@ -114,8 +94,6 @@ def is_extracted(location): Return True is the location is already extracted to the corresponding extraction location. """ - if on_linux and py2: - location = fsencode(location) return location and exists(get_extraction_path(location)) @@ -123,18 +101,14 @@ def get_extraction_path(path): """ Return a path where to extract. """ - if on_linux and py2: - path = fsencode(path) - return path.rstrip(PATHS_SEPS) + EXTRACT_SUFFIX + return path.rstrip('\\/') + EXTRACT_SUFFIX def remove_archive_suffix(path): """ Remove all the extracted suffix from a path. """ - if on_linux and py2: - path = fsencode(path) - return re.sub(EXTRACT_SUFFIX, EMPTY_STRING, path) + return re.sub(EXTRACT_SUFFIX, '', path) def remove_backslashes_and_dotdots(directory): @@ -142,21 +116,16 @@ def remove_backslashes_and_dotdots(directory): Walk a directory and rename the files if their names contain backslashes. Return a list of errors if any. """ - if on_linux and py2: - directory = fsencode(directory) errors = [] for top, _, files in os.walk(directory): for filename in files: - if not (WIN_PATH_SEP in filename or DOTDOT in filename): + if not ('\\' in filename or '..' in filename): continue try: - new_path = as_posixpath(filename) - new_path = new_path.strip(POSIX_PATH_SEP) - new_path = posixpath.normpath(new_path) - new_path = new_path.replace(DOTDOT, POSIX_PATH_SEP) - new_path = new_path.strip(POSIX_PATH_SEP) + new_path = as_posixpath(filename).strip('/') + new_path = posixpath.normpath(new_path).replace('..', '/').strip('/') new_path = posixpath.normpath(new_path) - segments = new_path.split(POSIX_PATH_SEP) + segments = new_path.split('/') directory = join(top, *segments[:-1]) create_dir(directory) shutil.move(join(top, filename), join(top, *segments)) @@ -180,9 +149,7 @@ def new_name(location, is_dir=False): the extension unchanged. """ assert location - if on_linux and py2: - location = fsencode(location) - location = location.rstrip(PATHS_SEPS) + location = location.rstrip('\\/') assert location parent = parent_directory(location) @@ -193,8 +160,8 @@ def new_name(location, is_dir=False): filename = file_name(location) # corner case - if filename in (DOT, DOT): - filename = UNDERSCORE + if filename in ('.', '..'): + filename = '_' # if unique, return this if filename.lower() not in siblings_lower: @@ -204,19 +171,19 @@ def new_name(location, is_dir=False): if is_dir: # directories do not have an "extension" base_name = filename - ext = EMPTY_STRING + ext = '' else: - base_name, dot, ext = filename.partition(DOT) + base_name, dot, ext = filename.partition('.') if dot: - ext = dot + ext + ext = f'.{ext}' else: base_name = filename - ext = EMPTY_STRING + ext = '' # find a unique filename, adding a counter int to the base_name counter = 1 while 1: - filename = base_name + UNDERSCORE + str(counter) + ext + filename = f'{base_name}_{counter}{ext}' if filename.lower() not in siblings_lower: break counter += 1 diff --git a/src/extractcode/api.py b/src/extractcode/api.py index 428b76c..4842785 100644 --- a/src/extractcode/api.py +++ b/src/extractcode/api.py @@ -1,31 +1,22 @@ # -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals """ diff --git a/src/extractcode/archive.py b/src/extractcode/archive.py index d3a8080..a088929 100644 --- a/src/extractcode/archive.py +++ b/src/extractcode/archive.py @@ -1,42 +1,32 @@ # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from collections import namedtuple import logging import os -from commoncode import compat from commoncode import fileutils from commoncode import filetype from commoncode import functional from commoncode.ignore import is_ignored from commoncode.system import on_linux -from commoncode.system import py2 from typecode import contenttype @@ -154,8 +144,6 @@ def get_best_handler(location, kinds=all_kinds): """ Return the best handler of None for the file at location. """ - if on_linux and py2: - location = fileutils.fsencode(location) location = os.path.abspath(os.path.expanduser(location)) if not filetype.is_file(location): return @@ -173,9 +161,6 @@ def get_handlers(location): Return an iterable of (handler, type_matched, mime_matched, extension_matched,) for this `location`. """ - if on_linux and py2: - location = fileutils.fsencode(location) - if filetype.is_file(location): T = contenttype.get_type(location) @@ -197,8 +182,6 @@ def get_handlers(location): mime_matched = handler.mimetypes and any(m in mtype for m in handler.mimetypes) exts = handler.extensions if exts: - if on_linux and py2: - exts = tuple(fileutils.fsencode(e) for e in exts) extension_matched = exts and location.lower().endswith(exts) if TRACE_DEEP: @@ -326,13 +309,10 @@ def extract_twice(location, target_dir, extractor1, extractor2): hard to trace and debug very quickly. A depth of two is simple and sane and covers most common cases. """ - if on_linux and py2: - location = fileutils.fsencode(location) - target_dir = fileutils.fsencode(target_dir) abs_location = os.path.abspath(os.path.expanduser(location)) - abs_target_dir = compat.unicode(os.path.abspath(os.path.expanduser(target_dir))) + abs_target_dir = str(os.path.abspath(os.path.expanduser(target_dir))) # extract first the intermediate payload to a temp dir - temp_target = compat.unicode(fileutils.get_temp_dir(prefix='extractcode-extract-')) + temp_target = str(fileutils.get_temp_dir(prefix='extractcode-extract-')) warnings = extractor1(abs_location, temp_target) if TRACE: logger.debug('extract_twice: temp_target: %(temp_target)r' % locals()) @@ -364,9 +344,9 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2): and a fallback extractor will succeed. """ abs_location = os.path.abspath(os.path.expanduser(location)) - abs_target_dir = compat.unicode(os.path.abspath(os.path.expanduser(target_dir))) + abs_target_dir = str(os.path.abspath(os.path.expanduser(target_dir))) # attempt extract first to a temp dir - temp_target1 = compat.unicode(fileutils.get_temp_dir(prefix='extractcode-extract1-')) + temp_target1 = str(fileutils.get_temp_dir(prefix='extractcode-extract1-')) try: warnings = extractor1(abs_location, temp_target1) if TRACE: @@ -374,7 +354,7 @@ def extract_with_fallback(location, target_dir, extractor1, extractor2): fileutils.copytree(temp_target1, abs_target_dir) except: try: - temp_target2 = compat.unicode(fileutils.get_temp_dir(prefix='extractcode-extract2-')) + temp_target2 = str(fileutils.get_temp_dir(prefix='extractcode-extract2-')) warnings = extractor2(abs_location, temp_target2) if TRACE: logger.debug('extract_with_fallback: temp_target2: %(temp_target2)r' % locals()) @@ -395,8 +375,8 @@ def try_to_extract(location, target_dir, extractor): but do not care if this fails. """ abs_location = os.path.abspath(os.path.expanduser(location)) - abs_target_dir = compat.unicode(os.path.abspath(os.path.expanduser(target_dir))) - temp_target = compat.unicode(fileutils.get_temp_dir(prefix='extractcode-extract1-')) + abs_target_dir = str(os.path.abspath(os.path.expanduser(target_dir))) + temp_target = str(fileutils.get_temp_dir(prefix='extractcode-extract1-')) warnings = [] try: warnings = extractor(abs_location, temp_target) diff --git a/src/extractcode/cli.py b/src/extractcode/cli.py index d59ddf7..2903c41 100644 --- a/src/extractcode/cli.py +++ b/src/extractcode/cli.py @@ -1,50 +1,40 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals -from functools import partial -from os import path +import os +import functools import click click.disable_unicode_literals_warning = True from commoncode import cliutils -from commoncode import compat from commoncode import fileutils from commoncode import filetype from commoncode.text import toascii from extractcode.api import extract_archives - __version__ = '2020.09.21' - -echo_stderr = partial(click.secho, err=True) +echo_stderr = functools.partial(click.secho, err=True) def print_version(ctx, param, value): @@ -53,26 +43,17 @@ def print_version(ctx, param, value): echo_stderr('ExtractCode version ' + __version__) ctx.exit() + info_text = ''' -ExtractCode is a mostly universal archive and compressed files extractor, with +ExtractCode is a mostly universal archive and compressed files extractor, with a particular focus on code archives. -Visit https://github.com/nexB/scancode-toolkit/ for support and download. +Visit https://aboutcode.org and https://github.com/nexB/extractcode/ for support and download. ''' -notice_path = path.join(path.abspath(path.dirname(__file__)), 'NOTICE') +notice_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'NOTICE') notice_text = open(notice_path).read() - -delimiter = '\n\n\n' -[notice_text, extra_notice_text] = notice_text.split(delimiter, 1) -extra_notice_text = delimiter + extra_notice_text - -delimiter = '\n\n ' -[notice_text, acknowledgment_text] = notice_text.split(delimiter, 1) -acknowledgment_text = delimiter + acknowledgment_text - -notice = acknowledgment_text.strip().replace(' ', '') - + def print_about(ctx, param, value): """ @@ -80,7 +61,7 @@ def print_about(ctx, param, value): """ if not value or ctx.resilient_parsing: return - click.echo(info_text + notice_text + acknowledgment_text + extra_notice_text) + click.echo(info_text + notice_text) ctx.exit() @@ -115,7 +96,7 @@ class ExtractCommand(cliutils.BaseCommand): @click.command(name='extractcode', epilog=epilog_text, cls=ExtractCommand) @click.pass_context -@click.argument('input', metavar='', type=click.Path(exists=True, readable=True, path_type=fileutils.PATH_TYPE)) +@click.argument('input', metavar='', type=click.Path(exists=True, readable=True)) @click.option('--verbose', is_flag=True, default=False, help='Print verbose file-by-file progress messages.') @click.option('--quiet', is_flag=True, default=False, help='Do not print any summary or progress message.') @@ -130,11 +111,11 @@ def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, ignore, """extract archives and compressed files found in the file or directory tree. Archives found inside an extracted archive are extracted recursively. - Extraction for each archive is done in-place in a new directory named + Extraction for each archive is done in-place in a new directory named '-extract' created side-by-side with an archive. """ - abs_location = fileutils.as_posixpath(path.abspath(path.expanduser(input))) + abs_location = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input))) def extract_event(item): """ @@ -145,7 +126,7 @@ def extract_event(item): if not item: return '' source = item.source - if not isinstance(source, compat.unicode): + if not isinstance(source, str): source = toascii(source, translit=True).decode('utf-8', 'replace') if verbose: if item.done: @@ -153,7 +134,7 @@ def extract_event(item): line = source and get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or '' else: line = source and fileutils.file_name(source) or '' - if not isinstance(line, compat.unicode): + if not isinstance(line, str): line = toascii(line, translit=True).decode('utf-8', 'replace') return 'Extracting: %(line)s' % locals() @@ -168,7 +149,7 @@ def display_extract_summary(): has_errors = has_errors or bool(xev.errors) has_warnings = has_warnings or bool(xev.warnings) source = fileutils.as_posixpath(xev.source) - if not isinstance(source, compat.unicode): + if not isinstance(source, str): source = toascii(source, translit=True).decode('utf-8', 'replace') source = get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) for e in xev.errors: @@ -191,7 +172,7 @@ def display_extract_summary(): extract_result_with_errors = [] unique_extract_events_with_errors = set() has_extract_errors = False - + extractibles = extract_archives( abs_location, recurse=not shallow, replace_originals=replace_originals, ignore_pattern=ignore) @@ -222,7 +203,7 @@ def get_relative_path(path, len_base_path, base_is_dir): base path of `len_base_path` length where the base is a directory if `base_is_dir` True or a file otherwise. """ - path = fileutils.fsdecode(path) + path = os.fsdecode(path) if base_is_dir: rel_path = path[len_base_path:] else: diff --git a/src/extractcode/extract.py b/src/extractcode/extract.py index b08f90e..21a6e33 100644 --- a/src/extractcode/extract.py +++ b/src/extractcode/extract.py @@ -1,43 +1,35 @@ # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +import logging +import traceback from collections import namedtuple from functools import partial -import logging from os.path import abspath from os.path import expanduser from os.path import join -import traceback from commoncode import fileutils from commoncode import ignore import extractcode -from extractcode import archive logger = logging.getLogger(__name__) TRACE = False @@ -189,7 +181,7 @@ def extract_files( logger.debug('extract:walk not recurse: skipped file: %(loc)r' % locals()) continue - if not archive.should_extract(loc, kinds, ignore_pattern): + if not extractcode.archive.should_extract(loc, kinds, ignore_pattern): if TRACE: logger.debug('extract:walk: skipped file: not should_extract: %(loc)r' % locals()) continue @@ -219,9 +211,9 @@ def extract_files( def extract_file( - location, - target, - kinds=extractcode.default_kinds, + location, + target, + kinds=extractcode.default_kinds, verbose=False, ): """ @@ -230,13 +222,11 @@ def extract_file( """ warnings = [] errors = [] - extractor = archive.get_extractor(location, kinds) + extractor = extractcode.archive.get_extractor(location, kinds) if TRACE: emodule = getattr(extractor, '__module__', '') ename = getattr(extractor, '__name__', '') - logger.debug( - 'extract_file: extractor: for: {location} with kinds: {kinds}: {emodule}.{ename}' - .format(**locals())) + logger.debug(f'extract_file: extractor: for: {location} with kinds: {kinds}: {emodule}.{ename}') if extractor: yield ExtractEvent(location, target, done=False, warnings=[], errors=[]) diff --git a/src/extractcode/libarchive2.py b/src/extractcode/libarchive2.py index 70eeeb5..6b72456 100644 --- a/src/extractcode/libarchive2.py +++ b/src/extractcode/libarchive2.py @@ -1,31 +1,22 @@ # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from functools import partial import locale @@ -43,11 +34,8 @@ import attr from commoncode import command -from commoncode import compat from commoncode import fileutils from commoncode import paths -from commoncode.system import py2 -from commoncode.system import py3 from commoncode import text import extractcode @@ -262,7 +250,7 @@ def close(self): free_archive(self.archive_struct) self.archive_struct = None - def iter(self, verbose=False): + def iter(self): """ Yield Entry for this archive. """ @@ -387,13 +375,7 @@ def get_path(self, func, func_w): path = func(self.entry_struct) if not path: path = func_w(self.entry_struct) - - if py2 and isinstance(path, compat.unicode): - # FIXME: encoding MAY fail if the encoding is NOT UTF-8! - # .... should we transliterate there? - path = path.encode('utf-8') - - if py3 and not isinstance(path, compat.unicode): + if not isinstance(path, str): path = text.as_unicode(path) return path diff --git a/src/extractcode/patch.py b/src/extractcode/patch.py index da8ab59..8a197ca 100644 --- a/src/extractcode/patch.py +++ b/src/extractcode/patch.py @@ -1,30 +1,22 @@ # -# Copyright (c) 2015 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals import posixpath import logging @@ -34,13 +26,11 @@ from commoncode import paths from commoncode import fileutils -from commoncode.system import py2 -from commoncode.system import py3 from commoncode import text -import extractcode -from extractcode import ExtractErrorFailedToExtract import typecode.contenttype +import extractcode +from extractcode import ExtractErrorFailedToExtract """ Low level utilities to parse patch files and treat them as if they were @@ -93,15 +83,9 @@ def extract(location, target_dir): # write the location proper, with a suffix extension to avoid # recursive extraction - if py2: - mode = 'wb' - eol = b'\n' - if py3: - mode = 'w' - eol = u'\n' subfile_path = base_subfile_path + extractcode.EXTRACT_SUFFIX - with open(subfile_path, mode) as subfile: - subfile.write(eol.join(text)) + with open(subfile_path, 'w') as subfile: + subfile.write('\n'.join(text)) return [] @@ -113,9 +97,11 @@ def is_patch(location, include_extracted=False): """ T = typecode.contenttype.get_type(location) file_name = fileutils.file_name(location) - patch_like = ('diff ' in T.filetype_file.lower() - or '.diff' in file_name - or '.patch' in file_name) + patch_like = ( + 'diff ' in T.filetype_file.lower() + or '.diff' in file_name + or '.patch' in file_name + ) if not patch_like: return False diff --git a/src/extractcode/sevenzip.py b/src/extractcode/sevenzip.py index ebc8498..b34bfbd 100644 --- a/src/extractcode/sevenzip.py +++ b/src/extractcode/sevenzip.py @@ -1,30 +1,23 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from collections import defaultdict import io @@ -42,18 +35,13 @@ from commoncode.system import on_mac from commoncode.system import on_macos_14_or_higher from commoncode.system import on_windows -from commoncode.system import py3 from commoncode import text import extractcode from extractcode import ExtractErrorFailedToExtract from extractcode import ExtractWarningIncorrectEntry -if py3: - from shlex import quote as shlex_quote # NOQA -else: - from pipes import quote as shlex_quote # NOQA - +from shlex import quote as shlex_quote """ Low level support for p/7zip-based archive extraction. @@ -95,8 +83,12 @@ def get_7z_errors(stdout, stderr): if not stdout or not stdout.strip(): return - # ERROR: Can not create symbolic link : A required privilege is not held by the client. : .\2-SYMTYPE - find_7z_errors = re.compile('^Error:(.*)$', re.MULTILINE | re.DOTALL | re.IGNORECASE).findall # NOQA + # ERROR: Can not create symbolic link : A required privilege is not held by + # the client. : .\2-SYMTYPE + find_7z_errors = re.compile( + '^Error:(.*)$', + re.MULTILINE | re.DOTALL | re.IGNORECASE + ).findall stdlow = stderr.lower() for err, msg in sevenzip_errors: @@ -558,29 +550,11 @@ def parse_7z_listing(location, utf=False): We ignore the header and footer in a listing. """ - if utf or py3: - # read to unicode - with io.open(location, 'r', encoding='utf-8') as listing: - text = listing.read() - text = text.replace(u'\r\n', u'\n') - - end_of_header = u'----------\n' - path_key = u'Path' - kv_sep = u'=' - path_blocks_sep = u'\n\n' - line_sep = u'\n' - - else: - # read to bytes - with io.open(location, 'rb') as listing: - text = listing.read() - text = text.replace(b'\r\n', b'\n') - - end_of_header = b'----------\n' - path_key = b'Path' - kv_sep = b'=' - path_blocks_sep = b'\n\n' - line_sep = b'\n' + # read to unicode + with io.open(location, 'r', encoding='utf-8') as listing: + text = listing.read() + # normalize line endings to POSIX + text = text.replace('\r\n', '\n') if TRACE: logger.debug('parse_7z_listing: initial text: type: ' + repr(type(text))) @@ -588,7 +562,8 @@ def parse_7z_listing(location, utf=False): print(text) print('--------------------------------------') - # for now we ignore the header + # for now we ignore the header, and only start dealing with text after that + end_of_header = '----------\n' _header, _, paths = text.rpartition(end_of_header) if not paths: @@ -601,8 +576,11 @@ def parse_7z_listing(location, utf=False): # (unless there is a \n in file name which is an error condition) # - ends with an empty line # then we have a global footer + two_empty_lines = '\n\n' + path_key = 'Path' + path_blocks = [pb for pb in paths.split(two_empty_lines) if pb and path_key in pb] - path_blocks = [pb for pb in paths.split(path_blocks_sep) if pb and path_key in pb] + key_value_sep = '=' entries = [] @@ -613,22 +591,22 @@ def parse_7z_listing(location, utf=False): continue # we have a weird case of path with line returns in the file name # we concatenate these in the first Path line - while len(lines) > 1 and lines[0].startswith(path_key) and kv_sep not in lines[1]: + while len(lines) > 1 and lines[0].startswith(path_key) and key_value_sep not in lines[1]: first_line = lines[0] second_line = lines.pop(1) - first_line = line_sep.join([first_line, second_line]) + first_line = '\n'.join([first_line, second_line]) lines[0] = first_line - dangling_lines = [line for line in lines if kv_sep not in line] + dangling_lines = [line for line in lines if key_value_sep not in line] entry_errors = [] if dangling_lines: emsg = 'Invalid 7z listing path block missing "=" as key/value separator: {}'.format(repr(path_block)) entry_errors.append(emsg) entry_attributes = {} - key_lines = [line for line in lines if kv_sep in line] + key_lines = [line for line in lines if key_value_sep in line] for line in key_lines: - k, _, v = line.partition(kv_sep) + k, _, v = line.partition(key_value_sep) k = k.strip() v = v.strip() entry_attributes[k] = v @@ -643,15 +621,6 @@ def parse_7z_listing(location, utf=False): return entries -def filter_entries(entries): - """ - Given an iterable of entries, return two list of entries: - a list of valid entries that can be extracted and a list entries that cannot - be extracted. - """ - # extractible - - @attr.s(slots=True) class Entry(object): """ diff --git a/src/extractcode/uncompress.py b/src/extractcode/uncompress.py index 4dcc2eb..f584e84 100644 --- a/src/extractcode/uncompress.py +++ b/src/extractcode/uncompress.py @@ -1,50 +1,34 @@ # -# Copyright (c) 2018 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals -from functools import partial +import bz2 import gzip import logging import os import shutil -try: - # These imports add support for multistream BZ2 files - # This is a Python2 backport for bz2file from Python3 - # Because of http://bugs.python.org/issue20781 - from bz2file import BZ2File -except ImportError: - from bz2 import BZ2File - +from functools import partial from commoncode import fileutils -from commoncode.system import py2 -from extractcode import EXTRACT_SUFFIX +from extractcode import EXTRACT_SUFFIX DEBUG = False logger = logging.getLogger(__name__) @@ -65,7 +49,9 @@ def uncompress(location, target_dir, decompressor, suffix=EXTRACT_SUFFIX): # name when present. if DEBUG: logger.debug('uncompress: ' + location) + tmp_loc, warnings = uncompress_file(location, decompressor) + target_location = os.path.join(target_dir, os.path.basename(location) + suffix) if os.path.exists(target_location): fileutils.delete(target_location) @@ -87,7 +73,9 @@ def uncompress_file(location, decompressor): warnings = [] base_name = fileutils.file_base_name(location) - target_location = os.path.join(fileutils.get_temp_dir(prefix='extractcode-extract-'), base_name) + target_location = os.path.join(fileutils.get_temp_dir( + prefix='extractcode-extract-'), base_name) + with decompressor(location, 'rb') as compressed: with open(target_location, 'wb') as uncompressed: buffer_size = 32 * 1024 * 1024 @@ -96,8 +84,10 @@ def uncompress_file(location, decompressor): if not chunk: break uncompressed.write(chunk) + if getattr(decompressor, 'has_trailing_garbage', False): warnings.append(location + ': Trailing garbage found and ignored.') + return target_location, warnings @@ -106,7 +96,7 @@ def uncompress_bzip2(location, target_dir): Uncompress a bzip2 compressed file at location in the target_dir. Return a list warnings messages. """ - return uncompress(location, target_dir, BZ2File) + return uncompress(location, target_dir, decompressor=bz2.BZ2File) def uncompress_gzip(location, target_dir): @@ -115,38 +105,7 @@ def uncompress_gzip(location, target_dir): Return a list warnings messages. """ - return uncompress(location, target_dir, GzipFileWithTrailing) - - -class _GzipFileWithTrailing(gzip.GzipFile): - """ - A subclass of gzip.GzipFile supporting files with trailing garbage. Ignore - the garbage. - """ - # TODO: what is first_file?? - first_file = True - gzip_magic = b'\037\213' - has_trailing_garbage = False - - def _read_gzip_header(self): - # read the first two bytes - magic = self.fileobj.read(2) - # rewind two bytes back - self.fileobj.seek(-2, os.SEEK_CUR) - is_gzip = magic != self.gzip_magic - if is_gzip and not self.first_file: - self.first_file = False - self.has_trailing_garbage = True - raise EOFError('Trailing garbage found') - - self.first_file = False - gzip.GzipFile._read_gzip_header(self) - -if py2: - GzipFileWithTrailing = _GzipFileWithTrailing -else: - # FIXME: there is no easy way to monkey patch the gzip.py code in Python 3 - GzipFileWithTrailing = gzip.GzipFile + return uncompress(location, target_dir, decompressor=gzip.GzipFile) def get_compressed_file_content(location, decompressor): @@ -163,5 +122,5 @@ def get_compressed_file_content(location, decompressor): return content, warnings -get_gz_compressed_file_content = partial(get_compressed_file_content, decompressor=GzipFileWithTrailing) -get_bz2_compressed_file_content = partial(get_compressed_file_content, decompressor=BZ2File) +get_gz_compressed_file_content = partial(get_compressed_file_content, decompressor=gzip.GzipFile) +get_bz2_compressed_file_content = partial(get_compressed_file_content, decompressor=bz2.BZ2File) diff --git a/tests/extractcode/extractcode_assert_utils.py b/tests/extractcode/extractcode_assert_utils.py index 786c90c..537cf1d 100644 --- a/tests/extractcode/extractcode_assert_utils.py +++ b/tests/extractcode/extractcode_assert_utils.py @@ -1,43 +1,33 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2015 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from collections import OrderedDict import json import os import ntpath import posixpath -from commoncode import compat from commoncode import filetype from commoncode import fileutils from commoncode.testcase import FileBasedTesting from commoncode.system import on_windows -from commoncode.system import py2 -from commoncode.system import py3 """ Shared archiving test utils. @@ -50,14 +40,10 @@ def check_size(expected_size, location): def check_results_with_expected_json(results, expected_loc, regen=False): if regen: - if py2: - wmode = 'wb' - if py3: - wmode = 'w' - with open(expected_loc, wmode) as ex: + with open(expected_loc, 'w') as ex: json.dump(results, ex, indent=2, separators=(',', ':')) - with open(expected_loc, 'rb') as ex: - expected = json.load(ex, encoding='utf-8', object_pairs_hook=OrderedDict) + with open(expected_loc) as ex: + expected = json.load(ex) try: assert expected == results except AssertionError: @@ -92,13 +78,12 @@ def check_files(test_dir, expected, regen=False): expected_is_json_file = True # this is a path to a JSON file if regen: - wmode = 'wb' if py2 else 'w' - with open(expected, wmode) as ex: + with open(expected, 'w') as ex: json.dump(result, ex, indent=2, separators=(',', ':')) expected_content = result else: - with open(expected, 'rb') as ex: - expected_content = json.load(ex, encoding='utf-8', object_pairs_hook=OrderedDict) + with open(expected) as ex: + expected_content = json.load(ex) else: expected_content = expected @@ -160,7 +145,7 @@ def to_posix(path): the windows explorer (except as a UNC or share name). It will be a valid path everywhere in Python. It will not be valid for windows command line operations. """ - is_unicode = isinstance(path, compat.unicode) + is_unicode = isinstance(path, str) ntpath_sep = is_unicode and u'\\' or '\\' posixpath_sep = is_unicode and u'/' or '/' if is_posixpath(path): diff --git a/tests/extractcode/test_archive.py b/tests/extractcode/test_archive.py index 87c0243..3f313de 100644 --- a/tests/extractcode/test_archive.py +++ b/tests/extractcode/test_archive.py @@ -1,46 +1,34 @@ # -*- coding: utf-8 -*- # -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals import io import os -from unittest.case import skipIf import pytest import commoncode.date -from commoncode import compat from commoncode import fileutils from commoncode.system import on_linux from commoncode.system import on_mac from commoncode.system import on_windows -from commoncode.system import py2 -from commoncode.system import py3 from extractcode_assert_utils import BaseArchiveTestCase from extractcode_assert_utils import check_files @@ -404,7 +392,7 @@ def test_extract_targz_with_mixed_case_and_symlink(self): assert [] == result import json exp_file = self.get_test_loc('archive/tgz/mixed_case_and_symlink.tgz.expected') - with io.open(exp_file, encoding='utf-8') as ef: + with open(exp_file) as ef: expected_files = json.load(ef) check_files(test_dir, list(map(str, expected_files))) @@ -468,18 +456,8 @@ def test_uncompress_concatenated_gzip(self): assert b'f1content\nf2content\n' == open(result, 'rb').read() assert [] == warnings - @pytest.mark.skipif(py3, reason='Fails for now on Python 3') - def test_uncompress_gzip_with_trailing_data_py2(self): - test_file = self.get_test_loc('archive/gzip/trailing_data.gz') - test_dir = self.get_temp_dir() - warnings = archive.uncompress_gzip(test_file, test_dir) - result = os.path.join(test_dir, 'trailing_data.gz-extract') - assert os.path.exists(result) - assert [] == warnings - - @pytest.mark.xfail - @pytest.mark.skipif(py2, reason='Fails for now on Python 3') - def test_uncompress_gzip_with_trailing_data_py3(self): + @pytest.mark.xfail(reason='Fails for now on Python 3') + def test_uncompress_gzip_with_trailing_data(self): test_file = self.get_test_loc('archive/gzip/trailing_data.gz') test_dir = self.get_temp_dir() warnings = archive.uncompress_gzip(test_file, test_dir) @@ -619,13 +597,8 @@ def test_uncompress_bzip2_with_trailing_data(self): def test_uncompress_bzip2_broken(self): test_file = self.get_test_loc('archive/bz2/bz2_not_tarred_broken.bz2') test_dir = self.get_temp_dir() - if py2: - expected = Exception('invalid data stream') - else: - expected = Exception('Invalid data stream') - - self.assertRaisesInstance(expected, archive.uncompress_bzip2, - test_file, test_dir) + expected = Exception('Invalid data stream') + self.assertRaisesInstance(expected, archive.uncompress_bzip2, test_file, test_dir) def test_uncompress_bzip2_with_invalid_path(self): test_file = self.get_test_loc('archive/bz2/bz_invalidpath.bz2') @@ -650,7 +623,7 @@ def test_sevenzip_extract_can_handle_bz2_multistream_differently(self): test_dir = self.get_temp_dir() sevenzip.extract(test_file, test_dir) expected = self.get_test_loc('archive/bz2/bzip2_multistream/expected.csv') - # the extraction dir is not created with suffix by z7 + # the extraction dir is not created with suffix by 7z result = os.path.join(test_dir, 'example-file.csv') expected_extracted = open(expected, 'rb').read() expected_result = open(result, 'rb').read() @@ -859,19 +832,7 @@ def test_extract_zip_with_relative_path_deeply_nested(self): except: assert self.expected_deeply_nested_relative_path_alternative == result - @pytest.mark.xfail - @pytest.mark.skipif(on_windows or py3, reason='Expectations are different on Windows') - def test_extract_zip_with_relative_path_deeply_nested_with_7zip_posix_py2(self): - test_file = self.get_test_loc('archive/zip/relative_nested.zip') - test_dir = self.get_temp_dir() - try: - sevenzip.extract(test_file, test_dir) - self.fail('Should raise an exception') - except ExtractErrorFailedToExtract as e: - assert 'Unknown extraction error' == str(e) - - @pytest.mark.xfail - @pytest.mark.skipif(on_windows or py2, reason='Expectations are different on Windows') + @pytest.mark.xfail(reason='Expectations are different on Windows and this may fail on Windows') def test_extract_zip_with_relative_path_deeply_nested_with_7zip_posix_py3(self): test_file = self.get_test_loc('archive/zip/relative_nested.zip') test_dir = self.get_temp_dir() @@ -1199,12 +1160,7 @@ def test_extract_python_testtar_tar_archive_with_special_files(self): # https://hg.python.org/cpython/raw-file/bff88c866886/Lib/test/testtar.tar test_dir = self.get_temp_dir() result = archive.extract_tar(test_file, test_dir) - if py2: - expected_warnings = [ - "'pax/bad-pax-\\xe4\\xf6\\xfc': \nPathname can't be converted from UTF-8 to current locale."] - else: - expected_warnings = [ - u"'pax/bad-pax-äöü': \nPathname can't be converted from UTF-8 to current locale."] + expected_warnings = [u"'pax/bad-pax-äöü': \nPathname can't be converted from UTF-8 to current locale."] assert sorted(expected_warnings) == sorted(result) @@ -1240,8 +1196,6 @@ def test_extract_python_testtar_tar_archive_with_special_files(self): 'ustar/sparse', 'ustar/umlauts-AOUaouss' ] - if on_linux and py2: - expected = [bytes(e) for e in expected] check_files(test_dir, expected) def test_extract_rubygem(self): @@ -1249,8 +1203,6 @@ def test_extract_rubygem(self): test_dir = self.get_temp_dir() archive.extract_tar(test_file, test_dir) expected = ['checksums.yaml.gz', 'data.tar.gz', 'metadata.gz'] - if on_linux and py2: - expected = [bytes(e) for e in expected] check_files(test_dir, expected) @@ -1384,10 +1336,6 @@ def test_extract_ar_with_relative_path_and_backslashes_in_names_libarch(self): # 7zip is better, but has a security bug for now # GNU ar works fine otherwise, but there are portability issues expected = ['dot', 'dot_1'] - - if on_linux and py2: - expected = [bytes(e) for e in expected] - check_files(test_dir, expected) def test_extract_ar_with_relative_path_and_backslashes_in_names_7z(self): @@ -1515,9 +1463,6 @@ def test_extract_cpio_broken2(self): test_dir = self.get_temp_dir() result = archive.extract_cpio(test_file, test_dir) expected = sorted(['elfinfo-1.0.tar.gz', 'elfinfo.spec']) - if on_linux and py2: - expected = [e.encode('utf-8') for e in expected] - assert expected == sorted(os.listdir(test_dir)) assert ["'elfinfo.spec': \nSkipped 72 bytes before finding valid header"] == result @@ -1785,13 +1730,12 @@ def test_extract_rar_with_password(self): test_file = self.get_test_loc('archive/rar/rar_password.rar') test_dir = self.get_temp_dir() expected = Exception('Prefix found') - self.assertRaisesInstance(expected, archive.extract_rar, - test_file, test_dir) + self.assertRaisesInstance(expected, archive.extract_rar, test_file, test_dir) def test_extract_rar_with_non_ascii_path(self): test_file = self.get_test_loc('archive/rar/non_ascii_corrupted.rar') # The bug only occurs if the path was given as Unicode - test_file = compat.unicode(test_file) + test_file = str(test_file) test_dir = self.get_temp_dir() # raise an exception but still extracts some expected = Exception('Prefix found') @@ -1999,16 +1943,15 @@ def test_extract_dia_basic(self): result = os.path.join(test_dir, 'dia.dia-extract') assert os.path.exists(result) - @pytest.mark.skipif(py3, reason='Fails for now on Python 3') - def test_extract_dia_with_trailing_data_py2(self): + @pytest.mark.xfail(reason='Fails for now on Python 3') + def test_extract_dia_with_trailing_data(self): test_file = self.get_test_loc('archive/dia/dia_trailing.dia') test_dir = self.get_temp_dir() archive.uncompress_gzip(test_file, test_dir) result = os.path.join(test_dir, 'dia_trailing.dia-extract') assert os.path.exists(result) - @pytest.mark.xfail - @pytest.mark.skipif(py2, reason='Fails for now on Python 3') + @pytest.mark.xfail(reason='Fails for now on Python 3') def test_extract_dia_with_trailing_data_py3(self): test_file = self.get_test_loc('archive/dia/dia_trailing.dia') test_dir = self.get_temp_dir() @@ -2278,8 +2221,8 @@ def check_extract_weird_names( listed in the `test_file.excepted` file exist in the extracted target directory. Regen expected file if True. """ - if not isinstance(test_file, compat.unicode): - test_file = compat.unicode(test_file) + if not isinstance(test_file, str): + test_file = str(test_file) test_file = self.get_test_loc(test_file) test_dir = self.get_temp_dir() @@ -2299,7 +2242,7 @@ def check_extract_weird_names( len_test_dir = len(test_dir) extracted = sorted(path[len_test_dir:] for path in fileutils.resource_iter(test_dir, with_dirs=False)) - extracted = [compat.unicode(p) for p in extracted] + extracted = [str(p) for p in extracted] extracted = [to_posix(p) for p in extracted] if on_linux: @@ -2312,11 +2255,7 @@ def check_extract_weird_names( expected_file = test_file + '_' + expected_suffix + '_' + os_suffix + '.expected' import json if regen: - if py2: - wmode = 'wb' - if py3: - wmode = 'w' - with open(expected_file, wmode) as ef: + with open(expected_file, 'w') as ef: ef.write(json.dumps(extracted, indent=2)) expected = json.loads(open(expected_file).read()) @@ -2677,57 +2616,23 @@ def test_extract_zip_with_weird_filenames_with_sevenzip_win(self): @pytest.mark.skipif(not on_windows, reason='Run only on Windows because of specific test expectations.') class TestExtractArchiveWithIllegalFilenamesWithSevenzipOnWinWarning(ExtractArchiveWithIllegalFilenamesTestCase): - if py2: - - # The results are not correct but not a problem: we use libarchive for these - @pytest.mark.xfail - def test_extract_7zip_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.7z') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) - - else: - - def test_extract_7zip_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.7z') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) - - if py2: - - @pytest.mark.xfail # not a problem: we use libarchive for these - def test_extract_ar_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.ar') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) - - else: - - def test_extract_ar_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.ar') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) - - if py2: - - @pytest.mark.xfail # not a problem: we use libarchive for these - def test_extract_cpio_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.cpio') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) + def test_extract_7zip_with_weird_filenames_with_sevenzip_win(self): + test_file = self.get_test_loc('archive/weird_names/weird_names.7z') + self.check_extract_weird_names( + sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', + check_warnings=True, check_only_warnings=True) - else: + def test_extract_ar_with_weird_filenames_with_sevenzip_win(self): + test_file = self.get_test_loc('archive/weird_names/weird_names.ar') + self.check_extract_weird_names( + sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', + check_warnings=True, check_only_warnings=True) - def test_extract_cpio_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.cpio') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) + def test_extract_cpio_with_weird_filenames_with_sevenzip_win(self): + test_file = self.get_test_loc('archive/weird_names/weird_names.cpio') + self.check_extract_weird_names( + sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', + check_warnings=True, check_only_warnings=True) def test_extract_iso_with_weird_filenames_with_sevenzip_win(self): test_file = self.get_test_loc('archive/weird_names/weird_names.iso') @@ -2748,22 +2653,11 @@ def test_extract_tar_with_weird_filenames_with_sevenzip_win(self): sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', check_warnings=True, check_only_warnings=True) - if py2: - - @pytest.mark.xfail # not a problem: we use libarchive for these - def test_extract_zip_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.zip') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) - - else: - - def test_extract_zip_with_weird_filenames_with_sevenzip_win(self): - test_file = self.get_test_loc('archive/weird_names/weird_names.zip') - self.check_extract_weird_names( - sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', - check_warnings=True, check_only_warnings=True) + def test_extract_zip_with_weird_filenames_with_sevenzip_win(self): + test_file = self.get_test_loc('archive/weird_names/weird_names.zip') + self.check_extract_weird_names( + sevenzip.extract, test_file, expected_warnings=[], expected_suffix='7zip', + check_warnings=True, check_only_warnings=True) class TestZipSlip(BaseArchiveTestCase): diff --git a/tests/extractcode/test_extract.py b/tests/extractcode/test_extract.py index 6629710..b6f89e9 100644 --- a/tests/extractcode/test_extract.py +++ b/tests/extractcode/test_extract.py @@ -1,29 +1,23 @@ # -*- coding: utf-8 -*- # -# Copyright (c) 2015 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import, print_function import io import os @@ -35,7 +29,6 @@ from commoncode.fileutils import as_posixpath from commoncode.system import on_linux from commoncode.system import on_windows -from commoncode.system import py3 from commoncode.testcase import FileBasedTesting import extractcode @@ -386,7 +379,6 @@ def test_extract_tree_shallow_then_recursive(self): def test_uncompress_corrupted_archive_with_zlib(self): from extractcode import archive - import zlib test_file = self.get_test_loc('extract/corrupted/a.tar.gz', copy=True) test_dir = self.get_temp_dir() expected = Exception('Error -3 while decompressing') @@ -399,7 +391,7 @@ def test_uncompress_corrupted_archive_with_libarchive(self): expected = Exception('gzip decompression failed') self.assertRaisesInstance(expected, libarchive2.extract, test_file, test_dir) - @pytest.mark.skipif(py3 and not on_linux, reason='Expectations are different on Windows and macOS') + @pytest.mark.skipif(not on_linux, reason='Expectations are different on Windows and macOS') def test_extract_tree_with_corrupted_archives_linux(self): expected = ( 'a.tar.gz', @@ -413,7 +405,7 @@ def test_extract_tree_with_corrupted_archives_linux(self): assert result.errors[0].startswith('gzip decompression failed') assert not result.warnings - @pytest.mark.skipif(py3 and on_linux, reason='Expectations are different on Windows and macOS') + @pytest.mark.skipif(on_linux, reason='Expectations are different on Windows and macOS') def test_extract_tree_with_corrupted_archives_mac_win(self): expected = ( 'a.tar.gz', @@ -857,7 +849,7 @@ def test_extract_always_returns_a_generator_and_not_a_list(self): test_dir = self.get_test_loc('extract/generator', copy=True) result = extract.extract(test_dir) assert isinstance(result, GeneratorType) - + def test_extract_ignore_file(self): test_dir = self.get_test_loc('extract/ignore', copy=True) expected = [ @@ -869,7 +861,6 @@ def test_extract_ignore_file(self): 'gamma/gamma.zip', 'gamma/gamma.zip-extract/c.txt' ] - from extractcode import default_kinds result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('alpha.zip',))) check_no_error(result) check_files(test_dir, expected) @@ -889,7 +880,6 @@ def test_extract_ignore_directory(self): 'beta.tar-extract/c.txt', 'gamma/gamma.zip', ] - from extractcode import default_kinds result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('gamma',))) check_no_error(result) check_files(test_dir, expected) @@ -909,7 +899,6 @@ def test_extract_ignore_pattern(self): 'gamma/gamma.zip', 'gamma/gamma.zip-extract/c.txt' ] - from extractcode import default_kinds result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('b*.zip',))) check_no_error(result) - check_files(test_dir, expected) \ No newline at end of file + check_files(test_dir, expected) diff --git a/tests/extractcode/test_extractcode.py b/tests/extractcode/test_extractcode.py index 3ed0a7d..0a7dafb 100644 --- a/tests/extractcode/test_extractcode.py +++ b/tests/extractcode/test_extractcode.py @@ -1,29 +1,23 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function from os.path import dirname from os.path import exists diff --git a/tests/extractcode/test_extractcode_cli.py b/tests/extractcode/test_extractcode_cli.py index 19e621a..bad30c9 100644 --- a/tests/extractcode/test_extractcode_cli.py +++ b/tests/extractcode/test_extractcode_cli.py @@ -1,31 +1,23 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals import os @@ -34,14 +26,12 @@ import pytest from commoncode.fileutils import as_posixpath -from commoncode.fileutils import fsencode from commoncode.fileutils import resource_iter from commoncode.testcase import FileDrivenTesting from commoncode.system import on_linux from commoncode.system import on_windows -from commoncode.system import py2 -from extractcode import cli +from extractcode import cli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data') @@ -51,8 +41,6 @@ the actual command outputs as if using a TTY or not. """ -EMPTY_STRING = b'' if on_linux and py2 else '' - def test_extractcode_command_can_take_an_empty_directory(monkeypatch): test_dir = test_env.get_temp_dir() @@ -186,19 +174,16 @@ def test_usage_and_help_return_a_correct_script_name_on_all_platforms(monkeypatc def test_extractcode_command_can_extract_archive_with_unicode_names_verbose(monkeypatch): monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) test_dir = test_env.get_test_loc('cli/unicodearch', copy=True) - if on_linux and py2: - test_dir = fsencode(test_dir) runner = CliRunner() result = runner.invoke(cli.extractcode, ['--verbose', test_dir]) assert result.exit_code == 0 assert 'Sanders' in result.output - uni_arch = b'unicodepath.tgz' if on_linux and py2 else u'unicodepath.tgz' - uni_path = b'/unicodepath/' if on_linux and py2 else u'/unicodepath/' - - file_result = [f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) if not f.endswith(uni_arch)] - file_result = [EMPTY_STRING.join(f.partition(uni_path)[1:]) for f in file_result] + file_result = [ + f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) + if not f.endswith('unicodepath.tgz')] + file_result = [''.join(f.partition('/unicodepath/')[1:]) for f in file_result] file_result = [f for f in file_result if f] expected = [ '/unicodepath/Ho_', @@ -211,17 +196,14 @@ def test_extractcode_command_can_extract_archive_with_unicode_names_verbose(monk def test_extractcode_command_can_extract_archive_with_unicode_names(monkeypatch): monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) test_dir = test_env.get_test_loc('cli/unicodearch', copy=True) - if on_linux: - test_dir = fsencode(test_dir) runner = CliRunner() result = runner.invoke(cli.extractcode, [test_dir]) assert result.exit_code == 0 - uni_arch = b'unicodepath.tgz' if on_linux and py2 else 'unicodepath.tgz' - uni_path = b'/unicodepath/' if on_linux and py2 else '/unicodepath/' - - file_result = [f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) if not f.endswith(uni_arch)] - file_result = [EMPTY_STRING.join(f.partition(uni_path)[1:]) for f in file_result] + file_result = [ + f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) + if not f.endswith('unicodepath.tgz')] + file_result = [''.join(f.partition('/unicodepath/')[1:]) for f in file_result] file_result = [f for f in file_result if f] expected = [ '/unicodepath/Ho_', @@ -237,7 +219,9 @@ def test_extractcode_command_can_extract_shallow(monkeypatch): runner = CliRunner() result = runner.invoke(cli.extractcode, ['--shallow', test_dir]) assert result.exit_code == 0 - file_result = [f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) if not f.endswith('unicodepath.tgz')] + file_result = [ + f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) + if not f.endswith('unicodepath.tgz')] file_result = [''.join(f.partition('/top.zip-extract/')[1:]) for f in file_result] file_result = [f for f in file_result if f] # this checks that the zip in top.zip are not extracted @@ -248,17 +232,18 @@ def test_extractcode_command_can_extract_shallow(monkeypatch): ] assert sorted(expected) == sorted(file_result) + def test_extractcode_command_can_ignore(monkeypatch): monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) test_dir = test_env.get_test_loc('cli/extract_ignore', copy=True) - if on_linux: - test_dir = fsencode(test_dir) runner = CliRunner() result = runner.invoke(cli.extractcode, ['--ignore', '*.tar', test_dir]) assert result.exit_code == 0 - file_result = [f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) if not f.endswith('a.tar') or not f.endswith('b.tar')] - file_result = [EMPTY_STRING.join(f.partition('/a.zip-extract/')[1:]) for f in file_result] + file_result = [ + f for f in map(as_posixpath, resource_iter(test_dir, with_dirs=False)) + if not f.endswith('a.tar') or not f.endswith('b.tar')] + file_result = [''.join(f.partition('/a.zip-extract/')[1:]) for f in file_result] file_result = [f for f in file_result if f] expected = [ '/a.zip-extract/a.txt', @@ -268,6 +253,7 @@ def test_extractcode_command_can_ignore(monkeypatch): ] assert sorted(expected) == sorted(file_result) + @pytest.mark.skipif(on_windows, reason='FIXME: this test fails on Windows until we have support for long file names.') def test_extractcode_command_can_extract_nuget(monkeypatch): test_dir = test_env.get_test_loc('cli/extract_nuget', copy=True) @@ -276,4 +262,4 @@ def test_extractcode_command_can_extract_nuget(monkeypatch): result = runner.invoke(cli.extractcode, ['--verbose', test_dir], catch_exceptions=False) if result.exit_code != 0: print(result.output) - assert 'ERROR extracting' not in result.output \ No newline at end of file + assert 'ERROR extracting' not in result.output diff --git a/tests/extractcode/test_libarchive2.py b/tests/extractcode/test_libarchive2.py index 9980f5f..21f0f57 100644 --- a/tests/extractcode/test_libarchive2.py +++ b/tests/extractcode/test_libarchive2.py @@ -1,30 +1,23 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from commoncode import fileutils diff --git a/tests/extractcode/test_patch.py b/tests/extractcode/test_patch.py index d396a54..6ecb3cc 100644 --- a/tests/extractcode/test_patch.py +++ b/tests/extractcode/test_patch.py @@ -1,40 +1,32 @@ + +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals import io import json import os from unittest.case import expectedFailure -from commoncode.system import py2 -from commoncode.system import py3 from commoncode.testcase import FileBasedTesting from commoncode.text import as_unicode + from extractcode import patch @@ -64,14 +56,9 @@ def check_patch(test_file, expected_file, regen=False): for s, t, lines in result] if regen: - if py2: - wmode = 'wb' - if py3: - wmode = 'w' - - with io.open(expected_file, wmode) as regened: + with io.open(expected_file, 'w') as regened: json.dump(result, regened, indent=2) - with io.open(expected_file, encoding='utf-8') as expect: + with open(expected_file) as expect: expected = json.load(expect) assert expected == result diff --git a/tests/extractcode/test_sevenzip.py b/tests/extractcode/test_sevenzip.py index a5dd2bf..e2ee3dc 100644 --- a/tests/extractcode/test_sevenzip.py +++ b/tests/extractcode/test_sevenzip.py @@ -1,29 +1,22 @@ # -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. # ScanCode is a trademark of nexB Inc. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function import os import json @@ -31,12 +24,11 @@ from unittest.case import skipIf from commoncode.testcase import FileBasedTesting -from commoncode.system import py2 -from commoncode.system import py3 from commoncode.system import on_windows from commoncode import fileutils -from extractcode import sevenzip + from extractcode import ExtractErrorFailedToExtract +from extractcode import sevenzip class TestSevenZip(FileBasedTesting): @@ -44,15 +36,11 @@ class TestSevenZip(FileBasedTesting): def check_results_with_expected_json(self, results, expected_loc, clean_dates=False, regen=False): if regen: - if py2: - wmode = 'wb' - if py3: - wmode = 'w' - with open(expected_loc, wmode) as ex: + with open(expected_loc, 'w') as ex: json.dump(results, ex, indent=2, separators=(',', ':')) - with open(expected_loc, 'rb') as ex: - expected = json.load(ex, encoding='utf-8') + with open(expected_loc) as ex: + expected = json.load(ex) if clean_dates: if isinstance(results, list): self.clean_dates(results) From bd366355edc4d2e6e70d32eb81041c62fb7d4abe Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 12:55:33 +0100 Subject: [PATCH 32/34] Update CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 11413dc..d9183fc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,17 @@ Release notes vNext ----- +Version 21.1.15 +------------- + +*2021-01-15* +- Drop support for Python 2 +- Use the latest CommonCode and TypeCode libraries + +*2020-11-13* +- Add azure-pipelines CI support + + Version 20.10 ------------- From 31b3a432c1354cac75b3af81b9ba85a8cb57d339 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 13:16:03 +0100 Subject: [PATCH 33/34] Improve test expected failures Signed-off-by: Philippe Ombredanne --- tests/extractcode/test_archive.py | 15 +++++++-------- tests/extractcode/test_sevenzip.py | 14 ++++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/extractcode/test_archive.py b/tests/extractcode/test_archive.py index 3f313de..48f2a04 100644 --- a/tests/extractcode/test_archive.py +++ b/tests/extractcode/test_archive.py @@ -19,16 +19,16 @@ # limitations under the License. # -import io import os import pytest -import commoncode.date +from commoncode import date as commoncode_date from commoncode import fileutils from commoncode.system import on_linux from commoncode.system import on_mac from commoncode.system import on_windows +from commoncode.testcase import is_same from extractcode_assert_utils import BaseArchiveTestCase from extractcode_assert_utils import check_files @@ -40,7 +40,6 @@ from extractcode import ExtractErrorFailedToExtract from extractcode import libarchive2 from extractcode import sevenzip -from extractcode.libarchive2 import ArchiveError """ For each archive type --when possible-- we are testing extraction of: @@ -383,7 +382,7 @@ def test_extract_targz_with_trailing_data2(self): test_dir2 = self.get_temp_dir() test_file2 = self.get_test_loc('archive/tgz/no_trailing.tar.gz') archive.extract_tar(test_file2, test_dir2) - assert commoncode.testcase.is_same(test_dir1, test_dir2) + assert is_same(test_dir1, test_dir2) def test_extract_targz_with_mixed_case_and_symlink(self): test_file = self.get_test_loc('archive/tgz/mixed_case_and_symlink.tgz') @@ -960,7 +959,7 @@ def test_extract_zip_with_timezone(self): ] # DST sends a monkey wrench.... so we only test the date, not the time for loc, expected_date in expected: - result = commoncode.date.get_file_mtime(loc) + result = commoncode_date.get_file_mtime(loc) assert result.startswith(expected_date) def test_extract_zip_with_timezone_2(self): @@ -974,7 +973,7 @@ def test_extract_zip_with_timezone_2(self): (os.path.join(test_dir, 'primes2.txt'), ('2009-12-05', '2009-12-06',)) ] for loc, expected_date in expected: - result = commoncode.date.get_file_mtime(loc) + result = commoncode_date.get_file_mtime(loc) assert result.startswith(expected_date) def test_extract_zip_with_backslash_in_path_1(self): @@ -1268,7 +1267,7 @@ def test_extract_ar_verify_dates(self): ] # DST sends a monkey wrench.... so we only test the date, not the time for loc, expected_date in expected: - result = commoncode.date.get_file_mtime(loc) + result = commoncode_date.get_file_mtime(loc) assert result.startswith(expected_date) def test_extract_ar_broken_7z(self): @@ -2318,7 +2317,7 @@ def test_extract_ar_with_weird_filenames_with_libarchive_win(self): self.check_extract_weird_names( libarchive2.extract, test_file, expected_warnings=[], expected_suffix='libarch') self.fail('Exception not raised.') - except ArchiveError as ae: + except libarchive2.ArchiveError as ae: assert str(ae).startswith('Incorrect file header signature') def test_extract_cpio_with_weird_filenames_with_libarchive_win(self): diff --git a/tests/extractcode/test_sevenzip.py b/tests/extractcode/test_sevenzip.py index e2ee3dc..ee32439 100644 --- a/tests/extractcode/test_sevenzip.py +++ b/tests/extractcode/test_sevenzip.py @@ -21,11 +21,12 @@ import os import json import posixpath -from unittest.case import skipIf +import pytest + +from commoncode import fileutils from commoncode.testcase import FileBasedTesting from commoncode.system import on_windows -from commoncode import fileutils from extractcode import ExtractErrorFailedToExtract from extractcode import sevenzip @@ -135,7 +136,7 @@ def test_extract_of_tar_with_aboslute_path(self): class TestSevenZipListEntries(TestSevenZip): - @skipIf(on_windows, 'Windows file-by-file extracton is not working well') + @pytest.mark.skipif(on_windows, reason='Windows file-by-file extracton is not working well') def test_list_entries_of_special_tar(self): test_loc = self.get_test_loc('sevenzip/special.tar') expected_loc = test_loc + '-entries-expected.json' @@ -145,7 +146,7 @@ def test_list_entries_of_special_tar(self): results = entries + errors self.check_results_with_expected_json(results, expected_loc, regen=False) - @skipIf(not on_windows, 'Windows file-by-file extracton is not working well') + @pytest.mark.skipif(not on_windows, reason='Windows file-by-file extracton is not working well') def test_list_entries_of_special_tar_win(self): test_loc = self.get_test_loc('sevenzip/special.tar') expected_loc = test_loc + '-entries-expected-win.json' @@ -155,7 +156,7 @@ def test_list_entries_of_special_tar_win(self): results = entries + errors self.check_results_with_expected_json(results, expected_loc, clean_dates=True, regen=False) - @skipIf(on_windows, 'Windows file-by-file extracton is not working well') + @pytest.mark.skipif(on_windows, reason='Windows file-by-file extracton is not working well') def test_list_entries_with_weird_names_7z(self): test_loc = self.get_test_loc('sevenzip/weird_names.7z') expected_loc = test_loc + '-entries-expected.json' @@ -165,7 +166,7 @@ def test_list_entries_with_weird_names_7z(self): results = entries + errors self.check_results_with_expected_json(results, expected_loc, regen=False) - @skipIf(not on_windows, 'Windows file-by-file extracton is not working well') + @pytest.mark.skipif(not on_windows, reason='Windows file-by-file extracton is not working well') def test_list_entries_with_weird_names_7z_win(self): test_loc = self.get_test_loc('sevenzip/weird_names.7z') expected_loc = test_loc + '-entries-expected-win.json' @@ -288,6 +289,7 @@ def test_extract_file_by_file_with_weird_names_7z(self): def test_extract_file_by_file_weird_names_zip(self): self.check_extract_file_by_file('sevenzip/weird_names.zip', regen=False) + @pytest.mark.xfail(on_windows, reason='Fails on Windows becasue it has file names that cannot be extracted there') def test_extract_file_by_file_weird_names_ar(self): self.check_extract_file_by_file('sevenzip/weird_names.ar', regen=False) From fa463129a56ea04abfab318fe4d88e992758dd09 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Jan 2021 13:33:28 +0100 Subject: [PATCH 34/34] Format source code Signed-off-by: Philippe Ombredanne --- src/extractcode/api.py | 1 - src/extractcode/archive.py | 1 - src/extractcode/cli.py | 2 +- tests/extractcode/test_libarchive2.py | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/extractcode/api.py b/src/extractcode/api.py index 4842785..a5bb86c 100644 --- a/src/extractcode/api.py +++ b/src/extractcode/api.py @@ -18,7 +18,6 @@ # limitations under the License. # - """ Note: this API is unstable and still evolving. """ diff --git a/src/extractcode/archive.py b/src/extractcode/archive.py index a088929..3946bb7 100644 --- a/src/extractcode/archive.py +++ b/src/extractcode/archive.py @@ -45,7 +45,6 @@ from extractcode.uncompress import uncompress_gzip from extractcode.uncompress import uncompress_bzip2 - logger = logging.getLogger(__name__) TRACE = False TRACE_DEEP = False diff --git a/src/extractcode/cli.py b/src/extractcode/cli.py index 2903c41..5591e5d 100644 --- a/src/extractcode/cli.py +++ b/src/extractcode/cli.py @@ -53,7 +53,7 @@ def print_version(ctx, param, value): notice_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'NOTICE') notice_text = open(notice_path).read() - + def print_about(ctx, param, value): """ diff --git a/tests/extractcode/test_libarchive2.py b/tests/extractcode/test_libarchive2.py index 21f0f57..783ac39 100644 --- a/tests/extractcode/test_libarchive2.py +++ b/tests/extractcode/test_libarchive2.py @@ -24,11 +24,11 @@ from extractcode_assert_utils import check_files from extractcode_assert_utils import BaseArchiveTestCase - """ Minimal smoke tests for libarchive2. """ + class TestExtractorTest(BaseArchiveTestCase): def test_libarchive_extract_can_extract_to_relative_paths(self):