diff --git a/.bzrignore b/.bzrignore deleted file mode 100644 index ceb3376..0000000 --- a/.bzrignore +++ /dev/null @@ -1,9 +0,0 @@ -*.egg-info -./build -./dist -.coverage -__pycache__ -.tox -coverage.xml -nosetests.xml -docs/_build diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..96c89ce --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Ignore all Git auto CR/LF line endings conversions +* -text +pyproject.toml export-subst diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 0000000..8d8aa55 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,32 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-24.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.13] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Dependencies + run: ./configure --dev + + - name: Check documentation and HTML for errors and dead links + run: make docs-check + + - name: Check documentation for style errors + run: make doc8 + + diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 0000000..7da0a40 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,87 @@ +name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch + + +# This is executed automatically on a tag in the main branch + +# Summary of the steps: +# - build wheels and sdist +# - upload wheels and sdist to PyPI +# - create gh-release and upload wheels and dists there +# TODO: smoke test wheels and sdist +# TODO: add changelog to release text body + +# WARNING: this is designed only for packages building as pure Python wheels + +on: + workflow_dispatch: + push: + tags: + - "v*.*.*" + +jobs: + build-pypi-distribs: + name: Build and publish library to PyPI + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install pypa/build and twine + run: python -m pip install --user --upgrade build twine pkginfo + + - name: Build a binary wheel and a source tarball + run: python -m build --wheel --sdist --outdir dist/ + + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + + - name: Upload built archives + uses: actions/upload-artifact@v4 + with: + name: pypi_archives + path: dist/* + + + create-gh-release: + name: Create GH release + needs: + - build-pypi-distribs + runs-on: ubuntu-24.04 + + steps: + - name: Download built archives + uses: actions/download-artifact@v4 + with: + name: pypi_archives + path: dist + + - name: Create GH release + uses: softprops/action-gh-release@v2 + with: + draft: true + files: dist/* + + + create-pypi-release: + name: Create PyPI release + needs: + - create-gh-release + runs-on: ubuntu-24.04 + environment: pypi-publish + permissions: + id-token: write + + steps: + - name: Download built archives + uses: actions/download-artifact@v4 + with: + name: pypi_archives + path: dist + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index c22fc0a..68a909e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,76 @@ +# Python compiled files *.py[cod] + +# virtualenv and other misc bits +/src/*.egg-info *.egg-info -./build -./dist +/dist +/build +/bin +/lib +/scripts +/Scripts +/Lib +/pip-selfcheck.json +/tmp +/venv +.Python +/include +/Include +/local +*/local/* +/local/ +/share/ +/tcl/ +/.eggs/ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.cache .coverage -__pycache__ -.tox +.coverage.* +nosetests.xml +htmlcov + +# Translations +*.mo + +# IDEs +.project +.pydevproject +.idea +org.eclipse.core.resources.prefs +.vscode +.vs + +# Sphinx docs/_build -/venv/ -dist -build +docs/bin +docs/build +docs/include +docs/Lib +doc/pyvenv.cfg +pyvenv.cfg + +# Various junk and temp files +.DS_Store +*~ +.*.sw[po] +.build +.ve +*.bak +/.cache/ + +# pyenv +/.python-version +/man/ +/.pytest_cache/ +lib64 +tcl + +# Ignore Jupyter Notebook related temp files +.ipynb_checkpoints/ +/.ruff_cache/ +.env diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..27c1595 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,29 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build in latest ubuntu/python +build: + os: ubuntu-22.04 + tools: + python: "3.13" + +# Build PDF & ePub +formats: + - epub + - pdf + +# Where the Sphinx conf.py file is located +sphinx: + configuration: docs/source/conf.py + +# Setting the python version and doc build requirements +python: + install: + - method: pip + path: . + extra_requirements: + - dev diff --git a/CHANGES.txt b/CHANGELOG.rst similarity index 97% rename from CHANGES.txt rename to CHANGELOG.rst index bdb34a9..3a4a929 100644 --- a/CHANGES.txt +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ ``pkginfo2`` Changelog ======================= +30.1.0 (2025-08-27) +-------------------- + +- Fixes entrypoint module name typo +- Added aboutcode-org/skeleton files and restructured modules +- Enabled tests in Github actions/azure pipelines + 30.0.0 (2022-01-28) -------------------- @@ -213,7 +220,7 @@ - Fix bug in introspection of installed packages missing the ``__package__`` attribute. - + 0.7 (2010-11-04) ---------------- diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 0000000..590ba19 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,86 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, gender identity and +expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies both within project spaces and in public +spaces when an individual is representing the project or its community. +Examples of representing a project or community include using an +official project e-mail address, posting via an official social media +account, or acting as an appointed representative at an online or +offline event. Representation of a project may be further defined and +clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at pombredanne@gmail.com +or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . +All complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project’s leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_ , +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org diff --git a/MANIFEST.in b/MANIFEST.in index 6788bfc..0f19707 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,25 @@ -graft docs/examples/ -CHANGES.txt +graft src +graft docs +graft etc + +include *.LICENSE +include NOTICE +include *.ABOUT +include *.toml +include *.yml +include *.rst +include *.png +include setup.* +include configure* +include requirements* +include .dockerignore +include .gitignore +include .readthedocs.yml +include manage.py +include Dockerfile* +include Makefile +include MANIFEST.in + +include .VERSION + +global-exclude *.py[co] __pycache__ *.*~ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..44fcba5 --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +# Python version can be specified with `$ PYTHON_EXE=python3.x make conf` +PYTHON_EXE?=python3 +VENV=venv +ACTIVATE?=. ${VENV}/bin/activate; + + +conf: + @echo "-> Install dependencies" + ./configure + +dev: + @echo "-> Configure and install development dependencies" + ./configure --dev + +doc8: + @echo "-> Run doc8 validation" + @${ACTIVATE} doc8 --quiet docs/ *.rst + +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix + +check: + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ + +clean: + @echo "-> Clean the Python env" + ./configure --clean + +test: + @echo "-> Run the test suite" + ${VENV}/bin/pytest -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + +docs: + rm -rf docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ + +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/README.txt b/README.rst similarity index 99% rename from README.txt rename to README.rst index 5b98503..ebb9987 100644 --- a/README.txt +++ b/README.rst @@ -16,6 +16,5 @@ This is a fork of http://bazaar.launchpad.net/~tseaver/pkginfo removing the ability to import and eval arbitrary code and work with modules known to the current interpreter. Use importlib_metadata for this if you need it. - Please see the `pkginfo2 repo at `_ for more documentation. diff --git a/TODO.txt b/TODO.rst similarity index 100% rename from TODO.txt rename to TODO.rst diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..4626572 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,72 @@ + +################################################################################ +# We use Azure to run the full tests suites on multiple Python 3.x +# on multiple Windows, macOS and Linux versions all on 64 bits +# These jobs are using VMs with Azure-provided Python builds +################################################################################ + +jobs: + + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.13'] + test_suites: + all: make check + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython + image_name: macOS-13 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs . --ignore=tests/examples/ --ignore=tests/wonky/ diff --git a/configure b/configure new file mode 100755 index 0000000..6d317d4 --- /dev/null +++ b/configure @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/ for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +set -e +#set -x + +################################ +# A configuration script to set things up: +# create a virtualenv and install or update thirdparty packages. +# Source this script for initial configuration +# Use configure --help for details +# +# NOTE: please keep in sync with Windows script configure.bat +# +# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz +# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +################################ +CLI_ARGS=$1 + +################################ +# Defaults. Change these variables to customize this script +################################ + +# Requirement arguments passed to pip and used by default or with --dev. +REQUIREMENTS="--editable . --constraint requirements.txt" +DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" + +# where we create a virtualenv +VIRTUALENV_DIR=venv + +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" + +# extra arguments passed to pip +PIP_EXTRA_ARGS=" " + +# the URL to download virtualenv.pyz if needed +VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +################################ + + +################################ +# Current directory where this script lives +CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + + +################################ +# Install with or without and index. With "--no-index" this is using only local wheels +# This is an offline mode with no index and no network operations +# NO_INDEX="--no-index " +NO_INDEX="" + + +################################ +# Thirdparty package locations and index handling +# Find packages from the local thirdparty directory if present +THIRDPARDIR=$CFG_ROOT_DIR/thirdparty +if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then + PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR" +fi + + +################################ +# Set the quiet flag to empty if not defined +if [[ "$CFG_QUIET" == "" ]]; then + CFG_QUIET=" " +fi + + +################################ +# Find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi + fi +} + + +################################ +create_virtualenv() { + # create a virtualenv for Python + # Note: we do not use the bundled Python 3 "venv" because its behavior and + # presence is not consistent across Linux distro and sometimes pip is not + # included either by default. The virtualenv.pyz app cures all these issues. + + VENV_DIR="$1" + if [ ! -f "$CFG_BIN_DIR/python" ]; then + + mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" + + if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then + VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" + else + VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + fi + + $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ + --pip embed --setuptools embed \ + --seeder pip \ + --never-download \ + --no-periodic-update \ + --no-vcs-ignore \ + $CFG_QUIET \ + "$CFG_ROOT_DIR/$VENV_DIR" + fi +} + + +################################ +install_packages() { + # install requirements in virtualenv + # note: --no-build-isolation means that pip/wheel/setuptools will not + # be reinstalled a second time and reused from the virtualenv and this + # speeds up the installation. + # We always have the PEP517 build dependencies installed already. + + "$CFG_BIN_DIR/pip" install \ + --upgrade \ + --no-build-isolation \ + $CFG_QUIET \ + $PIP_EXTRA_ARGS \ + $1 +} + + +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + set +e + exit +} + + +################################ +# Main command line entry point +CFG_REQUIREMENTS=$REQUIREMENTS + +# We are using getopts to parse option arguments that start with "-" +while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + esac;; + esac +done + + +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" + +find_python +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" + + +set +e diff --git a/configure.bat b/configure.bat new file mode 100644 index 0000000..15ab701 --- /dev/null +++ b/configure.bat @@ -0,0 +1,203 @@ +@echo OFF +@setlocal + +@rem Copyright (c) nexB Inc. and others. All rights reserved. +@rem SPDX-License-Identifier: Apache-2.0 +@rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +@rem See https://github.com/aboutcode-org/ for support or download. +@rem See https://aboutcode.org for more information about nexB OSS projects. + + +@rem ################################ +@rem # A configuration script to set things up: +@rem # create a virtualenv and install or update thirdparty packages. +@rem # Source this script for initial configuration +@rem # Use configure --help for details + +@rem # NOTE: please keep in sync with POSIX script configure + +@rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz +@rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +@rem ################################ + + +@rem ################################ +@rem # Defaults. Change these variables to customize this script +@rem ################################ + +@rem # Requirement arguments passed to pip and used by default or with --dev. +set "REQUIREMENTS=--editable . --constraint requirements.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" + +@rem # where we create a virtualenv +set "VIRTUALENV_DIR=venv" + +@rem # Cleanable files and directories to delete with the --clean option +set "CLEANABLE=build dist venv .cache .eggs" + +@rem # extra arguments passed to pip +set "PIP_EXTRA_ARGS= " + +@rem # the URL to download virtualenv.pyz if needed +set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +@rem ################################ + + +@rem ################################ +@rem # Current directory where this script lives +set CFG_ROOT_DIR=%~dp0 +set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + + +@rem ################################ +@rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory +if exist "%CFG_ROOT_DIR%\thirdparty" ( + set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" +) + + +@rem ################################ +@rem # Set the quiet flag to empty if not defined +if not defined CFG_QUIET ( + set "CFG_QUIET= " +) + + +@rem ################################ +@rem # Main command line entry point +set "CFG_REQUIREMENTS=%REQUIREMENTS%" + +:again +if not "%1" == "" ( + if "%1" EQU "--help" (goto cli_help) + if "%1" EQU "--clean" (goto clean) + if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + ) + shift + goto again +) + +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" + + +@rem ################################ +@rem # Find a proper Python to run +@rem # Use environment variables or a file if available. +@rem # Otherwise the latest Python by default. +if not defined PYTHON_EXECUTABLE ( + @rem # check for a file named PYTHON_EXECUTABLE + if exist "%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ( + set /p PYTHON_EXECUTABLE=<"%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" + ) else ( + set "PYTHON_EXECUTABLE=py" + ) +) + + +@rem ################################ +:create_virtualenv +@rem # create a virtualenv for Python +@rem # Note: we do not use the bundled Python 3 "venv" because its behavior and +@rem # presence is not consistent across Linux distro and sometimes pip is not +@rem # included either by default. The virtualenv.pyz app cures all these issues. + +if not exist "%CFG_BIN_DIR%\python.exe" ( + if not exist "%CFG_BIN_DIR%" ( + mkdir "%CFG_BIN_DIR%" + ) + + if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ + --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" + ) else ( + if not exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ( + curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% + + if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% + ) + ) + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ + --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" + ) +) + +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) + + +@rem ################################ +:install_packages +@rem # install requirements in virtualenv +@rem # note: --no-build-isolation means that pip/wheel/setuptools will not +@rem # be reinstalled a second time and reused from the virtualenv and this +@rem # speeds up the installation. +@rem # We always have the PEP517 build dependencies installed already. + +"%CFG_BIN_DIR%\pip" install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% + + +@rem ################################ +:create_bin_junction +@rem # Create junction to bin to have the same directory between linux and windows +if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( + rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" +) +mklink /J "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) + +exit /b 0 + + +@rem ################################ +:cli_help + echo An initial configuration script + echo " usage: configure [options]" + echo " " + echo The default is to configure for regular use. Use --dev for development. + echo " " + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo " " + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + exit /b 0 + + +@rem ################################ +:clean +@rem # Remove cleanable file and directories and files from the root dir. +echo "* Cleaning ..." +for %%F in (%CLEANABLE%) do ( + rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 + del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 +) +exit /b 0 diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index c69a453..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,194 +0,0 @@ -# -*- coding: utf-8 -*- -# -# pkginfo documentation build configuration file, created by -# sphinx-quickstart on Wed Apr 8 19:26:04 2009. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -#sys.path.append(os.path.abspath('.')) - -# General configuration -# --------------------- - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', -] - -doctest_path = [os.path.abspath('..')] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'pkginfo' -copyright = u'2009-2013, Tres Seaver' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '1.2' -# The full version, including alpha/beta/rc tags. -release = '1.2' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -#unused_docs = [] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = ['.build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - - -# Options for HTML output -# ----------------------- - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -#html_style = 'default.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_use_modindex = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/. -#html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pkginfodoc' - - -# Options for LaTeX output -# ------------------------ - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, document class [howto/manual]). -latex_documents = [ - ('index', 'pkginfo.tex', u'pkginfo Documentation', u'Tres Seaver', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_use_modindex = True diff --git a/docs/examples/mypackage-0.1/setup.py b/docs/examples/mypackage-0.1/setup.py deleted file mode 100644 index 0f3abda..0000000 --- a/docs/examples/mypackage-0.1/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -from setuptools import setup - -setup( - name='mypackage', - version='0.1', - author='Tres Seaver', - author_email='tseaver@palladion.com', - url='http://pypi.python.org/pypi/pkginfo', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console (Text Based)', - ], -) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..4a3c1a4 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,47 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +if "%SPHINXAUTOBUILD%" == "" ( + set SPHINXAUTOBUILD=sphinx-autobuild +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +if "%1" == "docs" goto docs + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:docs +@echo +@echo Starting up the docs server... +@echo +%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000..5863ccf --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,26 @@ +/* this is the container for the pages */ +.wy-nav-content { + max-width: 100%; + padding: 0px 40px 0px 0px; + margin-top: 0px; +} + +.wy-nav-content-wrap { + border-right: solid 1px; +} + +div.rst-content { + max-width: 1300px; + border: 0; + padding: 10px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } +} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..8dcd65f --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,122 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = "pkginfo2" +copyright = "2009-2013, Tres Seaver" +author = "Tres Seaver, AboutCode.org authors and contributors" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.intersphinx", + "sphinx_reredirects", + "sphinx_rtd_theme", + "sphinx_rtd_dark_mode", + "sphinx.ext.extlinks", + "sphinx_copybutton", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", +] + + +# Redirects for olds pages +# See https://documatt.gitlab.io/sphinx-reredirects/usage.html +redirects = {} + +# This points to aboutcode.readthedocs.io +# In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot +# Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 + +intersphinx_mapping = { + "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), + "scancode-workbench": ( + "https://scancode-workbench.readthedocs.io/en/develop/", + None, + ), +} + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +master_doc = "index" + +html_context = { + "display_github": True, + "github_user": "nexB", + "github_repo": "nexb-skeleton", + "github_version": "develop", # branch + "conf_py_path": "/docs/source/", # path in the checkout to the docs root +} + +html_css_files = [ + "theme_overrides.css", +] + + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = True + +# Define CSS and HTML abbreviations used in .rst files. These are examples. +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` +rst_prolog = """ +.. |psf| replace:: Python Software Foundation + +.. # define a hard line break for HTML +.. |br| raw:: html + +
+ +.. role:: red + +.. role:: img-title + +.. role:: img-title-para + +""" + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = {"classoptions": ",openany,oneside"} +latex_documents = [ + ("index", "pkginfo.tex", "pkginfo Documentation", "Tres Seaver", "manual"), +] diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst new file mode 100644 index 0000000..2a719a5 --- /dev/null +++ b/docs/source/contribute/contrib_doc.rst @@ -0,0 +1,271 @@ +.. _contrib_doc_dev: + +Contributing to the Documentation +================================= + +.. _contrib_doc_setup_local: + +Setup Local Build +----------------- + +To get started, check out and configure the repository for development:: + + git clone https://github.com/aboutcode-org/.git + + cd your-repo + ./configure --dev + +(Or use "make dev") + +.. note:: + + In case of windows, run ``configure --dev``. + +This will install and configure all requirements foer development including for docs development. + +Now you can build the HTML documentation locally:: + + source venv/bin/activate + make docs + +This will build a local instance of the ``docs/_build`` directory:: + + open docs/_build/index.html + + +To validate the documentation style and content, use:: + + source venv/bin/activate + make doc8 + make docs-check + + +.. _doc_ci: + +Continuous Integration +---------------------- + +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: + +1. Successful Builds (By using ``sphinx-build``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) + +You myst run these scripts locally before creating a pull request:: + + make doc8 + make check-docs + + +.. _doc_style_docs8: + +Style Checks Using ``doc8`` +--------------------------- + +How To Run Style Tests +^^^^^^^^^^^^^^^^^^^^^^ + +In the project root, run the following commands:: + + make doc8 + +A sample output is:: + + Scanning... + Validating... + docs/source/misc/licence_policy_plugin.rst:37: D002 Trailing whitespace + docs/source/misc/faq.rst:45: D003 Tabulation used for indentation + docs/source/misc/faq.rst:9: D001 Line too long + docs/source/misc/support.rst:6: D005 No newline at end of file + ======== + Total files scanned = 34 + Total files ignored = 0 + Total accumulated errors = 326 + Detailed error counts: + - CheckCarriageReturn = 0 + - CheckIndentationNoTab = 75 + - CheckMaxLineLength = 190 + - CheckNewlineEndOfFile = 13 + - CheckTrailingWhitespace = 47 + - CheckValidity = 1 + +Now fix the errors and run again till there isn't any style error in the documentation. + + +What is Checked? +^^^^^^^^^^^^^^^^ + +PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. + +What is checked: + + - invalid rst format - D000 + - lines should not be longer than 100 characters - D001 + + - RST exception: line with no whitespace except in the beginning + - RST exception: lines with http or https URLs + - RST exception: literal blocks + - RST exception: rst target directives + + - no trailing whitespace - D002 + - no tabulation for indentation - D003 + - no carriage returns (use UNIX newlines) - D004 + - no newline at end of file - D005 + + +.. _doc_interspinx: + +Interspinx +---------- + +AboutCode documentation uses +`Intersphinx `_ +to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. + +To link sections in the same documentation, standart reST labels are used. Refer +`Cross-Referencing `_ +for more information. + +For example:: + + .. _my-reference-label: + + Section to cross-reference + -------------------------- + + This is the text of the section. + + It refers to the section itself, see :ref:`my-reference-label`. + +Now, using Intersphinx, you can create these labels in one Sphinx Documentation and then referance +these labels from another Sphinx Documentation, hosted in different locations. + +You just have to add the following in the ``conf.py`` file for your Sphinx Documentation, where you +want to add the links:: + + extensions = [ + 'sphinx.ext.intersphinx' + ] + + intersphinx_mapping = {'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None)} + +To show all Intersphinx links and their targets of an Intersphinx mapping file, run:: + + python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/en/latest/objects.inv + +.. WARNING:: + + ``python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/objects.inv`` will give + error. + +This enables you to create links to the ``aboutcode`` Documentation in your own Documentation, +where you modified the configuration file. Links can be added like this:: + + For more details refer :ref:`aboutcode:doc_style_guide`. + +You can also not use the ``aboutcode`` label assigned to all links from aboutcode.readthedocs.io, +if you don't have a label having the same name in your Sphinx Documentation. Example:: + + For more details refer :ref:`doc_style_guide`. + +If you have a label in your documentation which is also present in the documentation linked by +Intersphinx, and you link to that label, it will create a link to the local label. + +For more information, refer this tutorial named +`Using Intersphinx `_. + + +.. _doc_style_conv: + +Style Conventions for the Documentaion +-------------------------------------- + +1. Headings + + (`Refer `_) + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s Style + Guide for documenting which you may follow: + + # with overline, for parts + + * with overline, for chapters + + =, for sections + + -, for subsections + + ^, for sub-subsections + + ", for paragraphs + +2. Heading Underlines + + Do not use underlines that are longer/shorter than the title headline itself. As in: + + :: + + Correct : + + Extra Style Checks + ------------------ + + Incorrect : + + Extra Style Checks + ------------------------ + +.. note:: + + Underlines shorter than the Title text generates Errors on sphinx-build. + + +3. Internal Links + + Using ``:ref:`` is advised over standard reStructuredText links to sections (like + ```Section title`_``) because it works across files, when section headings are changed, will + raise warnings if incorrect, and works for all builders that support cross-references. + However, external links are created by using the standard ```Section title`_`` method. + +4. Eliminate Redundancy + + If a section/file has to be repeated somewhere else, do not write the exact same section/file + twice. Use ``.. include: ../README.rst`` instead. Here, ``../`` refers to the documentation + root, so file location can be used accordingly. This enables us to link documents from other + upstream folders. + +5. Using ``:ref:`` only when necessary + + Use ``:ref:`` to create internal links only when needed, i.e. it is referenced somewhere. + Do not create references for all the sections and then only reference some of them, because + this created unnecessary references. This also generates ERROR in ``restructuredtext-lint``. + +6. Spelling + + You should check for spelling errors before you push changes. `Aspell `_ + is a GNU project Command Line tool you can use for this purpose. Download and install Aspell, + then execute ``aspell check `` for all the files changed. Be careful about not + changing commands or other stuff as Aspell gives prompts for a lot of them. Also delete the + temporary ``.bak`` files generated. Refer the `manual `_ for more + information on how to use. + +7. Notes and Warning Snippets + + Every ``Note`` and ``Warning`` sections are to be kept in ``rst_snippets/note_snippets/`` and + ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are + frequently used in multiple files. + + +Converting from Markdown +------------------------ + +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. + +This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for +Sphinx/ReadTheDocs hosting. diff --git a/docs/distributions.rst b/docs/source/distributions.rst similarity index 90% rename from docs/distributions.rst rename to docs/source/distributions.rst index dbfae1a..5517001 100644 --- a/docs/distributions.rst +++ b/docs/source/distributions.rst @@ -30,7 +30,7 @@ distutils: .. doctest:: - >>> mypackage = SDist('docs/examples/mypackage-0.1.tar.gz') + >>> mypackage = SDist('tests/examples/mypackage-0.1.tar.gz') After creation, the ``SDist`` instance will have attributes corrsponding the the fields defined in the PEP corresponding to the metadata version, @@ -63,7 +63,7 @@ with no occurences in the ``PKG-INFO`` file will map onto an empty sequence: >>> print(list(mypackage.supported_platforms)) [] -See `Metadata Versions `_ for an example with a non-empty, +See :ref:`metadata-versions` for an example with a non-empty, "multiple-use" field. Introspecting Unpacked Source Distributions @@ -75,10 +75,10 @@ setup.py at the top level: .. doctest:: - >>> mypackage = UnpackedSDist('docs/examples/mypackage-0.1') + >>> mypackage = UnpackedSDist('tests/examples/mypackage-0.1') >>> print(mypackage.name) mypackage - >>> myotherpackage = UnpackedSDist('docs/examples/mypackage-0.1/setup.py') + >>> myotherpackage = UnpackedSDist('tests/examples/mypackage-0.1/setup.py') >>> print(myotherpackage.name) mypackage @@ -110,7 +110,7 @@ generated via ``setup.py bdist_egg``. .. doctest:: - >>> mypackage = BDist('docs/examples/mypackage-0.1-py2.6.egg') + >>> mypackage = BDist('tests/examples/mypackage-0.1-py2.6.egg') After that, they have the same metadata as other ``Distribution`` objects, @@ -122,7 +122,7 @@ generated via ``setup.py bdist_wheel``. .. doctest:: - >>> mypackage = Wheel('docs/examples/mypackage-0.1-cp26-none-linux_x86_64.whl') + >>> mypackage = Wheel('tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl') After that, they have the same metadata as other ``Distribution`` objects, diff --git a/docs/index.rst b/docs/source/index.rst similarity index 90% rename from docs/index.rst rename to docs/source/index.rst index 3db597f..c57d4be 100644 --- a/docs/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ :mod:`pkginfo2` documentation -============================ +============================== This package provides an API for querying the distutils metadata written in the ``PKG-INFO`` file inside a source distriubtion (an ``sdist``) or a @@ -13,6 +13,7 @@ Contents: .. toctree:: :maxdepth: 2 + contribute/contrib_doc distributions metadata indexes diff --git a/docs/indexes.rst b/docs/source/indexes.rst similarity index 100% rename from docs/indexes.rst rename to docs/source/indexes.rst diff --git a/docs/metadata.rst b/docs/source/metadata.rst similarity index 93% rename from docs/metadata.rst rename to docs/source/metadata.rst index 9a46c31..6672a71 100644 --- a/docs/metadata.rst +++ b/docs/source/metadata.rst @@ -1,3 +1,5 @@ +.. _metadata-versions: + Metadata Versions ================= @@ -19,7 +21,7 @@ which were not defined under version '1.0': .. doctest:: >>> from pkginfo import SDist - >>> mypackage = SDist('docs/examples/mypackage-0.1.tar.gz', + >>> mypackage = SDist('tests/examples/mypackage-0.1.tar.gz', ... metadata_version='1.1') >>> print([str(x) for x in mypackage.classifiers]) ['Development Status :: 4 - Beta', 'Environment :: Console (Text Based)'] diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml new file mode 100644 index 0000000..d80e8df --- /dev/null +++ b/etc/ci/azure-container-deb.yml @@ -0,0 +1,50 @@ +parameters: + job_name: '' + container: '' + python_path: '' + python_version: '' + package_manager: apt-get + install_python: '' + install_packages: | + set -e -x + sudo apt-get -y update + sudo apt-get -y install \ + build-essential \ + xz-utils zlib1g bzip2 libbz2-1.0 tar \ + sqlite3 libxml2-dev libxslt1-dev \ + software-properties-common openssl + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: 'ubuntu-22.04' + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml new file mode 100644 index 0000000..a64138c --- /dev/null +++ b/etc/ci/azure-container-rpm.yml @@ -0,0 +1,51 @@ +parameters: + job_name: '' + image_name: 'ubuntu-22.04' + container: '' + python_path: '' + python_version: '' + package_manager: yum + install_python: '' + install_packages: | + set -e -x + sudo yum groupinstall -y "Development Tools" + sudo yum install -y \ + openssl openssl-devel \ + sqlite-devel zlib-devel xz-devel bzip2-devel \ + bzip2 tar unzip zip \ + libxml2-devel libxslt-devel + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml new file mode 100644 index 0000000..9fdc7f1 --- /dev/null +++ b/etc/ci/azure-posix.yml @@ -0,0 +1,39 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each tsuite in parameters.test_suites }}: + ${{ tsuite.key }}: + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' + + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml new file mode 100644 index 0000000..26b4111 --- /dev/null +++ b/etc/ci/azure-win.yml @@ -0,0 +1,39 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each tsuite in parameters.test_suites }}: + ${{ tsuite.key }}: + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' + + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/install_sudo.sh b/etc/ci/install_sudo.sh new file mode 100644 index 0000000..77f4210 --- /dev/null +++ b/etc/ci/install_sudo.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + + +if [[ "$1" == "apt-get" ]]; then + apt-get update -y + apt-get -o DPkg::Options::="--force-confold" install -y sudo + +elif [[ "$1" == "yum" ]]; then + yum install -y sudo + +elif [[ "$1" == "dnf" ]]; then + dnf install -y sudo + +fi diff --git a/etc/ci/macports-ci b/etc/ci/macports-ci new file mode 100644 index 0000000..ac474e4 --- /dev/null +++ b/etc/ci/macports-ci @@ -0,0 +1,304 @@ +#! /bin/bash + +# Copyright (c) 2019 Giovanni Bussi + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +export COLUMNS=80 + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "COLUMNS=$COLUMNS" >> "$GITHUB_ENV" +fi + +# file to be source at the end of subshell: +export MACPORTS_CI_SOURCEME="$(mktemp)" + +( +# start subshell +# this allows to use the script in two ways: +# 1. as ./macports-ci +# 2. as source ./macports-ci +# as of now, choice 2 only changes the env var COLUMNS. + +MACPORTS_VERSION=2.6.4 +MACPORTS_PREFIX=/opt/local +MACPORTS_SYNC=tarball + +action=$1 +shift + +case "$action" in +(install) + +echo "macports-ci: install" + +KEEP_BREW=yes + +for opt +do + case "$opt" in + (--source) SOURCE=yes ;; + (--binary) SOURCE=no ;; + (--keep-brew) KEEP_BREW=yes ;; + (--remove-brew) KEEP_BREW=no ;; + (--version=*) MACPORTS_VERSION="${opt#--version=}" ;; + (--prefix=*) MACPORTS_PREFIX="${opt#--prefix=}" ;; + (--sync=*) MACPORTS_SYNC="${opt#--sync=}" ;; + (*) echo "macports-ci: unknown option $opt" + exit 1 ;; + esac +done + +if test "$KEEP_BREW" = no ; then + echo "macports-ci: removing homebrew" + pushd "$(mktemp -d)" + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > uninstall + chmod +x uninstall + ./uninstall --force + popd +else + echo "macports-ci: keeping HomeBrew" +fi + +echo "macports-ci: prefix=$MACPORTS_PREFIX" + +if test "$MACPORTS_PREFIX" != /opt/local ; then + echo "macports-ci: Installing on non standard prefix $MACPORTS_PREFIX can be only made from sources" + SOURCE=yes +fi + +if test "$SOURCE" = yes ; then + echo "macports-ci: Installing from source" +else + echo "macports-ci: Installing from binary" +fi + +echo "macports-ci: Sync mode=$MACPORTS_SYNC" + +pushd "$(mktemp -d)" + +OSX_VERSION="$(sw_vers -productVersion | grep -o '^[0-9][0-9]*\.[0-9][0-9]*')" + +if test "$OSX_VERSION" == 10.10 ; then + OSX_NAME=Yosemite +elif test "$OSX_VERSION" == 10.11 ; then + OSX_NAME=ElCapitan +elif test "$OSX_VERSION" == 10.12 ; then + OSX_NAME=Sierra +elif test "$OSX_VERSION" == 10.13 ; then + OSX_NAME=HighSierra +elif test "$OSX_VERSION" == 10.14 ; then + OSX_NAME=Mojave +elif test "$OSX_VERSION" == 10.15 ; then + OSX_NAME=Catalina +else + echo "macports-ci: Unknown OSX version $OSX_VERSION" + exit 1 +fi + +echo "macports-ci: OSX version $OSX_VERSION $OSX_NAME" + +MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg + +# this is a workaround needed because binary installer MacPorts-2.6.3-10.12-Sierra.pkg is broken +if [ "$SOURCE" != yes ] && [ "$MACPORTS_PKG" = "MacPorts-2.6.3-10.12-Sierra.pkg" ] ; then + echo "macports-ci: WARNING $MACPORTS_PKG installer is broken" + echo "macports-ci: reverting to 2.6.2 installer followed by selfupdate" + MACPORTS_VERSION=2.6.2 + MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg +fi + +URL="https://distfiles.macports.org/MacPorts" +URL="https://github.com/macports/macports-base/releases/download/v$MACPORTS_VERSION/" + +echo "macports-ci: Base URL is $URL" + +if test "$SOURCE" = yes ; then +# download source: + curl -LO $URL/MacPorts-${MACPORTS_VERSION}.tar.bz2 + tar xjf MacPorts-${MACPORTS_VERSION}.tar.bz2 + cd MacPorts-${MACPORTS_VERSION} +# install + ./configure --prefix="$MACPORTS_PREFIX" --with-applications-dir="$MACPORTS_PREFIX/Applications" >/dev/null && + sudo make install >/dev/null +else + +# download installer: + curl -LO $URL/$MACPORTS_PKG +# install: + sudo installer -verbose -pkg $MACPORTS_PKG -target / +fi + +# update: +export PATH="$MACPORTS_PREFIX/bin:$PATH" + +echo "PATH=\"$MACPORTS_PREFIX/bin:\$PATH\"" > "$MACPORTS_CI_SOURCEME" + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "$MACPORTS_PREFIX/bin" >> "$GITHUB_PATH" +fi + + +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +case "$MACPORTS_SYNC" in +(rsync) + echo "macports-ci: Using rsync" + ;; +(github) + echo "macports-ci: Using github" + pushd "$MACPORTS_PREFIX"/var/macports/sources + sudo mkdir -p github.com/macports/macports-ports/ + sudo chown -R $USER:admin github.com + git clone https://github.com/macports/macports-ports.git github.com/macports/macports-ports/ + awk '{if($NF=="[default]") print "file:///opt/local/var/macports/sources/github.com/macports/macports-ports/"; else print}' "$SOURCES" > $HOME/$$.tmp + sudo mv -f $HOME/$$.tmp "$SOURCES" + popd + ;; +(tarball) + echo "macports-ci: Using tarball" + awk '{if($NF=="[default]") print "https://distfiles.macports.org/ports.tar.gz [default]"; else print}' "$SOURCES" > $$.tmp + sudo mv -f $$.tmp "$SOURCES" + ;; +(*) + echo "macports-ci: Unknown sync mode $MACPORTS_SYNC" + ;; +esac + +i=1 +# run through a while to retry upon failure +while true +do + echo "macports-ci: Trying to selfupdate (iteration $i)" +# here I test for the presence of a known portfile +# this check confirms that ports were installed +# notice that port -N selfupdate && break is not sufficient as a test +# (sometime it returns a success even though ports have not been installed) +# for some misterious reasons, running without "-d" does not work in some case + sudo port -d -N selfupdate 2>&1 | grep -v DEBUG | awk '{if($1!="x")print}' + port info xdrfile > /dev/null && break || true + sleep 5 + i=$((i+1)) + if ((i>20)) ; then + echo "macports-ci: Failed after $i iterations" + exit 1 + fi +done + +echo "macports-ci: Selfupdate successful after $i iterations" + +dir="$PWD" +popd +sudo rm -fr $dir + +;; + +(localports) + +echo "macports-ci: localports" + +for opt +do + case "$opt" in + (*) ports="$opt" ;; + esac +done + +if ! test -d "$ports" ; then + echo "macports-ci: Please provide a port directory" + exit 1 +fi + +w=$(which port) + +MACPORTS_PREFIX="${w%/bin/port}" + +cd "$ports" + +ports="$(pwd)" + +echo "macports-ci: Portdir fullpath: $ports" +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +awk -v repo="file://$ports" '{if($NF=="[default]") print repo; print}' "$SOURCES" > $$.tmp +sudo mv -f $$.tmp "$SOURCES" + +portindex + +;; + +(ccache) +w=$(which port) +MACPORTS_PREFIX="${w%/bin/port}" + +echo "macports-ci: ccache" + +ccache_do=install + +for opt +do + case "$opt" in + (--save) ccache_do=save ;; + (--install) ccache_do=install ;; + (*) echo "macports-ci: ccache: unknown option $opt" + exit 1 ;; + esac +done + + +case "$ccache_do" in +(install) +# first install ccache +sudo port -N install ccache +# then tell macports to use it +CONF="${MACPORTS_PREFIX}"/etc/macports/macports.conf +awk '{if(match($0,"configureccache")) print "configureccache yes" ; else print }' "$CONF" > $$.tmp +sudo mv -f $$.tmp "$CONF" + +# notice that cache size is set to 512Mb, same as it is set by Travis-CI on linux +# might be changed in the future +test -f "$HOME"/.macports-ci-ccache/ccache.conf && + sudo rm -fr "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo mkdir -p "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo cp -a "$HOME"/.macports-ci-ccache/* "$MACPORTS_PREFIX"/var/macports/build/.ccache/ && + sudo echo "max_size = 512M" > "$MACPORTS_PREFIX"/var/macports/build/.ccache/ccache.conf && + sudo chown -R macports:admin "$MACPORTS_PREFIX"/var/macports/build/.ccache + +;; +(save) + +sudo rm -fr "$HOME"/.macports-ci-ccache +sudo mkdir -p "$HOME"/.macports-ci-ccache +sudo cp -a "$MACPORTS_PREFIX"/var/macports/build/.ccache/* "$HOME"/.macports-ci-ccache/ + +esac + +CCACHE_DIR="$MACPORTS_PREFIX"/var/macports/build/.ccache/ ccache -s + +;; + +(*) +echo "macports-ci: unknown action $action" + +esac + +) + +# allows setting env var if necessary: +source "$MACPORTS_CI_SOURCEME" diff --git a/etc/ci/macports-ci.ABOUT b/etc/ci/macports-ci.ABOUT new file mode 100644 index 0000000..60a11f8 --- /dev/null +++ b/etc/ci/macports-ci.ABOUT @@ -0,0 +1,16 @@ +about_resource: macports-ci +name: macports-ci +version: c9676e67351a3a519e37437e196cd0ee9c2180b8 +download_url: https://raw.githubusercontent.com/GiovanniBussi/macports-ci/c9676e67351a3a519e37437e196cd0ee9c2180b8/macports-ci +description: Simplify MacPorts setup on Travis-CI +homepage_url: https://github.com/GiovanniBussi/macports-ci +license_expression: mit +copyright: Copyright (c) Giovanni Bussi +attribute: yes +checksum_md5: 5d31d479132502f80acdaed78bed9e23 +checksum_sha1: 74b15643bd1a528d91b4a7c2169c6fc656f549c2 +package_url: pkg:github/giovannibussi/macports-ci@c9676e67351a3a519e37437e196cd0ee9c2180b8#macports-ci +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/etc/ci/mit.LICENSE b/etc/ci/mit.LICENSE new file mode 100644 index 0000000..e662c78 --- /dev/null +++ b/etc/ci/mit.LICENSE @@ -0,0 +1,5 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst new file mode 100755 index 0000000..5e54a2c --- /dev/null +++ b/etc/scripts/README.rst @@ -0,0 +1,112 @@ +This directory contains the tools to manage a directory of thirdparty Python +package source, wheels and metadata pin, build, update, document and publish to +a PyPI-like repo (GitHub release). + +NOTE: These are tested to run ONLY on Linux. + + +Thirdparty packages management scripts +====================================== + +Pre-requisites +-------------- + +* There are two run "modes": + + * To generate or update pip requirement files, you need to start with a clean + virtualenv as instructed below (This is to avoid injecting requirements + specific to the tools used here in the main requirements). + + * For other usages, the tools here can run either in their own isolated + virtualenv or in the the main configured development virtualenv. + These requireements need to be installed:: + + pip install --requirement etc/scripts/requirements.txt + +TODO: we need to pin the versions of these tools + + + +Generate or update pip requirement files +---------------------------------------- + +Scripts +~~~~~~~ + +**gen_requirements.py**: create/update requirements files from currently + installed requirements. + +**gen_requirements_dev.py** does the same but can subtract the main requirements + to get extra requirements used in only development. + + +Usage +~~~~~ + +The sequence of commands to run are: + + +* Start with these to generate the main pip requirements file:: + + ./configure --clean + ./configure + python etc/scripts/gen_requirements.py --site-packages-dir + +* You can optionally install or update extra main requirements after the + ./configure step such that these are included in the generated main requirements. + +* Optionally, generate a development pip requirements file by running these:: + + ./configure --clean + ./configure --dev + python etc/scripts/gen_requirements_dev.py --site-packages-dir + +* You can optionally install or update extra dev requirements after the + ./configure step such that these are included in the generated dev + requirements. + +Notes: we generate development requirements after the main as this step requires +the main requirements.txt to be up-to-date first. See **gen_requirements.py and +gen_requirements_dev.py** --help for details. + +Note: this does NOT hash requirements for now. + +Note: Be aware that if you are using "conditional" requirements (e.g. only for +OS or Python versions) in setup.py/setp.cfg/requirements.txt as these are NOT +yet supported. + + +Populate a thirdparty directory with wheels, sources, .ABOUT and license files +------------------------------------------------------------------------------ + +Scripts +~~~~~~~ + +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. + +* **check_thirdparty.py** will check a thirdparty directory for errors. + + +Upgrade virtualenv app +---------------------- + +The bundled virtualenv.pyz has to be upgraded by hand and is stored under +etc/thirdparty + +* Fetch https://github.com/pypa/get-virtualenv/raw//public/virtualenv.pyz + for instance https://github.com/pypa/get-virtualenv/raw/20.2.2/public/virtualenv.pyz + and save to thirdparty and update the ABOUT and LICENSE files as needed. + +* This virtualenv app contains also bundled pip, wheel and setuptools that are + essential for the installation to work. + + +Other files +=========== + +The other files and scripts are test, support and utility modules used by the +main scripts documented here. diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py new file mode 100644 index 0000000..65ae595 --- /dev/null +++ b/etc/scripts/check_thirdparty.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() +@click.option( + "-d", + "--dest", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + required=True, + help="Path to the thirdparty directory to check.", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) +@click.help_option("-h", "--help") +def check_thirdparty_dir( + dest, + wheels, + sdists, +): + """ + Check a thirdparty directory for problems and print these on screen. + """ + print("==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + check_thirdparty_dir() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 0000000..76a19a6 --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import sys +from collections import defaultdict + +import click + +import utils_requirements +import utils_thirdparty + +TRACE = False +TRACE_DEEP = False + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEX_URLS, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", +) +@click.option( + "--sdist-only", + "sdist_only", + type=str, + metavar="SDIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in sdist format (no wheels). " + "The command will not fail and exit if no wheel exists for these names", +) +@click.option( + "--wheel-only", + "wheel_only", + type=str, + metavar="WHEEL", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in wheel format (no sdist). " + "The command will not fail and exit if no sdist exists for these names", +) +@click.option( + "--no-dist", + "no_dist", + type=str, + metavar="DIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that do not come either in wheel or sdist format. " + "The command will not fail and exit if no distribution exists for these names", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, + use_cached_index, + sdist_only, + wheel_only, + no_dist, +): + """ + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. + + Download from the provided PyPI simple --index-url INDEX(s) URLs. + """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_required_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + + # create the environments matrix we need for wheels + environments = None + if wheels: + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_or_sdist_not_found = defaultdict(list) + + for name, version in sorted(required_name_versions): + nv = name, version + print(f"Processing: {name} @ {version}") + if wheels: + for environment in environments: + if TRACE: + print(f" ==> Fetching wheel for envt: {environment}") + + fetched = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + repos=repos, + ) + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) + if TRACE: + print(" NOT FOUND") + + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") + if TRACE: + print(" NOT FOUND") + + mia = [] + for nv, dists in wheels_or_sdist_not_found.items(): + name, _, version = nv.partition("==") + if name in no_dist: + continue + sdist_missing = sdists and "sdist" in dists and name not in wheel_only + if sdist_missing: + mia.append(f"SDist missing: {nv} {dists}") + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only + if wheels_missing: + mia.append(f"Wheels missing: {nv} {dists}") + + if mia: + for m in mia: + print(m) + raise Exception(mia) + + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print("==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py new file mode 100644 index 0000000..89d0626 --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT +# Copyright (c) 2010 David Wolever . All rights reserved. +# originally from https://github.com/wolever/pip2pi + +import hashlib +import os +import re +import shutil +from collections import defaultdict +from html import escape +from pathlib import Path +from typing import NamedTuple + +""" +Generate a PyPI simple index froma directory. +""" + + +class InvalidDistributionFilename(Exception): + pass + + +def get_package_name_from_filename(filename): + """ + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. + Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: + + >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") + 'foo' + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") + 'foo-bar' + >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") + 'cython' + >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") + 'python-ldap' + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass + """ + if not filename or not filename.endswith(dist_exts): + raise InvalidDistributionFilename(filename) + + filename = os.path.basename(filename) + + if filename.endswith(sdist_exts): + name_ver = None + extension = None + + for ext in sdist_exts: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + raise InvalidDistributionFilename(filename) + + name, _, version = name_ver.rpartition("-") + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(wheel_ext): + wheel_info = get_wheel_from_filename(filename) + + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group("name") + version = wheel_info.group("version") + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(app_ext): + name_ver, extension, _ = filename.rpartition(".pyz") + + if "-" in filename: + name, _, version = name_ver.rpartition("-") + else: + name = name_ver + + if not name: + raise InvalidDistributionFilename(filename) + + name = normalize_name(name) + return name + + +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in sorted(packages, key=lambda p: p.archive_file): + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = """ + + + Links for all packages + + """ + document.append(header) + + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): + """ + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. + + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links + """ + + directory = Path(directory) + + index_dir = directory / "simple" + if index_dir.exists(): + shutil.rmtree(str(index_dir), ignore_errors=True) + + index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) + + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '', + ] + + for pkg_file in directory.iterdir(): + pkg_filename = pkg_file.name + + if ( + not pkg_file.is_file() + or not pkg_filename.endswith(dist_exts) + or pkg_filename.startswith(".") + ): + continue + + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) + pkg_index_dir = index_dir / pkg_name + pkg_index_dir.mkdir(parents=True, exist_ok=True) + pkg_indexed_file = pkg_index_dir / pkg_filename + + link_target = Path("../..") / pkg_filename + pkg_indexed_file.symlink_to(link_target) + + if pkg_name not in packages_by_package_name: + esc_name = escape(pkg_name) + simple_html_index.append(f'{esc_name}
') + + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) + +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) + +if __name__ == "__main__": + import sys + + pkg_dir = sys.argv[1] + build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_pypi_simple.py.ABOUT b/etc/scripts/gen_pypi_simple.py.ABOUT new file mode 100644 index 0000000..4de5ded --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: gen_pypi_simple.py +name: gen_pypi_simple.py +license_expression: bsd-2-clause-views and mit +copyright: Copyright (c) nexB Inc. + Copyright (c) 2010 David Wolever + Copyright (c) The pip developers +notes: Originally from https://github.com/wolever/pip2pi and modified extensivley + Also partially derived from pip code diff --git a/etc/scripts/gen_pypi_simple.py.NOTICE b/etc/scripts/gen_pypi_simple.py.NOTICE new file mode 100644 index 0000000..6e0fbbc --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.NOTICE @@ -0,0 +1,56 @@ +SPDX-License-Identifier: BSD-2-Clause-Views AND mit + +Copyright (c) nexB Inc. +Copyright (c) 2010 David Wolever +Copyright (c) The pip developers + + +Original code: copyright 2010 David Wolever . All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of David Wolever. + + +Original code: Copyright (c) 2008-2020 The pip developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py new file mode 100644 index 0000000..1b87944 --- /dev/null +++ b/etc/scripts/gen_requirements.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import argparse +import pathlib + +import utils_requirements + +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" + + +def gen_requirements(): + description = """ + Create or replace the `--requirements-file` file FILE requirements file with all + locally installed Python packages.all Python packages found installed in `--site-packages-dir` + """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + + utils_requirements.lock_requirements( + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, + ) + + +if __name__ == "__main__": + gen_requirements() diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py new file mode 100644 index 0000000..8548205 --- /dev/null +++ b/etc/scripts/gen_requirements_dev.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import argparse +import pathlib + +import utils_requirements + +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" + + +def gen_dev_requirements(): + description = """ + Create or overwrite the `--dev-requirements-file` pip requirements FILE with + all Python packages found installed in `--site-packages-dir`. Exclude + package names also listed in the --main-requirements-file pip requirements + FILE (that are assume to the production requirements and therefore to always + be present in addition to the development requirements). + """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + + utils_requirements.lock_dev_requirements( + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, + ) + + +if __name__ == "__main__": + gen_dev_requirements() diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt new file mode 100644 index 0000000..7c514da --- /dev/null +++ b/etc/scripts/requirements.txt @@ -0,0 +1,12 @@ +aboutcode_toolkit +attrs +commoncode +click +requests +saneyaml +pip +setuptools +twine +wheel +build +packvers diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py new file mode 100644 index 0000000..0e9c360 --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -0,0 +1,131 @@ +""" +Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import sysconfig +from unittest.mock import patch + +import pytest + +import utils_pip_compatibility_tags + + +@pytest.mark.parametrize( + "version_info, expected", + [ + ((2,), "2"), + ((2, 8), "28"), + ((3,), "3"), + ((3, 6), "36"), + # Test a tuple of length 3. + ((3, 6, 5), "36"), + # Test a 2-digit minor version. + ((3, 10), "310"), + ], +) +def test_version_info_to_nodot(version_info, expected): + actual = utils_pip_compatibility_tags.version_info_to_nodot(version_info) + assert actual == expected + + +class Testcompatibility_tags: + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + get_config_var = sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + + return _mock_get_config_var + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + import pip._internal.utils.compatibility_tags + + mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") + + with patch("sysconfig.get_config_var", mock_gcf): + supported = pip._internal.utils.compatibility_tags.get_supported() + + for tag in supported: + assert "-" not in tag.interpreter + assert "-" not in tag.abi + assert "-" not in tag.platform + + +class TestManylinux2010Tags: + @pytest.mark.parametrize( + "manylinux2010,manylinux1", + [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ], + ) + def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): + """ + Specifying manylinux2010 implies manylinux1. + """ + groups = {} + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + for tag in supported: + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) + + for arches in groups.values(): + if arches == ["any"]: + continue + assert arches[:2] == [manylinux2010, manylinux1] + + +class TestManylinux2014Tags: + @pytest.mark.parametrize( + "manylinuxA,manylinuxB", + [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ], + ) + def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): + """ + Specifying manylinux2014 implies manylinux2010/manylinux1. + """ + groups = {} + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinuxA]) + for tag in supported: + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) + + expected_arches = [manylinuxA] + expected_arches.extend(manylinuxB) + for arches in groups.values(): + if arches == ["any"]: + continue + assert arches[:3] == expected_arches diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 0000000..07eee35 --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: test_utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: tests/unit/test_utils_compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#tests/unit/test_utils_compatibility_tags.py + +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py new file mode 100644 index 0000000..d291572 --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from utils_pypi_supported_tags import validate_platforms_for_pypi + +""" +Wheel platform checking tests + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +""" + + +def validate_wheel_filename_for_pypi(filename): + """ + Validate if the filename is a PyPI/warehouse-uploadable wheel file name + with supported platform tags. Return a list of unsupported platform tags or + an empty list if all tags are supported. + """ + from utils_thirdparty import Wheel + + wheel = Wheel.from_filename(filename) + return validate_platforms_for_pypi(wheel.platforms) + + +@pytest.mark.parametrize( + "plat", + [ + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_i686", + "manylinux1_x86_64", + "manylinux2010_i686", + "manylinux2010_x86_64", + "manylinux2014_i686", + "manylinux2014_x86_64", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "manylinux_2_5_i686", + "manylinux_2_12_x86_64", + "manylinux_2_17_aarch64", + "manylinux_2_17_armv7l", + "manylinux_2_17_ppc64", + "manylinux_2_17_ppc64le", + "manylinux_3_0_s390x", + "macosx_10_6_intel", + "macosx_10_13_x86_64", + "macosx_11_0_x86_64", + "macosx_10_15_arm64", + "macosx_11_10_universal2", + # A real tag used by e.g. some numpy wheels + ( + "macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." + "macosx_10_10_intel.macosx_10_10_x86_64" + ), + ], +) +def test_is_valid_pypi_wheel_return_true_for_supported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + assert not validate_wheel_filename_for_pypi(filename) + + +@pytest.mark.parametrize( + "plat", + [ + "linux_x86_64", + "linux_x86_64.win32", + "macosx_9_2_x86_64", + "macosx_12_2_arm64", + "macosx_10_15_amd64", + ], +) +def test_is_valid_pypi_wheel_raise_exception_for_aunsupported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + invalid = validate_wheel_filename_for_pypi(filename) + assert invalid diff --git a/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 0000000..176efac --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: test_utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: tests/unit/forklift/test_legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#tests/unit/forklift/test_legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Test for wheel platform checking copied and heavily modified on + 2020-12-24 from warehouse. This contains the basic functions to check if a + wheel file name is would be supported for uploading to PyPI. diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py new file mode 100644 index 0000000..374c06f --- /dev/null +++ b/etc/scripts/update_skeleton.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. AboutCode, and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import os +import subprocess + +import click + + +ABOUTCODE_PUBLIC_REPO_NAMES = [ + "aboutcode-toolkit", + "ahocode", + "bitcode", + "clearcode-toolkit", + "commoncode", + "container-inspector", + "debian-inspector", + "deltacode", + "elf-inspector", + "extractcode", + "fetchcode", + "gemfileparser2", + "gh-issue-sandbox", + "go-inspector", + "heritedcode", + "license-expression", + "license_copyright_pipeline", + "nuget-inspector", + "pip-requirements-parser", + "plugincode", + "purldb", + "pygmars", + "python-inspector", + "sanexml", + "saneyaml", + "scancode-analyzer", + "scancode-toolkit-contrib", + "scancode-toolkit-reference-scans", + "thirdparty-toolkit", + "tracecode-toolkit", + "tracecode-toolkit-strace", + "turbo-spdx", + "typecode", + "univers", +] + + +@click.command() +@click.help_option("-h", "--help") +def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): + """ + Update project files of AboutCode projects that use the skeleton + + This script will: + - Clone the repo + - Add the skeleton repo as a new origin + - Create a new branch named "update-skeleton-files" + - Merge in the new skeleton files into the "update-skeleton-files" branch + + The user will need to save merge commit messages that pop up when running + this script in addition to resolving the merge conflicts on repos that have + them. + """ + + # Create working directory + work_dir_path = Path("/tmp/update_skeleton/") + if not os.path.exists(work_dir_path): + os.makedirs(work_dir_path, exist_ok=True) + + for repo_name in repo_names: + # Move to work directory + os.chdir(work_dir_path) + + # Clone repo + repo_git = f"git@github.com:aboutcode-org/{repo_name}.git" + subprocess.run(["git", "clone", repo_git]) + + # Go into cloned repo + os.chdir(work_dir_path / repo_name) + + # Add skeleton as an origin + subprocess.run( + ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] + ) + + # Fetch skeleton files + subprocess.run(["git", "fetch", "skeleton"]) + + # Create and checkout new branch + subprocess.run(["git", "checkout", "-b", "update-skeleton-files"]) + + # Merge skeleton files into the repo + subprocess.run(["git", "merge", "skeleton/main", "--allow-unrelated-histories"]) + + +if __name__ == "__main__": + update_skeleton_files() diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py new file mode 100644 index 0000000..b6bff51 --- /dev/null +++ b/etc/scripts/utils_dejacode.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import io +import os +import zipfile + +import requests +import saneyaml +from packvers import version as packaging_version + +""" +Utility to create and retrieve package and ABOUT file data from DejaCode. +""" + +DEJACODE_API_KEY = os.environ.get("DEJACODE_API_KEY", "") +DEJACODE_API_URL = os.environ.get("DEJACODE_API_URL", "") + +DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" +DEJACODE_API_HEADERS = { + "Authorization": f"Token {DEJACODE_API_KEY}", + "Accept": "application/json; indent=4", +} + + +def can_do_api_calls(): + if not DEJACODE_API_KEY and DEJACODE_API_URL: + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + return False + else: + return True + + +def fetch_dejacode_packages(params): + """ + Return a list of package data mappings calling the package API with using + `params` or an empty list. + """ + if not can_do_api_calls(): + return [] + + response = requests.get( + DEJACODE_API_URL_PACKAGES, + params=params, + headers=DEJACODE_API_HEADERS, + timeout=10, + ) + + return response.json()["results"] + + +def get_package_data(distribution): + """ + Return a mapping of package data or None for a Distribution `distribution`. + """ + results = fetch_dejacode_packages(distribution.identifiers()) + + len_results = len(results) + + if len_results == 1: + return results[0] + + elif len_results > 1: + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + else: + print("Could not find package:", distribution.download_url) + + +def update_with_dejacode_data(distribution): + """ + Update the Distribution `distribution` with DejaCode package data. Return + True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + return distribution.update(package_data, keep_extra=False) + + print(f"No package found for: {distribution}") + + +def update_with_dejacode_about_data(distribution): + """ + Update the Distribution `distribution` wiht ABOUT code data fetched from + DejaCode. Return True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about" + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) + # note that this is YAML-formatted + about_text = response.json()["about_data"] + about_data = saneyaml.load(about_text) + + return distribution.update(about_data, keep_extra=True) + + print(f"No package found for: {distribution}") + + +def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): + """ + Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched + from DejaCode for a Distribution `distribution`. Return True if files were + fetched. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about_files" + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) + about_zip = response.content + with io.BytesIO(about_zip) as zf: + with zipfile.ZipFile(zf) as zi: + zi.extractall(path=dest_dir) + return True + + print(f"No package found for: {distribution}") + + +def find_latest_dejacode_package(distribution): + """ + Return a mapping of package data for the closest version to + a Distribution `distribution` or None. + Return the newest of the packages if prefer_newest is True. + Filter out version-specific attributes. + """ + ids = distribution.purl_identifiers(skinny=True) + packages = fetch_dejacode_packages(params=ids) + if not packages: + return + + for package_data in packages: + matched = ( + package_data["download_url"] == distribution.download_url + and package_data["version"] == distribution.version + and package_data["filename"] == distribution.filename + ) + + if matched: + return package_data + + # there was no exact match, find the latest version + # TODO: consider the closest version rather than the latest + # or the version that has the best data + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] + with_versions = sorted(with_versions) + latest_version, latest_package_version = sorted(with_versions)[-1] + print( + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", + ) + + return latest_package_version + + +def create_dejacode_package(distribution): + """ + Create a new DejaCode Package a Distribution `distribution`. + Return the new or existing package data. + """ + if not can_do_api_calls(): + return + + existing_package_data = get_package_data(distribution) + if existing_package_data: + return existing_package_data + + print(f"Creating new DejaCode package for: {distribution}") + + new_package_payload = { + # Trigger data collection, scan, and purl + "collect_data": 1, + } + + fields_to_carry_over = [ + "download_urltype", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + ] + + for field in fields_to_carry_over: + value = getattr(distribution, field, None) + if value: + new_package_payload[field] = value + + response = requests.post( + DEJACODE_API_URL_PACKAGES, + data=new_package_payload, + headers=DEJACODE_API_HEADERS, + timeout=10, + ) + new_package_data = response.json() + if response.status_code != 201: + raise Exception(f"Error, cannot create package for: {distribution}") + + print(f"New Package created at: {new_package_data['absolute_url']}") + return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py new file mode 100644 index 0000000..dd954bc --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -0,0 +1,192 @@ +""" +Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import re + +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms + +_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") + + +def version_info_to_nodot(version_info): + # type: (Tuple[int, ...]) -> str + # Only use up to the first two numbers. + return "".join(map(str, version_info[:2])) + + +def _mac_platforms(arch): + # type: (str) -> List[str] + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + mac_version = (int(major), int(minor)) + arches = [ + # Since we have always only checked that the platform starts + # with "macosx", for backwards-compatibility we extract the + # actual prefix provided by the user in case they provided + # something like "macosxcustom_". It may be good to remove + # this as undocumented or deprecate it in the future. + "{}_{}".format(name, arch[len("macosx_") :]) + for arch in mac_platforms(mac_version, actual_arch) + ] + else: + # arch pattern didn't match (?!) + arches = [arch] + return arches + + +def _custom_manylinux_platforms(arch): + # type: (str) -> List[str] + arches = [arch] + arch_prefix, arch_sep, arch_suffix = arch.partition("_") + if arch_prefix == "manylinux2014": + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {"i686", "x86_64"}: + arches.append("manylinux2010" + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) + elif arch_prefix == "manylinux2010": + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches.append("manylinux1" + arch_sep + arch_suffix) + return arches + + +def _get_custom_platforms(arch): + # type: (str) -> List[str] + arch_prefix, _arch_sep, _arch_suffix = arch.partition("_") + if arch.startswith("macosx"): + arches = _mac_platforms(arch) + elif arch_prefix in ["manylinux2014", "manylinux2010"]: + arches = _custom_manylinux_platforms(arch) + else: + arches = [arch] + return arches + + +def _expand_allowed_platforms(platforms): + # type: (Optional[List[str]]) -> Optional[List[str]] + if not platforms: + return None + + seen = set() + result = [] + + for p in platforms: + if p in seen: + continue + additions = [c for c in _get_custom_platforms(p) if c not in seen] + seen.update(additions) + result.extend(additions) + + return result + + +def _get_python_version(version): + # type: (str) -> PythonVersion + if len(version) > 1: + return int(version[0]), int(version[1:]) + else: + return (int(version[0]),) + + +def _get_custom_interpreter(implementation=None, version=None): + # type: (Optional[str], Optional[str]) -> str + if implementation is None: + implementation = interpreter_name() + if version is None: + version = interpreter_version() + return f"{implementation}{version}" + + +def get_supported( + version=None, # type: Optional[str] + platforms=None, # type: Optional[List[str]] + impl=None, # type: Optional[str] + abis=None, # type: Optional[List[str]] +): + # type: (...) -> List[Tag] + """ + Return a list of supported tags for each version specified in + `versions`. + + :param version: a string version, of the form "33" or "32", + or None. The version will be assumed to support our ABI. + :param platforms: specify a list of platforms you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abis: specify a list of abis you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] # type: List[Tag] + + python_version = None # type: Optional[PythonVersion] + if version is not None: + python_version = _get_python_version(version) + + interpreter = _get_custom_interpreter(impl, version) + + platforms = _expand_allowed_platforms(platforms) + + is_cpython = (impl or interpreter_name()) == "cp" + if is_cpython: + supported.extend( + cpython_tags( + python_version=python_version, + abis=abis, + platforms=platforms, + ) + ) + else: + supported.extend( + generic_tags( + interpreter=interpreter, + abis=abis, + platforms=platforms, + ) + ) + supported.extend( + compatible_tags( + python_version=python_version, + interpreter=interpreter, + platforms=platforms, + ) + ) + + return supported diff --git a/etc/scripts/utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 0000000..7bbb026 --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/utils/compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#src/pip/_internal/utils/compatibility_tags.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling \ No newline at end of file diff --git a/etc/scripts/utils_pypi_supported_tags.py b/etc/scripts/utils_pypi_supported_tags.py new file mode 100644 index 0000000..de9f21b --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py @@ -0,0 +1,105 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +""" +Wheel platform checking + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py + +This contains the basic functions to check if a wheel file name is would be +supported for uploading to PyPI. +""" + +# These platforms can be handled by a simple static list: +_allowed_platforms = { + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_x86_64", + "manylinux1_i686", + "manylinux2010_x86_64", + "manylinux2010_i686", + "manylinux2014_x86_64", + "manylinux2014_i686", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "linux_armv6l", + "linux_armv7l", +} +# macosx is a little more complicated: +_macosx_platform_re = re.compile(r"macosx_(?P\d+)_(\d+)_(?P.*)") +_macosx_arches = { + "ppc", + "ppc64", + "i386", + "x86_64", + "arm64", + "intel", + "fat", + "fat32", + "fat64", + "universal", + "universal2", +} +_macosx_major_versions = { + "10", + "11", +} + +# manylinux pep600 is a little more complicated: +_manylinux_platform_re = re.compile(r"manylinux_(\d+)_(\d+)_(?P.*)") +_manylinux_arches = { + "x86_64", + "i686", + "aarch64", + "armv7l", + "ppc64", + "ppc64le", + "s390x", +} + + +def is_supported_platform_tag(platform_tag): + """ + Return True if the ``platform_tag`` is supported on PyPI. + """ + if platform_tag in _allowed_platforms: + return True + m = _macosx_platform_re.match(platform_tag) + if m and m.group("major") in _macosx_major_versions and m.group("arch") in _macosx_arches: + return True + m = _manylinux_platform_re.match(platform_tag) + if m and m.group("arch") in _manylinux_arches: + return True + return False + + +def validate_platforms_for_pypi(platforms): + """ + Validate if the wheel platforms are supported platform tags on Pypi. Return + a list of unsupported platform tags or an empty list if all tags are + supported. + """ + + # Check that if it's a binary wheel, it's on a supported platform + invalid_tags = [] + for plat in platforms: + if not is_supported_platform_tag(plat): + invalid_tags.append(plat) + return invalid_tags diff --git a/etc/scripts/utils_pypi_supported_tags.py.ABOUT b/etc/scripts/utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 0000000..228a538 --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: warehouse/forklift/legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#warehouse/forklift/legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Wheel platform checking copied and heavily modified on 2020-12-24 from + warehouse. This contains the basic functions to check if a wheel file name is + would be supported for uploading to PyPI. diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py new file mode 100644 index 0000000..b9b2c0e --- /dev/null +++ b/etc/scripts/utils_requirements.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +import re +import subprocess + +""" +Utilities to manage requirements files and call pip. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" + + +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): + """ + Yield package (name, version) tuples for each requirement in a `requirement` + file. Only accept requirements pinned to an exact version. + """ + with open(requirements_file) as reqs: + req_lines = reqs.read().splitlines(False) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) + + +def get_required_name_versions(requirement_lines, with_unpinned=False): + """ + Yield required (name, version) tuples given a`requirement_lines` iterable of + requirement text lines. Only accept requirements pinned to an exact version. + """ + + for req_line in requirement_lines: + req_line = req_line.strip() + if not req_line or req_line.startswith("#"): + continue + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): + print(f"Requirement line is not supported: ignored: {req_line}") + continue + yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) + + +def get_required_name_version(requirement, with_unpinned=False): + """ + Return a (name, version) tuple given a`requirement` specifier string. + Requirement version must be pinned. If ``with_unpinned`` is True, unpinned + requirements are accepted and only the name portion is returned. + + For example: + >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected + >>> try: + ... assert not get_required_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) + """ + requirement = requirement and "".join(requirement.lower().split()) + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") + name, operator, version = split_req(requirement) + if not name: + raise ValueError(f"Name is required: {requirement}") + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") + return name, version + + +def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): + """ + Freeze and lock current installed requirements and save this to the + `requirements_file` requirements file. + """ + with open(requirements_file, "w") as fo: + fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) + + +def lock_dev_requirements( + dev_requirements_file="requirements-dev.txt", + main_requirements_file="requirements.txt", + site_packages_dir=None, +): + """ + Freeze and lock current installed development-only requirements and save + this to the `dev_requirements_file` requirements file. Development-only is + achieved by subtracting requirements from the `main_requirements_file` + requirements file from the current requirements using package names (and + ignoring versions). + """ + main_names = {n for n, _v in load_requirements(main_requirements_file)} + all_reqs = get_installed_reqs(site_packages_dir=site_packages_dir) + all_req_lines = all_reqs.splitlines(False) + all_req_nvs = get_required_name_versions(all_req_lines) + dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} + + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + with open(dev_requirements_file, "w") as fo: + fo.write(new_reqs) + + +def get_installed_reqs(site_packages_dir): + """ + Return the installed pip requirements as text found in `site_packages_dir` + as a text. + """ + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + if not req: + raise ValueError("req is required") + # do not allow multiple constraints and tags + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py new file mode 100644 index 0000000..bc68ac7 --- /dev/null +++ b/etc/scripts/utils_thirdparty.py @@ -0,0 +1,2286 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import email +import itertools +import os +import re +import shutil +import subprocess +import tempfile +import time +import urllib +from collections import defaultdict +from urllib.parse import quote_plus + +import attr +import license_expression +import packageurl +import requests +import saneyaml +from commoncode import fileutils +from commoncode.hash import multi_checksums +from commoncode.text import python_safe_name +from packvers import tags as packaging_tags +from packvers import version as packaging_version + +import utils_pip_compatibility_tags + +""" +Utilities to manage Python thirparty libraries source, binaries and metadata in +local directories and remote repositories. + +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development + + +Approach +-------- + +The processing is organized around these key objects: + +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: + + - one Sdist source Distribution + - a list of Wheel binary Distribution + +- A Distribution (either a Wheel or Sdist) is identified by and created from its + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +""" +Wheel downloader + +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} + + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message + + +""" + +TRACE = False +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False + +# Supported environments +PYTHON_VERSIONS = "310", "311", "312", "313", "314" + +PYTHON_DOT_VERSIONS_BY_VER = { + "310": "3.10", + "311": "3.11", + "312": "3.12", + "313": "3.13", + "314": "3.14", +} + + +def get_python_dot_version(version): + """ + Return a dot version from a plain, non-dot version. + """ + return PYTHON_DOT_VERSIONS_BY_VER[version] + + +ABIS_BY_PYTHON_VERSION = { + "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], + "312": ["cp312", "cp312m", "abi3"], + "313": ["cp313", "cp313m", "abi3"], + "314": ["cp314", "cp314m", "abi3"], +} + +PLATFORMS_BY_OS = { + "linux": [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + ], + "macos": [ + "macosx_10_6_intel", + "macosx_10_6_x86_64", + "macosx_10_9_intel", + "macosx_10_9_x86_64", + "macosx_10_10_intel", + "macosx_10_10_x86_64", + "macosx_10_11_intel", + "macosx_10_11_x86_64", + "macosx_10_12_intel", + "macosx_10_12_x86_64", + "macosx_10_13_intel", + "macosx_10_13_x86_64", + "macosx_10_14_intel", + "macosx_10_14_x86_64", + "macosx_10_15_intel", + "macosx_10_15_x86_64", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", + # 'macosx_11_0_arm64', + ], + "windows": [ + "win_amd64", + ], +} + +THIRDPARTY_DIR = "thirdparty" +CACHE_THIRDPARTY_DIR = ".cache/thirdparty" + +################################################################################ + +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ + +EXTENSIONS_APP = (".pyz",) +EXTENSIONS_SDIST = ( + ".tar.gz", + ".zip", + ".tar.xz", +) +EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + (".whl",) +EXTENSIONS_ABOUT = ( + ".ABOUT", + ".LICENSE", + ".NOTICE", +) +EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP + +LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" + +LICENSING = license_expression.Licensing() + +collect_urls = re.compile('href="([^"]+)"').findall + +################################################################################ +# Fetch wheels and sources locally +################################################################################ + + +class DistributionNotFound(Exception): + pass + + +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): + """ + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. + + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. + """ + if TRACE_DEEP: + print(f" download_wheel: {name}=={version} for envt: {environment}") + + if not repos: + repos = DEFAULT_PYPI_REPOS + + fetched_wheel_filenames = [] + + for repo in repos: + package = repo.get_package_version(name=name, version=version) + if not package: + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " + ) + continue + + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) + + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break + + return fetched_wheel_filenames + + +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): + """ + Download the sdist source distribution of package ``name`` and ``version`` + into the ``dest_dir`` directory. Return a fetched filename or None. + + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. + """ + if TRACE: + print(f" download_sdist: {name}=={version}") + + if not repos: + repos = DEFAULT_PYPI_REPOS + + fetched_sdist_filename = None + + for repo in repos: + package = repo.get_package_version(name=name, version=version) + + if not package: + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue + + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) + + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break + + return fetched_sdist_filename + + +################################################################################ +# +# Core models +# +################################################################################ + + +@attr.attributes +class NameVer: + name = attr.ib( + type=str, + metadata=dict(help="Python package name, lowercase and normalized."), + ) + + version = attr.ib( + type=str, + metadata=dict(help="Python package version string."), + ) + + @property + def normalized_name(self): + return NameVer.normalize_name(self.name) + + @staticmethod + def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + def sortable_name_version(self): + """ + Return a tuple of values to sort by name, then version. + This method is a suitable to use as key for sorting NameVer instances. + """ + return self.normalized_name, packaging_version.parse(self.version) + + @classmethod + def sorted(cls, namevers): + return sorted(namevers or [], key=cls.sortable_name_version) + + +@attr.attributes +class Distribution(NameVer): + # field names that can be updated from another Distribution or mapping + updatable_fields = [ + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + "extra_data", + ] + + filename = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="File name."), + ) + + path_or_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Path or URL"), + ) + + sha256 = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="SHA256 checksum."), + ) + + sha1 = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="SHA1 checksum."), + ) + + md5 = attr.ib( + repr=False, + type=int, + default=0, + metadata=dict(help="MD5 checksum."), + ) + + type = attr.ib( + repr=False, + type=str, + default="pypi", + metadata=dict(help="Package type"), + ) + + namespace = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Package URL namespace"), + ) + + qualifiers = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help="Package URL qualifiers"), + ) + + subpath = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Package URL subpath"), + ) + + size = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Size in bytes."), + ) + + primary_language = attr.ib( + repr=False, + type=str, + default="Python", + metadata=dict(help="Primary Programming language."), + ) + + description = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Description."), + ) + + homepage_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Homepage URL"), + ) + + notes = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Notes."), + ) + + copyright = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Copyright."), + ) + + license_expression = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="License expression"), + ) + + licenses = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help="List of license mappings."), + ) + + notice_text = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Notice text"), + ) + + extra_data = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help="Extra data"), + ) + + @property + def package_url(self): + """ + Return a Package URL string of self. + """ + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) + + @property + def download_url(self): + return self.get_best_download_url() + + def get_best_download_url(self, repos=tuple()): + """ + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. + """ + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE_DEEP: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + # FIXME: + fetch_and_save( + path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, + as_text=False, + ) + return self.filename + + @property + def about_filename(self): + return f"{self.filename}.ABOUT" + + @property + def about_download_url(self): + return f"{ABOUT_BASE_URL}/{self.about_filename}" + + @property + def notice_filename(self): + return f"{self.filename}.NOTICE" + + @property + def notice_download_url(self): + return f"{ABOUT_BASE_URL}/{self.notice_filename}" + + @classmethod + def from_path_or_url(cls, path_or_url): + """ + Return a distribution built from the data found in the filename of a + ``path_or_url`` string. Raise an exception if this is not a valid + filename. + """ + filename = os.path.basename(path_or_url.strip("/")) + dist = cls.from_filename(filename) + dist.path_or_url = path_or_url + return dist + + @classmethod + def get_dist_class(cls, filename): + if filename.endswith(".whl"): + return Wheel + elif filename.endswith( + ( + ".zip", + ".tar.gz", + ) + ): + return Sdist + raise InvalidDistributionFilename(filename) + + @classmethod + def from_filename(cls, filename): + """ + Return a distribution built from the data found in a `filename` string. + Raise an exception if this is not a valid filename + """ + filename = os.path.basename(filename.strip("/")) + clazz = cls.get_dist_class(filename) + return clazz.from_filename(filename) + + def has_key_metadata(self): + """ + Return True if this distribution has key metadata required for basic attribution. + """ + if self.license_expression == "public-domain": + # copyright not needed + return True + return self.license_expression and self.copyright and self.path_or_url + + def to_about(self): + """ + Return a mapping of ABOUT data from this distribution fields. + """ + about_data = dict( + about_resource=self.filename, + checksum_md5=self.md5, + checksum_sha1=self.sha1, + copyright=self.copyright, + description=self.description, + download_url=self.download_url, + homepage_url=self.homepage_url, + license_expression=self.license_expression, + name=self.name, + namespace=self.namespace, + notes=self.notes, + notice_file=self.notice_filename if self.notice_text else "", + package_url=self.package_url, + primary_language=self.primary_language, + qualifiers=self.qualifiers, + size=self.size, + subpath=self.subpath, + type=self.type, + version=self.version, + ) + + about_data.update(self.extra_data) + about_data = {k: v for k, v in sorted(about_data.items()) if v} + return about_data + + def to_dict(self): + """ + Return a mapping data from this distribution. + """ + return {k: v for k, v in attr.asdict(self).items() if v} + + def save_about_and_notice_files(self, dest_dir=THIRDPARTY_DIR): + """ + Save a .ABOUT file to `dest_dir`. Include a .NOTICE file if there is a + notice_text. + """ + + def save_if_modified(location, content): + if os.path.exists(location): + with open(location) as fi: + existing_content = fi.read() + if existing_content == content: + return False + + if TRACE: + print(f"Saving ABOUT (and NOTICE) files for: {self}") + with open(location, "w") as fo: + fo.write(content) + return True + + as_about = self.to_about() + + save_if_modified( + location=os.path.join(dest_dir, self.about_filename), + content=saneyaml.dump(as_about), + ) + + notice_text = self.notice_text and self.notice_text.strip() + if notice_text: + save_if_modified( + location=os.path.join(dest_dir, self.notice_filename), + content=notice_text, + ) + + def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): + """ + Update self with ABOUT data loaded from an `about_filename_or_data` + which is either a .ABOUT file in `dest_dir` or an ABOUT data mapping. + `about_filename_or_data` defaults to this distribution default ABOUT + filename if not provided. Load the notice_text if present from dest_dir. + """ + if not about_filename_or_data: + about_filename_or_data = self.about_filename + + if isinstance(about_filename_or_data, str): + # that's an about_filename + about_path = os.path.join(dest_dir, about_filename_or_data) + if os.path.exists(about_path): + with open(about_path) as fi: + about_data = saneyaml.load(fi.read()) + if not about_data: + return False + else: + return False + else: + about_data = about_filename_or_data + + md5 = about_data.pop("checksum_md5", None) + if md5: + about_data["md5"] = md5 + sha1 = about_data.pop("checksum_sha1", None) + if sha1: + about_data["sha1"] = sha1 + sha256 = about_data.pop("checksum_sha256", None) + if sha256: + about_data["sha256"] = sha256 + + about_data.pop("about_resource", None) + notice_text = about_data.pop("notice_text", None) + notice_file = about_data.pop("notice_file", None) + if notice_text: + about_data["notice_text"] = notice_text + elif notice_file: + notice_loc = os.path.join(dest_dir, notice_file) + if os.path.exists(notice_loc): + with open(notice_loc) as fi: + about_data["notice_text"] = fi.read() + return self.update(about_data, keep_extra=True) + + def load_remote_about_data(self): + """ + Fetch and update self with "remote" data Distribution ABOUT file and + NOTICE file if any. Return True if the data was updated. + """ + try: + about_text = CACHE.get( + path_or_url=self.about_download_url, + as_text=True, + ) + except RemoteNotFetchedException: + return False + + if not about_text: + return False + + about_data = saneyaml.load(about_text) + notice_file = about_data.pop("notice_file", None) + if notice_file: + try: + notice_text = CACHE.get( + path_or_url=self.notice_download_url, + as_text=True, + ) + if notice_text: + about_data["notice_text"] = notice_text + except RemoteNotFetchedException: + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") + return self.load_about_data(about_data) + + def get_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return a mapping of computed checksums for this dist filename is + `dest_dir`. + """ + dist_loc = os.path.join(dest_dir, self.filename) + if os.path.exists(dist_loc): + return multi_checksums(dist_loc, checksum_names=("md5", "sha1", "sha256")) + else: + return {} + + def set_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with checksums computed for this dist filename is `dest_dir`. + """ + self.update(self.get_checksums(dest_dir), overwrite=True) + + def validate_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return True if all checksums that have a value in this dist match + checksums computed for this dist filename is `dest_dir`. + """ + real_checksums = self.get_checksums(dest_dir) + for csk in ("md5", "sha1", "sha256"): + csv = getattr(self, csk) + rcv = real_checksums.get(csk) + if csv and rcv and csv != rcv: + return False + return True + + def get_license_keys(self): + try: + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) + except license_expression.ExpressionParseError: + return ["unknown"] + return keys + + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): + """ + Fetch license files if missing in `dest_dir`. + Return True if license files were fetched. + """ + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links + errors = [] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names += [self.extra_data.get("license_file")] + extra_lic_names = [ln for ln in extra_lic_names if ln] + lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] + for filename in lic_names + extra_lic_names: + floc = os.path.join(dest_dir, filename) + if os.path.exists(floc): + continue + + try: + # try remotely first + lic_url = get_license_link_for_filename(filename=filename, urls=urls) + + fetch_and_save( + path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, + as_text=True, + ) + if TRACE: + print(f"Fetched license from remote: {lic_url}") + + except: + try: + # try licensedb second + lic_url = f"{LICENSEDB_API_URL}/{filename}" + fetch_and_save( + path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, + as_text=True, + ) + if TRACE: + print(f"Fetched license from licensedb: {lic_url}") + + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + print(msg) + errors.append(msg) + + return errors + + def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): + """ + Return the text of the first PKG-INFO or METADATA file found in the + archive of this Distribution in `dest_dir`. Return None if not found. + """ + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: + shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) + # NOTE: we only care about the first one found in the dist + # which may not be 100% right + for pi in fileutils.resource_iter(location=td, with_dirs=False): + if pi.endswith( + ( + "PKG-INFO", + "METADATA", + ) + ): + with open(pi) as fi: + return fi.read() + + def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with data loaded from the PKG-INFO file found in the + archive of this Distribution in `dest_dir`. + """ + pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) + if not pkginfo_text: + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") + return + raw_data = email.message_from_string(pkginfo_text) + + classifiers = raw_data.get_all("Classifier") or [] + + declared_license = [raw_data["License"]] + [ + c for c in classifiers if c.startswith("License") + ] + license_expression = get_license_expression(declared_license) + other_classifiers = [c for c in classifiers if not c.startswith("License")] + + holder = raw_data["Author"] + holder_contact = raw_data["Author-email"] + copyright_statement = f"Copyright (c) {holder} <{holder_contact}>" + + pkginfo_data = dict( + name=raw_data["Name"], + declared_license=declared_license, + version=raw_data["Version"], + description=raw_data["Summary"], + homepage_url=raw_data["Home-page"], + copyright=copyright_statement, + license_expression=license_expression, + holder=holder, + holder_contact=holder_contact, + keywords=raw_data["Keywords"], + classifiers=other_classifiers, + ) + + return self.update(pkginfo_data, keep_extra=True) + + def update_from_other_dist(self, dist): + """ + Update self using data from another dist + """ + return self.update(dist.get_updatable_data()) + + def get_updatable_data(self, data=None): + data = data or self.to_dict() + return {k: v for k, v in data.items() if v and k in self.updatable_fields} + + def update(self, data, overwrite=False, keep_extra=True): + """ + Update self with a mapping of `data`. Keep unknown data as extra_data if + `keep_extra` is True. If `overwrite` is True, overwrite self with `data` + Return True if any data was updated, False otherwise. Raise an exception + if there are key data conflicts. + """ + package_url = data.get("package_url") + if package_url: + purl_from_data = packageurl.PackageURL.from_string(package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) + if purl_from_data != purl_from_self: + print( + f"Invalid dist update attempt, no same same purl with dist: " + f"{self} using data {data}." + ) + return + + data.pop("about_resource", None) + dl = data.pop("download_url", None) + if dl: + data["path_or_url"] = dl + + updated = False + extra = {} + for k, v in data.items(): + if isinstance(v, str): + v = v.strip() + if not v: + continue + + if hasattr(self, k): + value = getattr(self, k, None) + if not value or (overwrite and value != v): + try: + setattr(self, k, v) + except Exception as e: + raise Exception(f"{self}, {k}, {v}") from e + updated = True + + elif keep_extra: + # note that we always overwrite extra + extra[k] = v + updated = True + + self.extra_data.update(extra) + + return updated + + +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + +class InvalidDistributionFilename(Exception): + pass + + +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + +@attr.attributes +class Sdist(Distribution): + extension = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="File extension, including leading dot."), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a Sdist object built from a filename. + Raise an exception if this is not a valid sdist filename + """ + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: + raise InvalidDistributionFilename(filename) + + name, version, extension = name_ver_ext + + return cls( + type="pypi", + name=name, + version=version, + extension=extension, + filename=filename, + ) + + def to_filename(self): + """ + Return an sdist filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + return f"{self.name}-{self.version}.{self.extension}" + + +@attr.attributes +class Wheel(Distribution): + """ + Represents a wheel file. + + Copied and heavily modified from pip-20.3.1 copied from pip-20.3.1 + pip/_internal/models/wheel.py + + name: pip compatibility tags + version: 20.3.1 + download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py + copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + license_expression: mit + notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + + Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + """ + + get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, + ).match + + build = attr.ib( + type=str, + default="", + metadata=dict(help="Python wheel build."), + ) + + python_versions = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel Python version tags."), + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel ABI tags."), + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel platform tags."), + ) + + tags = attr.ib( + repr=False, + type=set, + default=attr.Factory(set), + metadata=dict(help="Set of all tags for this wheel."), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a wheel object built from a filename. + Raise an exception if this is not a valid wheel filename + """ + wheel_info = cls.get_wheel_from_filename(filename) + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group("name").replace("_", "-") + # we'll assume "_" means "-" due to wheel naming scheme + # (https://github.com/pypa/pip/issues/1150) + version = wheel_info.group("ver").replace("_", "-") + build = wheel_info.group("build") + python_versions = wheel_info.group("pyvers").split(".") + abis = wheel_info.group("abis").split(".") + platforms = wheel_info.group("plats").split(".") + + # All the tag combinations from this file + tags = { + packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms + } + + return cls( + filename=filename, + type="pypi", + name=name, + version=version, + build=build, + python_versions=python_versions, + abis=abis, + platforms=platforms, + tags=tags, + ) + + def is_supported_by_tags(self, tags): + """ + Return True is this wheel is compatible with one of a list of PEP 425 tags. + """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) + return not self.tags.isdisjoint(tags) + + def to_filename(self): + """ + Return a wheel filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + build = f"-{self.build}" if self.build else "" + pyvers = ".".join(self.python_versions) + abis = ".".join(self.abis) + plats = ".".join(self.platforms) + return f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" + + def is_pure(self): + """ + Return True if wheel `filename` is for a "pure" wheel e.g. a wheel that + runs on all Pythons 3 and all OSes. + + For example:: + + >>> Wheel.from_filename('aboutcode_toolkit-5.1.0-py2.py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py2-none-any.whl').is_pure() + False + >>> Wheel.from_filename('bitarray-0.8.1-cp36-cp36m-win_amd64.whl').is_pure() + False + >>> Wheel.from_filename('extractcode_7z-16.5-py2.py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-cp36-none-any.whl').is_pure() + False + >>> Wheel.from_filename('foo-4.7.1-py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() + False + """ + return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms + + +def is_pure_wheel(filename): + try: + return Wheel.from_filename(filename).is_pure() + except: + return False + + +@attr.attributes +class PypiPackage(NameVer): + """ + A Python package contains one or more wheels and one source distribution + from a repository. + """ + + sdist = attr.ib( + repr=False, + type=Sdist, + default=None, + metadata=dict(help="Sdist source distribution for this package."), + ) + + wheels = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help="List of Wheel for this package"), + ) + + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): + """ + Yield all the Wheel of this package supported and compatible with the + Environment `environment`. + """ + envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) + for wheel in self.wheels: + if wheel.is_supported_by_tags(envt_tags): + yield wheel + + @classmethod + def package_from_dists(cls, dists): + """ + Return a new PypiPackage built from an iterable of Wheels and Sdist + objects all for the same package name and version. + + For example: + >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp38'], abis=['cp38m'], + ... platforms=['linux_x86_64']) + >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp38'], abis=['cp38m'], + ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) + >>> sd = Sdist(name='bitarray', version='0.8.1') + >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) + >>> assert package.name == 'bitarray' + >>> assert package.version == '0.8.1' + >>> assert package.sdist == sd + >>> assert package.wheels == [w1, w2] + """ + dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") + if not dists: + return + + reference_dist = dists[0] + normalized_name = reference_dist.normalized_name + version = reference_dist.version + + package = PypiPackage(name=normalized_name, version=version) + + for dist in dists: + if dist.normalized_name != normalized_name: + if TRACE: + print( + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" + ) + continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue + + if isinstance(dist, Sdist): + package.sdist = dist + + elif isinstance(dist, Wheel): + package.wheels.append(dist) + + else: + raise Exception(f"Unknown distribution type: {dist}") + + if TRACE_DEEP: + print(f"package_from_dists: {package}") + + return package + + @classmethod + def packages_from_dir(cls, directory): + """ + Yield PypiPackages built from files found in at directory path. + """ + base = os.path.abspath(directory) + + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) + + @classmethod + def packages_from_many_paths_or_urls(cls, paths_or_urls): + """ + Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. + """ + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + + dists = NameVer.sorted(dists) + + for _projver, dists_of_package in itertools.groupby( + dists, + key=NameVer.sortable_name_version, + ): + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package + + @classmethod + def dists_from_paths_or_urls(cls, paths_or_urls): + """ + Return a list of Distribution given a list of + ``paths_or_urls`` to wheels or source distributions. + + Each Distribution receives two extra attributes: + - the path_or_url it was created from + - its filename + + For example: + >>> paths_or_urls =''' + ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl + ... https://example.com/bar/bitarray-0.8.1.tar.gz + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) + >>> for r in results: + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 + """ + dists = [] + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] + for path_or_url in installable: + try: + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, + ) + except InvalidDistributionFilename: + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") + continue + return dists + + def get_distributions(self): + """ + Yield all distributions available for this PypiPackage + """ + if self.sdist: + yield self.sdist + for wheel in self.wheels: + yield wheel + + def get_url_for_filename(self, filename): + """ + Return the URL for this filename or None. + """ + for dist in self.get_distributions(): + if dist.filename == filename: + return dist.path_or_url + + +@attr.attributes +class Environment: + """ + An Environment describes a target installation environment with its + supported Python version, ABI, platform, implementation and related + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. + """ + + python_version = attr.ib( + type=str, + default="", + metadata=dict(help="Python version supported by this environment."), + ) + + operating_system = attr.ib( + type=str, + default="", + metadata=dict(help="operating system supported by this environment."), + ) + + implementation = attr.ib( + type=str, + default="cp", + metadata=dict(help="Python implementation supported by this environment."), + repr=False, + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of platform tags supported by this environment."), + repr=False, + ) + + @classmethod + def from_pyver_and_os(cls, python_version, operating_system): + if "." in python_version: + python_version = "".join(python_version.split(".")) + + return cls( + python_version=python_version, + implementation="cp", + abis=ABIS_BY_PYTHON_VERSION[python_version], + platforms=PLATFORMS_BY_OS[operating_system], + operating_system=operating_system, + ) + + def get_pip_cli_options(self): + """ + Return a list of pip download command line options for this environment. + """ + options = [ + "--python-version", + self.python_version, + "--implementation", + self.implementation, + ] + for abi in self.abis: + options.extend(["--abi", abi]) + + for platform in self.platforms: + options.extend(["--platform", platform]) + + return options + + def tags(self): + """ + Return a set of all the PEP425 tags supported by this environment. + """ + return set( + utils_pip_compatibility_tags.get_supported( + version=self.python_version or None, + impl=self.implementation or None, + platforms=self.platforms or None, + abis=self.abis or None, + ) + ) + + +################################################################################ +# +# PyPI repo and link index for package wheels and sources +# +################################################################################ + + +@attr.attributes +class PypiSimpleRepository: + """ + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. + """ + + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), + ) + + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( + type=dict, + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), + ) + + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), + ) + + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), + ) + + def _get_package_versions_map(self, name): + """ + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest + """ + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions + + def get_package_versions(self, name): + """ + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. + """ + return dict(self._get_package_versions_map(name)) + + def get_package_version(self, name, version=None): + """ + Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. + """ + if not version: + versions = list(self._get_package_versions_map(name).values()) + # return the latest version + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) + + def fetch_links(self, normalized_name): + """ + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. + """ + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + return links + + +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} + + +@attr.attributes +class LinksRepository: + """ + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. + """ + + url = attr.ib( + type=str, + default="", + metadata=dict(help="Links directory URL"), + ) + + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of links available in this repo"), + ) + + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), + ) + + def __attrs_post_init__(self): + if not self.links: + self.links = self.find_links() + + def find_links(self, _CACHE=[]): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + if _CACHE: + return _CACHE + + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() + + if TRACE_DEEP: + print(f"Base URL {base_url}") + + text = CACHE.get( + path_or_url=links_url, + as_text=True, + force=not self.use_cached_index, + ) + + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue + + plink = urllib.parse.urlsplit(link) + + if plink.scheme: + # full URL kept as-is + url = link + + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" + + else: + # relative link + url = f"{links_url}/{link}" + + if TRACE_DEEP: + print(f"Adding URL: {url}") + + links.append(url) + + if TRACE: + print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) + return links + + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) + return _LINKS_REPO[url] + + +################################################################################ +# Globals for remote repos to be lazily created and cached on first use for the +# life of the session together with some convenience functions. +################################################################################ + + +def get_local_packages(directory=THIRDPARTY_DIR): + """ + Return the list of all PypiPackage objects built from a local directory. Return + an empty list if the package cannot be found. + """ + return list(PypiPackage.packages_from_dir(directory=directory)) + + +################################################################################ +# +# Basic file and URL-based operations using a persistent file-based Cache +# +################################################################################ + + +@attr.attributes +class Cache: + """ + A simple file-based cache based only on a filename presence. + This is used to avoid impolite fetching from remote locations. + """ + + directory = attr.ib(type=str, default=CACHE_THIRDPARTY_DIR) + + def __attrs_post_init__(self): + os.makedirs(self.directory, exist_ok=True) + + def get(self, path_or_url, as_text=True, force=False): + """ + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. + """ + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) + + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") + content = get_file_content(path_or_url=path_or_url, as_text=as_text) + wmode = "w" if as_text else "wb" + with open(cached, wmode) as fo: + fo.write(content) + return content + else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") + return get_local_file_content(path=cached, as_text=as_text) + + +CACHE = Cache() + + +def get_file_content(path_or_url, as_text=True): + """ + Fetch and return the content at `path_or_url` from either a local path or a + remote URL. Return the content as bytes is `as_text` is False. + """ + if path_or_url.startswith("https://"): + if TRACE_DEEP: + print(f"Fetching: {path_or_url}") + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + return content + + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + + else: + raise Exception(f"Unsupported URL scheme: {path_or_url}") + + +def get_local_file_content(path, as_text=True): + """ + Return the content at `url` as text. Return the content as bytes is + `as_text` is False. + """ + if path.startswith("file://"): + path = path[7:] + + mode = "r" if as_text else "rb" + with open(path, mode) as fo: + return fo.read() + + +class RemoteNotFetchedException(Exception): + pass + + +def get_remote_file_content( + url, + as_text=True, + headers_only=False, + headers=None, + _delay=0, +): + """ + Fetch and return a tuple of (headers, content) at `url`. Return content as a + text string if `as_text` is True. Otherwise return the content as bytes. + + If `header_only` is True, return only (headers, None). Headers is a mapping + of HTTP headers. + Retries multiple times to fetch if there is a HTTP 429 throttling response + and this with an increasing delay. + """ + time.sleep(_delay) + headers = headers or {} + # using a GET with stream=True ensure we get the the final header from + # several redirects and that we can ignore content there. A HEAD request may + # not get us this last header + print(f" DOWNLOADING: {url}") + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + status = response.status_code + if status != requests.codes.ok: # NOQA + if status == 429 and _delay < 20: + # too many requests: start some exponential delay + increased_delay = (_delay * 2) or 1 + + return get_remote_file_content( + url, + as_text=as_text, + headers_only=headers_only, + _delay=increased_delay, + ) + + else: + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") + + if headers_only: + return response.headers, None + + return response.headers, response.text if as_text else response.content + + +def fetch_and_save( + path_or_url, + dest_dir, + filename, + as_text=True, +): + """ + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. + """ + content = CACHE.get( + path_or_url=path_or_url, + as_text=as_text, + ) + output = os.path.join(dest_dir, filename) + wmode = "w" if as_text else "wb" + with open(output, wmode) as fo: + fo.write(content) + return content + + +################################################################################ +# +# Functions to update or fetch ABOUT and license files +# +################################################################################ + + +def clean_about_files( + dest_dir=THIRDPARTY_DIR, +): + """ + Given a thirdparty dir, clean ABOUT files + """ + local_packages = get_local_packages(directory=dest_dir) + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) + + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) + + +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): + """ + Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using + best efforts: + + - use existing ABOUT files + - try to load existing remote ABOUT files + - derive from existing distribution with same name and latest version that + would have such ABOUT file + - extract ABOUT file data from distributions PKGINFO or METADATA files + + Use available existing on-disk cached index if use_cached_index is True. + """ + + def get_other_dists(_package, _dist): + """ + Return a list of all the dists from `_package` that are not the `_dist` + object + """ + return [d for d in _package.get_distributions() if d != _dist] + + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # lets try to get from another dist of the same local package + for otherd in get_other_dists(local_package, local_dist): + updated = local_dist.update_from_other_dist(otherd) + if updated and local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # try to get another version of the same package that is not our version + other_local_packages = [ + p + for p in packages_by_name[local_package.name] + if p.version != local_package.version + ] + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) + for latest_local_dist in latest_local_dists: + latest_local_dist.load_about_data(dest_dir=dest_dir) + if not latest_local_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(latest_local_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) + continue + + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + + other_remote_packages = [ + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) + continue + + # try to get data from pkginfo (no license though) + local_dist.load_pkginfo_data(dest_dir=dest_dir) + + # FIXME: save as this is the last resort for now in all cases + # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir) + + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) + + if not local_dist.has_key_metadata(): + print(f"Unable to add essential ABOUT data for: {local_dist}") + if lic_errs: + lic_errs = "\n".join(lic_errs) + print(f"Failed to fetch some licenses:: {lic_errs}") + + +################################################################################ +# +# Functions to build new Python wheels including native on multiple OSes +# +################################################################################ + + +def call(args, verbose=TRACE): + """ + Call args in a subprocess and display output on the fly if ``trace`` is True. + Return a tuple of (returncode, stdout, stderr) + """ + if TRACE_DEEP: + print("Calling:", " ".join(args)) + with subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" + ) as process: + stdouts = [] + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + stdouts.append(line) + if verbose: + print(line.rstrip(), flush=True) + + stdout, stderr = process.communicate() + if not stdout.strip(): + stdout = "\n".join(stdouts) + return process.returncode, stdout, stderr + + +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, + dest_dir=THIRDPARTY_DIR, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, +): + """ + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", + ] + + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") + + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) + + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) + + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) + + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") + + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error + + +################################################################################ +# +# Functions to check for problems +# +################################################################################ + + +def check_about(dest_dir=THIRDPARTY_DIR): + try: + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + except subprocess.CalledProcessError as cpe: + print() + print("Invalid ABOUT files:") + print(cpe.output.decode("utf-8", errors="replace")) + + +def find_problems( + dest_dir=THIRDPARTY_DIR, + report_missing_sources=False, + report_missing_wheels=False, +): + """ + Print the problems found in `dest_dir`. + """ + + local_packages = get_local_packages(directory=dest_dir) + + for package in local_packages: + if report_missing_sources and not package.sdist: + print(f"{package.name}=={package.version}: Missing source distribution.") + if report_missing_wheels and not package.wheels: + print(f"{package.name}=={package.version}: Missing wheels.") + + for dist in package.get_distributions(): + dist.load_about_data(dest_dir=dest_dir) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) + if not dist.has_key_metadata(): + print(f" Missing key ABOUT data in file://{abpth}") + if "classifiers" in dist.extra_data: + print(f" Dangling classifiers data in file://{abpth}") + if not dist.validate_checksums(dest_dir): + print(f" Invalid checksums in file://{abpth}") + if not dist.sha1 and dist.md5: + print(f" Missing checksums in file://{abpth}") + + check_about(dest_dir=dest_dir) + + +def get_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ + if not declared_licenses: + return + try: + from packagedcode.licensing import get_only_expression_from_extracted_license + + return get_only_expression_from_extracted_license(declared_licenses) + except ImportError: + # Scancode is not installed, clean and join all the licenses + lics = [python_safe_name(l).lower() for l in declared_licenses] + return " AND ".join(lics).lower() diff --git a/etc/scripts/utils_thirdparty.py.ABOUT b/etc/scripts/utils_thirdparty.py.ABOUT new file mode 100644 index 0000000..8480349 --- /dev/null +++ b/etc/scripts/utils_thirdparty.py.ABOUT @@ -0,0 +1,15 @@ +about_resource: utils_thirdparty.py +package_url: pkg:github.com/pypa/pip/@20.3.1#src/pip/_internal/models/wheel.py +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/models/wheel.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + The models code has been heavily inspired from the ISC-licensed packaging-dists + https://github.com/uranusjr/packaging-dists by Tzu-ping Chung + \ No newline at end of file diff --git a/LICENSE.txt b/mit.LICENSE similarity index 100% rename from LICENSE.txt rename to mit.LICENSE diff --git a/pkginfo2/_compat.py b/pkginfo2/_compat.py deleted file mode 100644 index ff32652..0000000 --- a/pkginfo2/_compat.py +++ /dev/null @@ -1,34 +0,0 @@ -try: - STRING_TYPES = (str, unicode) -except NameError: #pragma NO COVER Python >= 3.0 - STRING_TYPES = (str,) - -try: - u = unicode -except NameError: #pragma NO COVER Python >= 3.0 - u = str - b = bytes -else: #pragma NO COVER Python < 3.0 - b = str - -try: - from StringIO import StringIO -except ImportError: #pragma NO COVER Python >= 3.0 - from io import StringIO - from io import BytesIO -else: #pragma NO COVER Python < 3.0 - BytesIO = StringIO - - -def must_decode(value): #pragma NO COVER - if type(value) is bytes: - try: - return value.decode('utf-8') - except UnicodeDecodeError: - return value.decode('latin1') - return value - -def must_encode(value): #pragma NO COVER - if type(value) is u: - return value.encode('utf-8') - return value diff --git a/pkginfo2/distribution.py b/pkginfo2/distribution.py deleted file mode 100644 index 392bb1b..0000000 --- a/pkginfo2/distribution.py +++ /dev/null @@ -1,168 +0,0 @@ -from email.parser import Parser - -from ._compat import StringIO -from ._compat import must_decode - - -def parse(fp): - return Parser().parse(fp) -def get(msg, header): - return _collapse_leading_ws(header, msg.get(header)) -def get_all(msg, header): - return [_collapse_leading_ws(header, x) for x in msg.get_all(header)] - -def _collapse_leading_ws(header, txt): - """ - ``Description`` header must preserve newlines; all others need not - """ - if header.lower() == 'description': # preserve newlines - return '\n'.join([x[8:] if x.startswith(' ' * 8) else x - for x in txt.strip().splitlines()]) - else: - return ' '.join([x.strip() for x in txt.splitlines()]) - - -HEADER_ATTRS_1_0 = ( # PEP 241 - ('Metadata-Version', 'metadata_version', False), - ('Name', 'name', False), - ('Version', 'version', False), - ('Platform', 'platforms', True), - ('Supported-Platform', 'supported_platforms', True), - ('Summary', 'summary', False), - ('Description', 'description', False), - ('Keywords', 'keywords', False), - ('Home-Page', 'home_page', False), - ('Author', 'author', False), - ('Author-email', 'author_email', False), - ('License', 'license', False), -) - -HEADER_ATTRS_1_1 = HEADER_ATTRS_1_0 + ( # PEP 314 - ('Classifier', 'classifiers', True), - ('Download-URL', 'download_url', False), - ('Requires', 'requires', True), - ('Provides', 'provides', True), - ('Obsoletes', 'obsoletes', True), -) - -HEADER_ATTRS_1_2 = HEADER_ATTRS_1_1 + ( # PEP 345 - ('Maintainer', 'maintainer', False), - ('Maintainer-email', 'maintainer_email', False), - ('Requires-Python', 'requires_python', False), - ('Requires-External', 'requires_external', True), - ('Requires-Dist', 'requires_dist', True), - ('Provides-Dist', 'provides_dist', True), - ('Obsoletes-Dist', 'obsoletes_dist', True), - ('Project-URL', 'project_urls', True), -) - -HEADER_ATTRS_2_0 = HEADER_ATTRS_1_2 #XXX PEP 426? - -HEADER_ATTRS_2_1 = HEADER_ATTRS_1_2 + ( # PEP 566 - ('Provides-Extra', 'provides_extras', True), - ('Description-Content-Type', 'description_content_type', False) -) - -HEADER_ATTRS_2_2 = HEADER_ATTRS_2_1 + ( # PEP 643 - ('Dynamic', 'dynamic', True), -) - -HEADER_ATTRS_2_4 = HEADER_ATTRS_2_2 + ( # PEP 639 - ('License-Expression', 'license_expression', False), - ('License-Files', 'license_files', True), -) - -HEADER_ATTRS = { - '1.0': HEADER_ATTRS_1_0, - '1.1': HEADER_ATTRS_1_1, - '1.2': HEADER_ATTRS_1_2, - '2.0': HEADER_ATTRS_2_0, - '2.1': HEADER_ATTRS_2_1, - '2.2': HEADER_ATTRS_2_2, - '2.3': HEADER_ATTRS_2_2, - '2.4': HEADER_ATTRS_2_4, -} - -class Distribution(object): - metadata_version = None - # version 1.0 - name = None - version = None - platforms = () - supported_platforms = () - summary = None - description = None - keywords = None - home_page = None - download_url = None - author = None - author_email = None - license = None - # version 1.1 - classifiers = () - requires = () - provides = () - obsoletes = () - # version 1.2 - maintainer = None - maintainer_email = None - requires_python = None - requires_external = () - requires_dist = () - provides_dist = () - obsoletes_dist = () - project_urls = () - # version 2.1 - provides_extras = () - description_content_type = None - # version 2.2 - dynamic = () - # version 2.4 - license_expression = None - license_files = () - - def extractMetadata(self): - data = self.read() - self.parse(data) - - def read(self): - raise NotImplementedError - - def _getHeaderAttrs(self): - if self.metadata_version in HEADER_ATTRS: - return HEADER_ATTRS[self.metadata_version] - else: - # If the specific version is not available, use the latest version - return HEADER_ATTRS[HEADER_ATTRS.keys()[-1]] - - def parse(self, data): - fp = StringIO(must_decode(data)) - msg = parse(fp) - - if 'Metadata-Version' in msg and self.metadata_version is None: - value = get(msg, 'Metadata-Version') - metadata_version = self.metadata_version = value - - for header_name, attr_name, multiple in self._getHeaderAttrs(): - - if attr_name == 'metadata_version': - continue - - if header_name in msg: - if multiple: - values = get_all(msg, header_name) - setattr(self, attr_name, values) - else: - value = get(msg, header_name) - if value != 'UNKNOWN': - setattr(self, attr_name, value) - - body = msg.get_payload() - if body: - setattr(self, 'description', body) - - def __iter__(self): - for header_name, attr_name, multiple in self._getHeaderAttrs(): - yield attr_name - - iterkeys = __iter__ diff --git a/pkginfo2/tests/__init__.py b/pkginfo2/tests/__init__.py deleted file mode 100644 index 409052f..0000000 --- a/pkginfo2/tests/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# requirements - - -def _checkSample(testcase, installed): - try: - import pkg_resources - except ImportError: # no setuptools :( - pass - else: - version = pkg_resources.require('pkginfo2')[0].version - testcase.assertEqual(installed.version, version) - testcase.assertEqual(installed.name, 'pkginfo2') - testcase.assertEqual(installed.keywords, - 'distribution sdist installed metadata' ) - testcase.assertEqual(list(installed.supported_platforms), []) - -def _checkClassifiers(testcase, installed): - testcase.assertEqual(list(installed.classifiers), - [ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Software Distribution', - ]) - - -def _defaultMetadataVersion(): - return '2.1' diff --git a/pkginfo2/tests/funny/funny.egg-info b/pkginfo2/tests/funny/funny.egg-info deleted file mode 100644 index d3f79ef..0000000 --- a/pkginfo2/tests/funny/funny.egg-info +++ /dev/null @@ -1,3 +0,0 @@ -Metadata-Version: 1.0 -Name: funny -Version: 0.1 diff --git a/pkginfo2/tests/test_distribution.py b/pkginfo2/tests/test_distribution.py deleted file mode 100644 index c0344a2..0000000 --- a/pkginfo2/tests/test_distribution.py +++ /dev/null @@ -1,450 +0,0 @@ -import unittest - -class DistributionTests(unittest.TestCase): - - def _getTargetClass(self): - from pkginfo2.distribution import Distribution - return Distribution - - def _makeOne(self, metadata_version='1.0'): - dist = self._getTargetClass()() - if metadata_version is not None: - dist.metadata_version = metadata_version - return dist - - def test_ctor_defaults(self): - sdist = self._makeOne(None) - self.assertEqual(sdist.metadata_version, None) - # version 1.0 - self.assertEqual(sdist.name, None) - self.assertEqual(sdist.version, None) - self.assertEqual(sdist.platforms, ()) - self.assertEqual(sdist.supported_platforms, ()) - self.assertEqual(sdist.summary, None) - self.assertEqual(sdist.description, None) - self.assertEqual(sdist.keywords, None) - self.assertEqual(sdist.home_page, None) - self.assertEqual(sdist.download_url, None) - self.assertEqual(sdist.author, None) - self.assertEqual(sdist.author_email, None) - self.assertEqual(sdist.license, None) - # version 1.1 - self.assertEqual(sdist.classifiers, ()) - self.assertEqual(sdist.requires, ()) - self.assertEqual(sdist.provides, ()) - self.assertEqual(sdist.obsoletes, ()) - # version 1.2 - self.assertEqual(sdist.maintainer, None) - self.assertEqual(sdist.maintainer_email, None) - self.assertEqual(sdist.requires_python, None) - self.assertEqual(sdist.requires_external, ()) - self.assertEqual(sdist.requires_dist, ()) - self.assertEqual(sdist.provides_dist, ()) - self.assertEqual(sdist.obsoletes_dist, ()) - self.assertEqual(sdist.project_urls, ()) - # version 2.1 - self.assertEqual(sdist.provides_extras, ()) - self.assertEqual(sdist.description_content_type, None) - # version 2.2 - self.assertEqual(sdist.dynamic, ()) - - def test_extractMetadata_raises_NotImplementedError(self): - # 'extractMetadata' calls 'read', which subclasses must override. - dist = self._makeOne(None) - self.assertRaises(NotImplementedError, dist.extractMetadata) - - def test_read_raises_NotImplementedError(self): - # Subclasses must override 'read'. - dist = self._makeOne(None) - self.assertRaises(NotImplementedError, dist.read) - - def test_parse_given_unicode(self): - from pkginfo2._compat import u - dist = self._makeOne() - dist.parse(u('Metadata-Version: 1.0\nName: lp722928_c3')) # no raise - - def test_parse_Metadata_Version_1_0(self): - from pkginfo2.distribution import HEADER_ATTRS_1_0 - dist = self._makeOne(None) - dist.parse('Metadata-Version: 1.0') - self.assertEqual(dist.metadata_version, '1.0') - self.assertEqual(list(dist), - [x[1] for x in HEADER_ATTRS_1_0]) - - def test_parse_Metadata_Version_1_1(self): - from pkginfo2.distribution import HEADER_ATTRS_1_1 - dist = self._makeOne(None) - dist.parse('Metadata-Version: 1.1') - self.assertEqual(dist.metadata_version, '1.1') - self.assertEqual(list(dist), - [x[1] for x in HEADER_ATTRS_1_1]) - - def test_parse_Metadata_Version_1_2(self): - from pkginfo2.distribution import HEADER_ATTRS_1_2 - dist = self._makeOne(None) - dist.parse('Metadata-Version: 1.2') - self.assertEqual(dist.metadata_version, '1.2') - self.assertEqual(list(dist), - [x[1] for x in HEADER_ATTRS_1_2]) - - def test_parse_Metadata_Version_2_1(self): - from pkginfo2.distribution import HEADER_ATTRS_2_1 - dist = self._makeOne(None) - dist.parse('Metadata-Version: 2.1') - self.assertEqual(dist.metadata_version, '2.1') - self.assertEqual(list(dist), - [x[1] for x in HEADER_ATTRS_2_1]) - - def test_parse_Metadata_Version_2_2(self): - from pkginfo2.distribution import HEADER_ATTRS_2_2 - dist = self._makeOne(None) - dist.parse('Metadata-Version: 2.2') - self.assertEqual(dist.metadata_version, '2.2') - self.assertEqual(list(dist), - [x[1] for x in HEADER_ATTRS_2_2]) - - def test_parse_Metadata_Version_unknown(self): - dist = self._makeOne(None) - dist.parse('Metadata-Version: 1.3') - self.assertEqual(dist.metadata_version, '1.3') - self.assertEqual(list(dist), []) - - def test_parse_Metadata_Version_override(self): - dist = self._makeOne('1.2') - dist.parse('Metadata-Version: 1.0') - self.assertEqual(dist.metadata_version, '1.2') - - def test_parse_Name(self): - dist = self._makeOne() - dist.parse('Name: foobar') - self.assertEqual(dist.name, 'foobar') - - def test_parse_Version(self): - dist = self._makeOne() - dist.parse('Version: 2.1.3b5') - self.assertEqual(dist.version, '2.1.3b5') - - def test_parse_Platform_single(self): - dist = self._makeOne() - dist.parse('Platform: Plan9') - self.assertEqual(list(dist.platforms), ['Plan9']) - - def test_parse_Platform_multiple(self): - dist = self._makeOne() - dist.parse('Platform: Plan9\nPlatform: AIX') - self.assertEqual(list(dist.platforms), ['Plan9', 'AIX']) - - def test_parse_Supported_Platform_single(self): - dist = self._makeOne() - dist.parse('Supported-Platform: Plan9') - self.assertEqual(list(dist.supported_platforms), ['Plan9']) - - def test_parse_Supported_Platform_multiple(self): - dist = self._makeOne() - dist.parse('Supported-Platform: i386-win32\n' - 'Supported-Platform: RedHat 7.2') - self.assertEqual(list(dist.supported_platforms), - ['i386-win32', 'RedHat 7.2']) - - def test_parse_Summary(self): - dist = self._makeOne() - dist.parse('Summary: Package for foo') - self.assertEqual(dist.summary, 'Package for foo') - - def test_parse_Description(self): - dist = self._makeOne() - dist.parse('Description: This package enables integration with ' - 'foo servers.') - self.assertEqual(dist.description, - 'This package enables integration with ' - 'foo servers.') - - def test_parse_Description_multiline(self): - dist = self._makeOne() - dist.parse('Description: This package enables integration with\n' - ' foo servers.') - self.assertEqual(dist.description, - 'This package enables integration with\n' - 'foo servers.') - - def test_parse_Description_in_payload(self): - dist = self._makeOne() - dist.parse('Foo: Bar\n' - '\n' - 'This package enables integration with\n' - 'foo servers.') - self.assertEqual(dist.description, - 'This package enables integration with\n' - 'foo servers.') - - def test_parse_Keywords(self): - dist = self._makeOne() - dist.parse('Keywords: bar foo qux') - self.assertEqual(dist.keywords, 'bar foo qux') - - def test_parse_Home_page(self): - dist = self._makeOne() - dist.parse('Home-page: http://example.com/package') - self.assertEqual(dist.home_page, 'http://example.com/package') - - def test_parse_Author(self): - dist = self._makeOne() - dist.parse('Author: J. Phredd Bloggs') - self.assertEqual(dist.author, 'J. Phredd Bloggs') - - def test_parse_Author_Email(self): - dist = self._makeOne() - dist.parse('Author-email: phreddy@example.com') - self.assertEqual(dist.author_email, 'phreddy@example.com') - - def test_parse_License(self): - dist = self._makeOne() - dist.parse('License: Poetic') - self.assertEqual(dist.license, 'Poetic') - - # Metadata version 1.1, defined in PEP 314. - def test_parse_Classifier_single(self): - dist = self._makeOne('1.1') - dist.parse('Classifier: Some :: Silly Thing') - self.assertEqual(list(dist.classifiers), ['Some :: Silly Thing']) - - def test_parse_Classifier_multiple(self): - dist = self._makeOne('1.1') - dist.parse('Classifier: Some :: Silly Thing\n' - 'Classifier: Or :: Other') - self.assertEqual(list(dist.classifiers), - ['Some :: Silly Thing', 'Or :: Other']) - - def test_parse_Download_URL(self): - dist = self._makeOne('1.1') - dist.parse('Download-URL: ' - 'http://example.com/package/mypackage-0.1.zip') - self.assertEqual(dist.download_url, - 'http://example.com/package/mypackage-0.1.zip') - - def test_parse_Requires_single_wo_version(self): - dist = self._makeOne('1.1') - dist.parse('Requires: SpanishInquisition') - self.assertEqual(list(dist.requires), ['SpanishInquisition']) - - def test_parse_Requires_single_w_version(self): - dist = self._makeOne('1.1') - dist.parse('Requires: SpanishInquisition (>=1.3)') - self.assertEqual(list(dist.requires), ['SpanishInquisition (>=1.3)']) - - def test_parse_Requires_multiple(self): - dist = self._makeOne('1.1') - dist.parse('Requires: SpanishInquisition\n' - 'Requires: SillyWalks (1.4)\n' - 'Requires: kniggits (>=2.3,<3.0)') - self.assertEqual(list(dist.requires), - ['SpanishInquisition', - 'SillyWalks (1.4)', - 'kniggits (>=2.3,<3.0)', - ]) - - def test_parse_Provides_single_wo_version(self): - dist = self._makeOne('1.1') - dist.parse('Provides: SillyWalks') - self.assertEqual(list(dist.provides), ['SillyWalks']) - - def test_parse_Provides_single_w_version(self): - dist = self._makeOne('1.1') - dist.parse('Provides: SillyWalks (1.4)') - self.assertEqual(list(dist.provides), ['SillyWalks (1.4)']) - - def test_parse_Provides_multiple(self): - dist = self._makeOne('1.1') - dist.parse('Provides: SillyWalks\n' - 'Provides: DeadlyJoke (3.1.4)') - self.assertEqual(list(dist.provides), - ['SillyWalks', - 'DeadlyJoke (3.1.4)', - ]) - - def test_parse_Obsoletes_single_no_version(self): - dist = self._makeOne('1.1') - dist.parse('Obsoletes: SillyWalks') - self.assertEqual(list(dist.obsoletes), ['SillyWalks']) - - def test_parse_Obsoletes_single_w_version(self): - dist = self._makeOne('1.1') - dist.parse('Obsoletes: SillyWalks (<=1.3)') - self.assertEqual(list(dist.obsoletes), ['SillyWalks (<=1.3)']) - - def test_parse_Obsoletes_multiple(self): - dist = self._makeOne('1.1') - dist.parse('Obsoletes: kniggits\n' - 'Obsoletes: SillyWalks (<=2.0)') - self.assertEqual(list(dist.obsoletes), - ['kniggits', - 'SillyWalks (<=2.0)', - ]) - - - # Metadata version 1.2, defined in PEP 345. - def test_parse_Maintainer(self): - dist = self._makeOne(metadata_version='1.2') - dist.parse('Maintainer: J. Phredd Bloggs') - self.assertEqual(dist.maintainer, 'J. Phredd Bloggs') - - def test_parse_Maintainer_Email(self): - dist = self._makeOne(metadata_version='1.2') - dist.parse('Maintainer-email: phreddy@example.com') - self.assertEqual(dist.maintainer_email, 'phreddy@example.com') - - def test_parse_Requires_Python_single_spec(self): - dist = self._makeOne('1.2') - dist.parse('Requires-Python: >2.4') - self.assertEqual(dist.requires_python, '>2.4') - - def test_parse_Requires_External_single_wo_version(self): - dist = self._makeOne('1.2') - dist.parse('Requires-External: libfoo') - self.assertEqual(list(dist.requires_external), ['libfoo']) - - def test_parse_Requires_External_single_w_version(self): - dist = self._makeOne('1.2') - dist.parse('Requires-External: libfoo (>=1.3)') - self.assertEqual(list(dist.requires_external), ['libfoo (>=1.3)']) - - def test_parse_Requires_External_multiple(self): - dist = self._makeOne('1.2') - dist.parse('Requires-External: libfoo\n' - 'Requires-External: libbar (1.4)\n' - 'Requires-External: libbaz (>=2.3,<3.0)') - self.assertEqual(list(dist.requires_external), - ['libfoo', - 'libbar (1.4)', - 'libbaz (>=2.3,<3.0)', - ]) - - - def test_parse_Requires_Dist_single_wo_version(self): - dist = self._makeOne('1.2') - dist.parse('Requires-Dist: SpanishInquisition') - self.assertEqual(list(dist.requires_dist), ['SpanishInquisition']) - - def test_parse_Requires_Dist_single_w_version(self): - dist = self._makeOne('1.2') - dist.parse('Requires-Dist: SpanishInquisition (>=1.3)') - self.assertEqual(list(dist.requires_dist), - ['SpanishInquisition (>=1.3)']) - - def test_parse_Requires_Dist_single_w_env_marker(self): - dist = self._makeOne('1.2') - dist.parse("Requires-Dist: SpanishInquisition; " - "python_version == '1.4'") - self.assertEqual(list(dist.requires_dist), - ["SpanishInquisition; python_version == '1.4'"]) - - def test_parse_Requires_Dist_multiple(self): - dist = self._makeOne('1.2') - dist.parse("Requires-Dist: SpanishInquisition\n" - "Requires-Dist: SillyWalks; python_version == '1.4'\n" - "Requires-Dist: kniggits (>=2.3,<3.0)") - self.assertEqual(list(dist.requires_dist), - ["SpanishInquisition", - "SillyWalks; python_version == '1.4'", - "kniggits (>=2.3,<3.0)", - ]) - - def test_parse_Provides_Dist_single_wo_version(self): - dist = self._makeOne('1.2') - dist.parse('Provides-Dist: SillyWalks') - self.assertEqual(list(dist.provides_dist), ['SillyWalks']) - - def test_parse_Provides_Dist_single_w_version(self): - dist = self._makeOne('1.2') - dist.parse('Provides-Dist: SillyWalks (1.4)') - self.assertEqual(list(dist.provides_dist), ['SillyWalks (1.4)']) - - def test_parse_Provides_Dist_single_w_env_marker(self): - dist = self._makeOne('1.2') - dist.parse("Provides-Dist: SillyWalks; sys.platform == 'os2'") - self.assertEqual(list(dist.provides_dist), - ["SillyWalks; sys.platform == 'os2'"]) - - def test_parse_Provides_Dist_multiple(self): - dist = self._makeOne('1.2') - dist.parse("Provides-Dist: SillyWalks\n" - "Provides-Dist: SpanishInquisition; sys.platform == 'os2'\n" - "Provides-Dist: DeadlyJoke (3.1.4)") - self.assertEqual(list(dist.provides_dist), - ["SillyWalks", - "SpanishInquisition; sys.platform == 'os2'", - "DeadlyJoke (3.1.4)", - ]) - - def test_parse_Obsoletes_Dist_single_no_version(self): - dist = self._makeOne('1.2') - dist.parse('Obsoletes-Dist: SillyWalks') - self.assertEqual(list(dist.obsoletes_dist), ['SillyWalks']) - - def test_parse_Obsoletes_Dist_single_w_version(self): - dist = self._makeOne('1.2') - dist.parse('Obsoletes-Dist: SillyWalks (<=1.3)') - self.assertEqual(list(dist.obsoletes_dist), ['SillyWalks (<=1.3)']) - - def test_parse_Obsoletes_Dist_single_w_env_marker(self): - dist = self._makeOne('1.2') - dist.parse("Obsoletes-Dist: SillyWalks; sys.platform == 'os2'") - self.assertEqual(list(dist.obsoletes_dist), - ["SillyWalks; sys.platform == 'os2'"]) - - def test_parse_Obsoletes_Dist_multiple(self): - dist = self._makeOne('1.2') - dist.parse("Obsoletes-Dist: kniggits\n" - "Obsoletes-Dist: SillyWalks; sys.platform == 'os2'\n" - "Obsoletes-Dist: DeadlyJoke (<=2.0)\n" - ) - self.assertEqual(list(dist.obsoletes_dist), - ["kniggits", - "SillyWalks; sys.platform == 'os2'", - "DeadlyJoke (<=2.0)", - ]) - - def test_parse_Project_URL_single_no_version(self): - dist = self._makeOne('1.2') - dist.parse('Project-URL: Bug tracker, http://bugs.example.com/grail') - self.assertEqual(list(dist.project_urls), - ['Bug tracker, http://bugs.example.com/grail']) - - def test_parse_Project_URL_multiple(self): - dist = self._makeOne('1.2') - dist.parse('Project-URL: Bug tracker, http://bugs.example.com/grail\n' - 'Project-URL: Repository, http://svn.example.com/grail') - self.assertEqual(list(dist.project_urls), - ['Bug tracker, http://bugs.example.com/grail', - 'Repository, http://svn.example.com/grail', - ]) - - # Metadata version 2.1, defined in PEP 566. - def test_parse_Provides_Extra_single(self): - dist = self._makeOne('2.1') - dist.parse('Provides-Extra: pdf') - self.assertEqual(list(dist.provides_extras), ['pdf']) - - def test_parse_Provides_Extra_multiple(self): - dist = self._makeOne('2.1') - dist.parse('Provides-Extra: pdf\n' - 'Provides-Extra: tex') - self.assertEqual(list(dist.provides_extras), ['pdf', 'tex']) - - def test_parse_Provides_Extra_single(self): - dist = self._makeOne('2.1') - dist.parse('Description-Content-Type: text/plain') - self.assertEqual(dist.description_content_type, 'text/plain') - - # Metadata version 2.2, defined in PEP 643. - def test_parse_Dynamic_single(self): - dist = self._makeOne('2.2') - dist.parse('Dynamic: Platforms') - self.assertEqual(list(dist.dynamic), ['Platforms']) - - def test_parse_Dynamic_multiple(self): - dist = self._makeOne('2.2') - dist.parse('Dynamic: Platforms\n' - 'Dynamic: Supported-Platforms') - self.assertEqual(list(dist.dynamic), - ['Platforms', 'Supported-Platforms']) diff --git a/pkginfo2/tests/wonky/EGG-INFO/PKG-INFO b/pkginfo2/tests/wonky/EGG-INFO/PKG-INFO deleted file mode 100644 index ba75092..0000000 --- a/pkginfo2/tests/wonky/EGG-INFO/PKG-INFO +++ /dev/null @@ -1,2 +0,0 @@ -Metadata-Version: 1.0 -Name: namespaced.wonky diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f106e69 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,129 @@ +[build-system] +requires = ["setuptools >= 50", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +# this is used populated when creating a git archive +# and when there is .git dir and/or there is no git installed +fallback_version = "9999.$Format:%h-%cs$" + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "bin", + "dist", + "build", + "_build", + "etc", + "local", + "ci", + "docs", + "man", + "share", + "samples", + ".cache", + ".settings", + "Include", + "include", + "Lib", + "lib", + "lib64", + "Lib64", + "Scripts", + "thirdparty", + "tmp", + "venv", + ".venv", + "tests/data", + "*/tests/test_data", + ".eggs", + "src/*/data", + "tests/*/data" +] + +python_files = "*.py" + +python_classes = "Test" +python_functions = "test" + +addopts = [ + "-rfExXw", + "--strict-markers", + "--doctest-modules" +] + +[tool.ruff] +line-length = 100 +extend-exclude = [] +target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "tests/**/*.py", + "doc/**/*.py", + "docs/**/*.py", + "*.py", + "." + +] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +# django migrations + "**/migrations/**/*" +] + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ +# "E", # pycodestyle +# "W", # pycodestyle warnings + "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit + "I", # isort +# "C9", # McCabe complexity +] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] + + +[tool.ruff.lint.isort] +force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] + + +[tool.doc8] +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 diff --git a/docs/examples/mypackage-0.1.bogus b/requirements-dev.txt similarity index 100% rename from docs/examples/mypackage-0.1.bogus rename to requirements-dev.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/setup.cfg b/setup.cfg index 6087597..39cb888 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,75 @@ -[easy_install] -zip_ok = false +[metadata] +name = pkginfo2 +version = 30.1.0 +license = MIT + +description = Query metadata from sdists / bdists / installed packages. Safer fork of pkginfo to avoid doing arbitrary imports and eval. +long_description = file:README.rst +long_description_content_type = text/x-rst +url = https://github.com/aboutcode-org/pkginfo2 + +author = Maintained by nexB, Inc. Authored by Tres Seaver, Agendaless Consulting +author_email = tseaver@agendaless.com + +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Operating System :: OS Independent + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 + Programming Language :: Python :: 3 :: Only + Topic :: Software Development :: Libraries :: Python Modules + Topic :: System :: Software Distribution + Topic :: Software Development + Topic :: Utilities + +keywords = + distribution + sdist + installed + metadata + +license_files = + mit.LICENSE + CHANGELOG.rst + CODE_OF_CONDUCT.rst + README.rst + +[options] +python_requires = >=3.10 + +package_dir = + =src +packages = find: +include_package_data = true +zip_safe = false + +install_requires = + + +[options.packages.find] +where = src + + +[options.extras_require] +dev = + pytest >= 7.0.1 + pytest-xdist >= 2 + aboutcode-toolkit >= 7.0.2 + twine + ruff + Sphinx>=5.0.2 + sphinx-rtd-theme>=1.0.0 + sphinx-reredirects >= 0.1.2 + doc8>=0.11.2 + sphinx-autobuild + sphinx-rtd-dark-mode>=1.3.0 + sphinx-copybutton + +[options.entry_points] +console_scripts = + pkginfo2 = pkginfo2.commandline:main diff --git a/setup.py b/setup.py index ab43d08..bac24a4 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,6 @@ -import os +#!/usr/bin/env python -from setuptools import setup +import setuptools -extras = { - 'test_suite': 'pkginfo2.tests', - 'zip_safe': False, -} - -here = os.path.abspath(os.path.dirname(__file__)) -README = open(os.path.join(here, 'README.txt')).read() -CHANGES = open(os.path.join(here, 'CHANGES.txt')).read() - -setup( - name='pkginfo2', - version='30.0.0', - description='Query metadata from sdists / bdists / installed packages. Safer fork of pkginfo to avoid doing arbitrary imports and eval()', - platforms=['Unix', 'Windows'], - long_description='\n\n'.join([README, CHANGES]), - long_description_content_type='text/x-rst', - keywords='distribution sdist installed metadata', - url='https://github.com/aboutcode-org/pkginfo2', - author='Maintained by nexB, Inc. Authored by Tres Seaver, Agendaless Consulting', - author_email='tseaver@agendaless.com', - license='MIT', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Software Distribution', - ], - entry_points={ - 'console_scripts': [ - 'pkginfo2 = pkginfo2.commandline:main', - ] - }, - packages=['pkginfo2', 'pkginfo2.tests'], - **extras -) +if __name__ == "__main__": + setuptools.setup() diff --git a/pkginfo2/__init__.py b/src/pkginfo2/__init__.py similarity index 100% rename from pkginfo2/__init__.py rename to src/pkginfo2/__init__.py diff --git a/src/pkginfo2/_compat.py b/src/pkginfo2/_compat.py new file mode 100644 index 0000000..fc37e63 --- /dev/null +++ b/src/pkginfo2/_compat.py @@ -0,0 +1,35 @@ +try: + STRING_TYPES = (str, unicode) +except NameError: # pragma NO COVER Python >= 3.0 + STRING_TYPES = (str,) + +try: + u = unicode +except NameError: # pragma NO COVER Python >= 3.0 + u = str + b = bytes +else: # pragma NO COVER Python < 3.0 + b = str + +try: + from StringIO import StringIO +except ImportError: # pragma NO COVER Python >= 3.0 + from io import StringIO + from io import BytesIO +else: # pragma NO COVER Python < 3.0 + BytesIO = StringIO + + +def must_decode(value): # pragma NO COVER + if type(value) is bytes: + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return value.decode("latin1") + return value + + +def must_encode(value): # pragma NO COVER + if type(value) is u: + return value.encode("utf-8") + return value diff --git a/pkginfo2/bdist.py b/src/pkginfo2/bdist.py similarity index 62% rename from pkginfo2/bdist.py rename to src/pkginfo2/bdist.py index 71ad9cf..0279ffc 100644 --- a/pkginfo2/bdist.py +++ b/src/pkginfo2/bdist.py @@ -3,37 +3,36 @@ from .distribution import Distribution -class BDist(Distribution): +class BDist(Distribution): def __init__(self, filename, metadata_version=None): self.filename = filename self.metadata_version = metadata_version self.extractMetadata() def read(self): - fqn = os.path.abspath( - os.path.normpath(self.filename)) + fqn = os.path.abspath(os.path.normpath(self.filename)) if not os.path.exists(fqn): - raise ValueError('No such file: %s' % fqn) + raise ValueError("No such file: %s" % fqn) - if fqn.endswith('.egg'): + if fqn.endswith(".egg"): archive = zipfile.ZipFile(fqn) names = archive.namelist() + def read_file(name): return archive.read(name) else: - raise ValueError('Not a known archive format: %s' % fqn) + raise ValueError("Not a known archive format: %s" % fqn) try: - tuples = [x.split('/') for x in names if 'PKG-INFO' in x] + tuples = [x.split("/") for x in names if "PKG-INFO" in x] schwarz = sorted([(len(x), x) for x in tuples]) for path in [x[1] for x in schwarz]: - candidate = '/'.join(path) + candidate = "/".join(path) data = read_file(candidate) - if b'Metadata-Version' in data: + if b"Metadata-Version" in data: return data finally: archive.close() - raise ValueError('No PKG-INFO in archive: %s' % fqn) - + raise ValueError("No PKG-INFO in archive: %s" % fqn) diff --git a/pkginfo2/commandline.py b/src/pkginfo2/commandline.py similarity index 61% rename from pkginfo2/commandline.py rename to src/pkginfo2/commandline.py index f3da8da..a647388 100644 --- a/pkginfo2/commandline.py +++ b/src/pkginfo2/commandline.py @@ -1,4 +1,5 @@ -"""Print the metadata for one or more Python package distributions. +""" +Print the metadata for one or more Python package distributions. Usage: %prog [options] path+ @@ -16,6 +17,7 @@ o an installed package: in this case, 'path' should be the importable name of the package. """ + try: from configparser import ConfigParser except ImportError: # pragma: NO COVER @@ -33,71 +35,108 @@ def _parse_options(args=None): parser = optparse.OptionParser(usage=__doc__) - parser.add_option("-m", "--metadata-version", default=None, - help="Override metadata version") - - parser.add_option("-f", "--field", dest="fields", action="append", - help="Specify an output field (repeatable)", - ) - - parser.add_option("-d", "--download-url-prefix", - dest="download_url_prefix", - help="Download URL prefix", - ) - - parser.add_option("--simple", dest="output", action="store_const", - const='simple', default='simple', - help="Output as simple key-value pairs", - ) - - parser.add_option("-s", "--skip", dest="skip", action="store_true", - default=True, - help="Skip missing values in simple output", - ) - - parser.add_option("-S", "--no-skip", dest="skip", action="store_false", - help="Don't skip missing values in simple output", - ) - - parser.add_option("--single", dest="output", action="store_const", - const='single', - help="Output delimited values", - ) - - parser.add_option("--item-delim", dest="item_delim", action="store", - default=';', - help="Delimiter for fields in single-line output", - ) - - parser.add_option("--sequence-delim", dest="sequence_delim", - action="store", default=',', - help="Delimiter for multi-valued fields", - ) - - parser.add_option("--csv", dest="output", action="store_const", - const='csv', - help="Output as CSV", - ) - - parser.add_option("--ini", dest="output", action="store_const", - const='ini', - help="Output as INI", - ) - - parser.add_option("--json", dest="output", action="store_const", - const='json', - help="Output as JSON", - ) + parser.add_option("-m", "--metadata-version", default=None, help="Override metadata version") + + parser.add_option( + "-f", + "--field", + dest="fields", + action="append", + help="Specify an output field (repeatable)", + ) + + parser.add_option( + "-d", + "--download-url-prefix", + dest="download_url_prefix", + help="Download URL prefix", + ) + + parser.add_option( + "--simple", + dest="output", + action="store_const", + const="simple", + default="simple", + help="Output as simple key-value pairs", + ) + + parser.add_option( + "-s", + "--skip", + dest="skip", + action="store_true", + default=True, + help="Skip missing values in simple output", + ) + + parser.add_option( + "-S", + "--no-skip", + dest="skip", + action="store_false", + help="Don't skip missing values in simple output", + ) + + parser.add_option( + "--single", + dest="output", + action="store_const", + const="single", + help="Output delimited values", + ) + + parser.add_option( + "--item-delim", + dest="item_delim", + action="store", + default=";", + help="Delimiter for fields in single-line output", + ) + + parser.add_option( + "--sequence-delim", + dest="sequence_delim", + action="store", + default=",", + help="Delimiter for multi-valued fields", + ) + + parser.add_option( + "--csv", + dest="output", + action="store_const", + const="csv", + help="Output as CSV", + ) + + parser.add_option( + "--ini", + dest="output", + action="store_const", + const="ini", + help="Output as INI", + ) + + parser.add_option( + "--json", + dest="output", + action="store_const", + const="json", + help="Output as JSON", + ) options, args = parser.parse_args(args) - if len(args)==0: + if len(args) == 0: parser.error("Pass one or more files or directories as arguments.") else: return options, args + class Base(object): _fields = None + def __init__(self, options): if options.fields: self._fields = options.fields @@ -105,6 +144,7 @@ def __init__(self, options): def finish(self): # pragma: NO COVER pass + class Simple(Base): def __init__(self, options): super(Simple, self).__init__(options) @@ -113,11 +153,13 @@ def __init__(self, options): def __call__(self, meta): for field in self._fields or list(meta): value = getattr(meta, field) - if (not self._skip) or (value is not None and value!=()): + if (not self._skip) or (value is not None and value != ()): print("%s: %s" % (field, value)) + class SingleLine(Base): _fields = None + def __init__(self, options): super(SingleLine, self).__init__(options) self._item_delim = options.item_delim @@ -136,15 +178,17 @@ def __call__(self, meta): values.append(value) print(self._item_delim.join(values)) + class CSV(Base): _writer = None + def __init__(self, options): super(CSV, self).__init__(options) self._sequence_delim = options.sequence_delim def __call__(self, meta): if self._fields is None: - self._fields = list(meta) # first dist wins + self._fields = list(meta) # first dist wins fields = self._fields if self._writer is None: self._writer = writer(sys.stdout) @@ -159,8 +203,10 @@ def __call__(self, meta): values.append(value) self._writer.writerow(values) + class INI(Base): _fields = None + def __init__(self, options): super(INI, self).__init__(options) self._parser = ConfigParser() @@ -168,21 +214,23 @@ def __init__(self, options): def __call__(self, meta): name = meta.name version = meta.version - section = '%s-%s' % (name, version) + section = "%s-%s" % (name, version) if self._parser.has_section(section): - raise ValueError('Duplicate distribution: %s' % section) + raise ValueError("Duplicate distribution: %s" % section) self._parser.add_section(section) for field in self._fields or list(meta): value = getattr(meta, field) if isinstance(value, (tuple, list)): - value = '\n\t'.join(value) + value = "\n\t".join(value) self._parser.set(section, field, value) def finish(self): self._parser.write(sys.stdout) # pragma: NO COVER + class JSON(Base): _fields = None + def __init__(self, options): super(JSON, self).__init__(options) self._mapping = OrderedDict() @@ -195,25 +243,26 @@ def __call__(self, meta): if value and not isinstance(value, (tuple, list)): value = str(value) if field in self._mapping: - raise ValueError('Duplicate field: %(field)r' % locals()) + raise ValueError("Duplicate field: %(field)r" % locals()) self._mapping[field] = value def finish(self): json.dump(self._mapping, sys.stdout, indent=2) + _FORMATTERS = { - 'simple': Simple, - 'single': SingleLine, - 'csv': CSV, - 'ini': INI, - 'json': JSON, + "simple": Simple, + "single": SingleLine, + "csv": CSV, + "ini": INI, + "json": JSON, } + def main(args=None): - """Entry point for pkginfo2 tool - """ + """Entry point for pkginfo2 tool""" options, paths = _parse_options(args) - format = getattr(options, 'output', 'simple') + format = getattr(options, "output", "simple") formatter = _FORMATTERS[format](options) for path in paths: @@ -224,8 +273,7 @@ def main(args=None): if options.download_url_prefix: if meta.download_url is None: filename = os.path.basename(path) - meta.download_url = '%s/%s' % (options.download_url_prefix, - filename) + meta.download_url = "%s/%s" % (options.download_url_prefix, filename) formatter(meta) diff --git a/pkginfo2/develop.py b/src/pkginfo2/develop.py similarity index 58% rename from pkginfo2/develop.py rename to src/pkginfo2/develop.py index d0f53b4..0af4594 100644 --- a/pkginfo2/develop.py +++ b/src/pkginfo2/develop.py @@ -5,33 +5,35 @@ from .distribution import Distribution -def _gather_py2(top, candidates): #pragma NO COVER Py3k + +def _gather_py2(top, candidates): # pragma NO COVER Py3k def _filter(candidates, dirname, fnames): for fname in fnames: fqn = os.path.join(dirname, fname) if os.path.isdir(fqn): - if fname == 'EGG-INFO' or fname.endswith('.egg-info'): + if fname == "EGG-INFO" or fname.endswith(".egg-info"): candidates.append(fqn) + os.path.walk(top, _filter, candidates) -def _gather_py3(top, candidates): #pragma NO COVER Python2 + +def _gather_py3(top, candidates): # pragma NO COVER Python2 for dirpath, dirnames, fnames in os.walk(top): for dirname in dirnames: fqn = os.path.join(dirpath, dirname) - if dirname == 'EGG-INFO' or dirname.endswith('.egg-info'): + if dirname == "EGG-INFO" or dirname.endswith(".egg-info"): candidates.append(fqn) -if sys.version_info[0] < 3: #pragma NO COVER Python2 + +if sys.version_info[0] < 3: # pragma NO COVER Python2 _gather = _gather_py2 -else: #pragma NO COVER Py3k +else: # pragma NO COVER Py3k _gather = _gather_py3 -class Develop(Distribution): +class Develop(Distribution): def __init__(self, path, metadata_version=None): - self.path = os.path.abspath( - os.path.normpath( - os.path.expanduser(path))) + self.path = os.path.abspath(os.path.normpath(os.path.expanduser(path))) self.metadata_version = metadata_version self.extractMetadata() @@ -39,8 +41,8 @@ def read(self): candidates = [self.path] _gather(self.path, candidates) for candidate in candidates: - path = os.path.join(candidate, 'PKG-INFO') + path = os.path.join(candidate, "PKG-INFO") if os.path.exists(path): - with io.open(path, errors='ignore') as f: + with io.open(path, errors="ignore") as f: return f.read() - warnings.warn('No PKG-INFO found for path: %s' % self.path) + warnings.warn("No PKG-INFO found for path: %s" % self.path) diff --git a/src/pkginfo2/distribution.py b/src/pkginfo2/distribution.py new file mode 100644 index 0000000..179164a --- /dev/null +++ b/src/pkginfo2/distribution.py @@ -0,0 +1,173 @@ +from email.parser import Parser + +from ._compat import StringIO +from ._compat import must_decode + + +def parse(fp): + return Parser().parse(fp) + + +def get(msg, header): + return _collapse_leading_ws(header, msg.get(header)) + + +def get_all(msg, header): + return [_collapse_leading_ws(header, x) for x in msg.get_all(header)] + + +def _collapse_leading_ws(header, txt): + """ + Return text after removing leading whitespace, and preserve newlines + if `Description`` header text. + """ + if header.lower() == "description": # preserve newlines + return "\n".join([x[8:] if x.startswith(" " * 8) else x for x in txt.strip().splitlines()]) + else: + return " ".join([x.strip() for x in txt.splitlines()]) + + +HEADER_ATTRS_1_0 = ( # PEP 241 + ("Metadata-Version", "metadata_version", False), + ("Name", "name", False), + ("Version", "version", False), + ("Platform", "platforms", True), + ("Supported-Platform", "supported_platforms", True), + ("Summary", "summary", False), + ("Description", "description", False), + ("Keywords", "keywords", False), + ("Home-Page", "home_page", False), + ("Author", "author", False), + ("Author-email", "author_email", False), + ("License", "license", False), +) + +HEADER_ATTRS_1_1 = HEADER_ATTRS_1_0 + ( # PEP 314 + ("Classifier", "classifiers", True), + ("Download-URL", "download_url", False), + ("Requires", "requires", True), + ("Provides", "provides", True), + ("Obsoletes", "obsoletes", True), +) + +HEADER_ATTRS_1_2 = HEADER_ATTRS_1_1 + ( # PEP 345 + ("Maintainer", "maintainer", False), + ("Maintainer-email", "maintainer_email", False), + ("Requires-Python", "requires_python", False), + ("Requires-External", "requires_external", True), + ("Requires-Dist", "requires_dist", True), + ("Provides-Dist", "provides_dist", True), + ("Obsoletes-Dist", "obsoletes_dist", True), + ("Project-URL", "project_urls", True), +) + +HEADER_ATTRS_2_0 = HEADER_ATTRS_1_2 # XXX PEP 426? + +HEADER_ATTRS_2_1 = HEADER_ATTRS_1_2 + ( # PEP 566 + ("Provides-Extra", "provides_extras", True), + ("Description-Content-Type", "description_content_type", False), +) + +HEADER_ATTRS_2_2 = HEADER_ATTRS_2_1 + ( # PEP 643 + ("Dynamic", "dynamic", True), +) + +HEADER_ATTRS_2_4 = HEADER_ATTRS_2_2 + ( # PEP 639 + ("License-Expression", "license_expression", False), + ("License-Files", "license_files", True), +) + +HEADER_ATTRS = { + "1.0": HEADER_ATTRS_1_0, + "1.1": HEADER_ATTRS_1_1, + "1.2": HEADER_ATTRS_1_2, + "2.0": HEADER_ATTRS_2_0, + "2.1": HEADER_ATTRS_2_1, + "2.2": HEADER_ATTRS_2_2, + "2.3": HEADER_ATTRS_2_2, + "2.4": HEADER_ATTRS_2_4, +} + + +class Distribution(object): + metadata_version = None + # version 1.0 + name = None + version = None + platforms = () + supported_platforms = () + summary = None + description = None + keywords = None + home_page = None + download_url = None + author = None + author_email = None + license = None + # version 1.1 + classifiers = () + requires = () + provides = () + obsoletes = () + # version 1.2 + maintainer = None + maintainer_email = None + requires_python = None + requires_external = () + requires_dist = () + provides_dist = () + obsoletes_dist = () + project_urls = () + # version 2.1 + provides_extras = () + description_content_type = None + # version 2.2 + dynamic = () + # version 2.4 + license_expression = None + license_files = () + + def extractMetadata(self): + data = self.read() + self.parse(data) + + def read(self): + raise NotImplementedError + + def _getHeaderAttrs(self): + if self.metadata_version in HEADER_ATTRS: + return HEADER_ATTRS[self.metadata_version] + else: + # If the specific version is not available, use the latest version + return HEADER_ATTRS[list(HEADER_ATTRS.keys())[-1]] + + def parse(self, data): + fp = StringIO(must_decode(data)) + msg = parse(fp) + + if "Metadata-Version" in msg and self.metadata_version is None: + value = get(msg, "Metadata-Version") + metadata_version = self.metadata_version = value + + for header_name, attr_name, multiple in self._getHeaderAttrs(): + if attr_name == "metadata_version": + continue + + if header_name in msg: + if multiple: + values = get_all(msg, header_name) + setattr(self, attr_name, values) + else: + value = get(msg, header_name) + if value != "UNKNOWN": + setattr(self, attr_name, value) + + body = msg.get_payload() + if body: + setattr(self, "description", body) + + def __iter__(self): + for header_name, attr_name, multiple in self._getHeaderAttrs(): + yield attr_name + + iterkeys = __iter__ diff --git a/pkginfo2/index.py b/src/pkginfo2/index.py similarity index 51% rename from pkginfo2/index.py rename to src/pkginfo2/index.py index 006f599..025f85d 100644 --- a/pkginfo2/index.py +++ b/src/pkginfo2/index.py @@ -1,15 +1,14 @@ from .distribution import Distribution -class Index(dict): +class Index(dict): def __setitem__(self, key, value): if not isinstance(value, Distribution): - raise ValueError('Not a distribution: %r.' % value) - if key != '%s-%s' % (value.name, value.version): - raise ValueError('Key must match -.') + raise ValueError("Not a distribution: %r." % value) + if key != "%s-%s" % (value.name, value.version): + raise ValueError("Key must match -.") super(Index, self).__setitem__(key, value) def add(self, distribution): - key = '%s-%s' % (distribution.name, distribution.version) + key = "%s-%s" % (distribution.name, distribution.version) self[key] = distribution - diff --git a/pkginfo2/installed.py b/src/pkginfo2/installed.py similarity index 68% rename from pkginfo2/installed.py rename to src/pkginfo2/installed.py index 84822c7..513c973 100644 --- a/pkginfo2/installed.py +++ b/src/pkginfo2/installed.py @@ -8,7 +8,6 @@ class Installed(Distribution): - def __init__(self, package, metadata_version=None): self.package = package _, self.package_name = os.path.split(package) @@ -21,12 +20,21 @@ def read(self): opj = os.path.join for candidate in os.listdir(self.package): - if not candidate.endswith(('.dist-info', '.egg-info', 'EGG-INFO',)): + if not candidate.endswith( + ( + ".dist-info", + ".egg-info", + "EGG-INFO", + ) + ): continue candidate = opj(self.package, candidate) - for metafile in ('METADATA', 'PKG-INFO'): - content = get_content(candidate, metafile=metafile,) + for metafile in ("METADATA", "PKG-INFO"): + content = get_content( + candidate, + metafile=metafile, + ) if content is not None: return content @@ -39,5 +47,5 @@ def get_content(candidate, metafile): else: return - with io.open(path, errors='ignore') as f: + with io.open(path, errors="ignore") as f: return f.read() diff --git a/pkginfo2/sdist.py b/src/pkginfo2/sdist.py similarity index 64% rename from pkginfo2/sdist.py rename to src/pkginfo2/sdist.py index 8c4e7cb..54837ee 100644 --- a/pkginfo2/sdist.py +++ b/src/pkginfo2/sdist.py @@ -5,8 +5,8 @@ from .distribution import Distribution -class SDist(Distribution): +class SDist(Distribution): def __init__(self, filename, metadata_version=None): self.filename = filename self.metadata_version = metadata_version @@ -15,42 +15,42 @@ def __init__(self, filename, metadata_version=None): @classmethod def _get_archive(cls, fqn): if not os.path.exists(fqn): - raise ValueError('No such file: %s' % fqn) + raise ValueError("No such file: %s" % fqn) if zipfile.is_zipfile(fqn): archive = zipfile.ZipFile(fqn) names = archive.namelist() + def read_file(name): return archive.read(name) elif tarfile.is_tarfile(fqn): archive = tarfile.TarFile.open(fqn) names = archive.getnames() + def read_file(name): return archive.extractfile(name).read() else: - raise ValueError('Not a known archive format: %s' % fqn) + raise ValueError("Not a known archive format: %s" % fqn) return archive, names, read_file - def read(self): - fqn = os.path.abspath( - os.path.normpath(self.filename)) + fqn = os.path.abspath(os.path.normpath(self.filename)) archive, names, read_file = self._get_archive(fqn) try: - tuples = [x.split('/') for x in names if 'PKG-INFO' in x] + tuples = [x.split("/") for x in names if "PKG-INFO" in x] schwarz = sorted([(len(x), x) for x in tuples]) for path in [x[1] for x in schwarz]: - candidate = '/'.join(path) + candidate = "/".join(path) data = read_file(candidate) - if b'Metadata-Version' in data: + if b"Metadata-Version" in data: return data finally: archive.close() - raise ValueError('No PKG-INFO in archive: %s' % fqn) + raise ValueError("No PKG-INFO in archive: %s" % fqn) class UnpackedSDist(SDist): @@ -60,16 +60,14 @@ def __init__(self, filename, metadata_version=None): elif os.path.isfile(filename): filename = os.path.dirname(filename) else: - raise ValueError('No such file: %s' % filename) + raise ValueError("No such file: %s" % filename) - super(UnpackedSDist, self).__init__( - filename, metadata_version=metadata_version) + super(UnpackedSDist, self).__init__(filename, metadata_version=metadata_version) def read(self): try: - pkg_info = os.path.join(self.filename, 'PKG-INFO') - with io.open(pkg_info, errors='ignore') as f: + pkg_info = os.path.join(self.filename, "PKG-INFO") + with io.open(pkg_info, errors="ignore") as f: return f.read() except Exception as e: - raise ValueError('Could not load %s as an unpacked sdist: %s' - % (self.filename, e)) + raise ValueError("Could not load %s as an unpacked sdist: %s" % (self.filename, e)) diff --git a/pkginfo2/utils.py b/src/pkginfo2/utils.py similarity index 77% rename from pkginfo2/utils.py rename to src/pkginfo2/utils.py index 2205d36..36a8f63 100644 --- a/pkginfo2/utils.py +++ b/src/pkginfo2/utils.py @@ -5,8 +5,10 @@ from .sdist import SDist from .wheel import Wheel + def get_metadata(path_or_module, metadata_version=None): - """ Try to create a Distribution 'path_or_module'. + """ + Try to create a Distribution 'path_or_module'. o 'path_or_module' may be a module object. @@ -24,21 +26,21 @@ def get_metadata(path_or_module, metadata_version=None): try: return BDist(path_or_module, metadata_version) - except (ValueError, IOError): #pragma NO COVER + except (ValueError, IOError): # pragma NO COVER pass try: return Wheel(path_or_module, metadata_version) - except (ValueError, IOError): #pragma NO COVER + except (ValueError, IOError): # pragma NO COVER pass if os.path.isdir(path_or_module): try: return Wheel(path_or_module, metadata_version) - except (ValueError, IOError): #pragma NO COVER + except (ValueError, IOError): # pragma NO COVER pass try: return Develop(path_or_module, metadata_version) - except (ValueError, IOError): #pragma NO COVER + except (ValueError, IOError): # pragma NO COVER pass diff --git a/pkginfo2/wheel.py b/src/pkginfo2/wheel.py similarity index 66% rename from pkginfo2/wheel.py rename to src/pkginfo2/wheel.py index 288f3b9..404ebcf 100644 --- a/pkginfo2/wheel.py +++ b/src/pkginfo2/wheel.py @@ -9,7 +9,6 @@ class Wheel(Distribution): - def __init__(self, filename, metadata_version=None): self.filename = filename self.metadata_version = metadata_version @@ -18,9 +17,9 @@ def __init__(self, filename, metadata_version=None): def read(self): fqn = os.path.abspath(os.path.normpath(self.filename)) if not os.path.exists(fqn): - raise ValueError('No such file: %s' % fqn) + raise ValueError("No such file: %s" % fqn) - if fqn.endswith('.whl'): + if fqn.endswith(".whl"): archive = zipfile.ZipFile(fqn) names = archive.namelist() @@ -29,35 +28,34 @@ def read_file(name): close = archive.close - elif fqn.endswith('.dist-info'): + elif fqn.endswith(".dist-info"): names = [os.path.join(fqn, p) for p in os.listdir(fqn)] def read_file(name): - with io.open(name, mode='rb') as inf: + with io.open(name, mode="rb") as inf: return inf.read() - close = lambda : None + close = lambda: None else: - raise ValueError('Not a known wheel archive format or ' - 'installed .dist-info: %s' % fqn) + raise ValueError("Not a known wheel archive format or installed .dist-info: %s" % fqn) try: - tuples = [x.split('/') for x in names if 'METADATA' in x] + tuples = [x.split("/") for x in names if "METADATA" in x] schwarz = sorted([(len(x), x) for x in tuples]) for path in [x[1] for x in schwarz]: - candidate = '/'.join(path) + candidate = "/".join(path) data = read_file(candidate) - if b'Metadata-Version' in data: + if b"Metadata-Version" in data: return data finally: close() - raise ValueError('No METADATA in archive: %s' % fqn) + raise ValueError("No METADATA in archive: %s" % fqn) def parse(self, data): super(Wheel, self).parse(data) fp = io.StringIO(must_decode(data)) msg = parse(fp) if self.description is None: - self.description = msg.get_payload() \ No newline at end of file + self.description = msg.get_payload() diff --git a/docs/examples/distlib-0.3.1-py2.py3-none-any.whl b/tests/examples/distlib-0.3.1-py2.py3-none-any.whl similarity index 100% rename from docs/examples/distlib-0.3.1-py2.py3-none-any.whl rename to tests/examples/distlib-0.3.1-py2.py3-none-any.whl diff --git a/docs/examples/mypackage-0.1-cp26-none-linux_x86_64.whl b/tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl similarity index 100% rename from docs/examples/mypackage-0.1-cp26-none-linux_x86_64.whl rename to tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl diff --git a/docs/examples/mypackage-0.1-py2.6.egg b/tests/examples/mypackage-0.1-py2.6.egg similarity index 100% rename from docs/examples/mypackage-0.1-py2.6.egg rename to tests/examples/mypackage-0.1-py2.6.egg diff --git a/tests/examples/mypackage-0.1.bogus b/tests/examples/mypackage-0.1.bogus new file mode 100644 index 0000000..e69de29 diff --git a/docs/examples/mypackage-0.1.dist-info/METADATA b/tests/examples/mypackage-0.1.dist-info/METADATA similarity index 100% rename from docs/examples/mypackage-0.1.dist-info/METADATA rename to tests/examples/mypackage-0.1.dist-info/METADATA diff --git a/docs/examples/mypackage-0.1.tar b/tests/examples/mypackage-0.1.tar similarity index 100% rename from docs/examples/mypackage-0.1.tar rename to tests/examples/mypackage-0.1.tar diff --git a/docs/examples/mypackage-0.1.tar.bz2 b/tests/examples/mypackage-0.1.tar.bz2 similarity index 100% rename from docs/examples/mypackage-0.1.tar.bz2 rename to tests/examples/mypackage-0.1.tar.bz2 diff --git a/docs/examples/mypackage-0.1.tar.gz b/tests/examples/mypackage-0.1.tar.gz similarity index 100% rename from docs/examples/mypackage-0.1.tar.gz rename to tests/examples/mypackage-0.1.tar.gz diff --git a/docs/examples/mypackage-0.1.zip b/tests/examples/mypackage-0.1.zip similarity index 100% rename from docs/examples/mypackage-0.1.zip rename to tests/examples/mypackage-0.1.zip diff --git a/docs/examples/mypackage-0.1/PKG-INFO b/tests/examples/mypackage-0.1/PKG-INFO similarity index 100% rename from docs/examples/mypackage-0.1/PKG-INFO rename to tests/examples/mypackage-0.1/PKG-INFO diff --git a/docs/examples/mypackage-0.1/README.txt b/tests/examples/mypackage-0.1/README.txt similarity index 100% rename from docs/examples/mypackage-0.1/README.txt rename to tests/examples/mypackage-0.1/README.txt diff --git a/docs/examples/mypackage-0.1/setup.cfg b/tests/examples/mypackage-0.1/setup.cfg similarity index 100% rename from docs/examples/mypackage-0.1/setup.cfg rename to tests/examples/mypackage-0.1/setup.cfg diff --git a/tests/examples/mypackage-0.1/setup.py b/tests/examples/mypackage-0.1/setup.py new file mode 100644 index 0000000..775eb85 --- /dev/null +++ b/tests/examples/mypackage-0.1/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup + +setup( + name="mypackage", + version="0.1", + author="Tres Seaver", + author_email="tseaver@palladion.com", + url="http://pypi.python.org/pypi/pkginfo", + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)", + ], +) diff --git a/docs/examples/nodistinfo-0.1-any.whl b/tests/examples/nodistinfo-0.1-any.whl similarity index 100% rename from docs/examples/nodistinfo-0.1-any.whl rename to tests/examples/nodistinfo-0.1-any.whl diff --git a/docs/examples/nopkginfo-0.1.egg b/tests/examples/nopkginfo-0.1.egg similarity index 100% rename from docs/examples/nopkginfo-0.1.egg rename to tests/examples/nopkginfo-0.1.egg diff --git a/docs/examples/nopkginfo-0.1.zip b/tests/examples/nopkginfo-0.1.zip similarity index 100% rename from docs/examples/nopkginfo-0.1.zip rename to tests/examples/nopkginfo-0.1.zip diff --git a/pkginfo2/tests/funny/__init__.py b/tests/funny/__init__.py similarity index 67% rename from pkginfo2/tests/funny/__init__.py rename to tests/funny/__init__.py index 6012a75..276e7fc 100644 --- a/pkginfo2/tests/funny/__init__.py +++ b/tests/funny/__init__.py @@ -1,2 +1,2 @@ # sample installed package w/ .egg-info file. -__package__ = 'funny' +__package__ = "funny" diff --git a/pkginfo2/tests/manky/NOT-A-PACKAGE.txt b/tests/manky/NOT-A-PACKAGE.txt similarity index 100% rename from pkginfo2/tests/manky/NOT-A-PACKAGE.txt rename to tests/manky/NOT-A-PACKAGE.txt diff --git a/pkginfo2/tests/wonky/namespaced/__init__.py b/tests/manky/namespaced/__init__.py similarity index 99% rename from pkginfo2/tests/wonky/namespaced/__init__.py rename to tests/manky/namespaced/__init__.py index 2e2033b..6d83202 100644 --- a/pkginfo2/tests/wonky/namespaced/__init__.py +++ b/tests/manky/namespaced/__init__.py @@ -1,7 +1,9 @@ # this is a namespace package try: import pkg_resources + pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/pkginfo2/tests/manky/namespaced/manky/__init__.py b/tests/manky/namespaced/manky/__init__.py similarity index 100% rename from pkginfo2/tests/manky/namespaced/manky/__init__.py rename to tests/manky/namespaced/manky/__init__.py diff --git a/pkginfo2/tests/silly/PKG-INFO b/tests/silly/PKG-INFO similarity index 100% rename from pkginfo2/tests/silly/PKG-INFO rename to tests/silly/PKG-INFO diff --git a/pkginfo2/tests/test_bdist.py b/tests/test_bdist.py similarity index 63% rename from pkginfo2/tests/test_bdist.py rename to tests/test_bdist.py index a2f5740..a299a8b 100644 --- a/pkginfo2/tests/test_bdist.py +++ b/tests/test_bdist.py @@ -1,9 +1,10 @@ import unittest -class BDistTests(unittest.TestCase): +class BDistTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.bdist import BDist + return BDist def _makeOne(self, filename=None, metadata_version=None): @@ -13,48 +14,56 @@ def _makeOne(self, filename=None, metadata_version=None): def _checkSample(self, bdist, filename): self.assertEqual(bdist.filename, filename) - self.assertEqual(bdist.name, 'mypackage') - self.assertEqual(bdist.version, '0.1') + self.assertEqual(bdist.name, "mypackage") + self.assertEqual(bdist.version, "0.1") self.assertEqual(bdist.keywords, None) def _checkClassifiers(self, bdist): - self.assertEqual(list(bdist.classifiers), - ['Development Status :: 4 - Beta', - 'Environment :: Console (Text Based)', - ]) + self.assertEqual( + list(bdist.classifiers), + [ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)", + ], + ) self.assertEqual(list(bdist.supported_platforms), []) def test_ctor_w_bogus_filename(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nonesuch-0.1-py2.6.egg' % d + filename = "%s/../tests/examples/nonesuch-0.1-py2.6.egg" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_w_non_egg(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d + filename = "%s/../tests/examples/mypackage-0.1.zip" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_wo_PKG_INFO(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nopkginfo-0.1.egg' % d + filename = "%s/../tests/examples/nopkginfo-0.1.egg" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_w_egg(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1-py2.6.egg' % d + filename = "%s/../tests/examples/mypackage-0.1-py2.6.egg" % d bdist = self._makeOne(filename) - self.assertEqual(bdist.metadata_version, '1.0') + self.assertEqual(bdist.metadata_version, "1.0") self._checkSample(bdist, filename) def test_ctor_w_egg_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1-py2.6.egg' % d - bdist = self._makeOne(filename, metadata_version='1.1') - self.assertEqual(bdist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1-py2.6.egg" % d + bdist = self._makeOne(filename, metadata_version="1.1") + self.assertEqual(bdist.metadata_version, "1.1") self._checkSample(bdist, filename) self._checkClassifiers(bdist) diff --git a/pkginfo2/tests/test_commandline.py b/tests/test_commandline.py similarity index 65% rename from pkginfo2/tests/test_commandline.py rename to tests/test_commandline.py index 54c9943..4846fa8 100644 --- a/pkginfo2/tests/test_commandline.py +++ b/tests/test_commandline.py @@ -1,15 +1,17 @@ import unittest -class Test__parse_options(unittest.TestCase): +class Test__parse_options(unittest.TestCase): def _callFUT(self, args): from pkginfo2.commandline import _parse_options + return _parse_options(args) def test_empty(self): import io import sys from pkginfo2.commandline import __doc__ as usage + firstline = usage.splitlines()[0] # parse_args emits "native" error output. @@ -23,13 +25,14 @@ def test_empty(self): self.assertTrue(firstline in buf.getvalue()) def test_nonempty(self): - options, args = self._callFUT(['foo']) - self.assertEqual(args, ['foo']) + options, args = self._callFUT(["foo"]) + self.assertEqual(args, ["foo"]) -class BaseTests(unittest.TestCase): +class BaseTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.commandline import Base + return Base def _makeOne(self, options): @@ -44,11 +47,12 @@ def test___init___w_fields(self): base = self._makeOne(_Options(fields=fields)) self.assertTrue(base._fields is fields) -class _FormatterBase(object): +class _FormatterBase(object): def _capture_output(self, func, *args, **kw): import io import sys + # Emulate stdout as wanting "native" strings if sys.version_info[0] < 3: buf = io.BytesIO() @@ -60,13 +64,15 @@ def _capture_output(self, func, *args, **kw): def _no_output(self, simple, meta): import sys + with _Monkey(sys, stdout=object()): # raise if write simple(meta) -class SimpleTests(unittest.TestCase, _FormatterBase): +class SimpleTests(unittest.TestCase, _FormatterBase): def _getTargetClass(self): from pkginfo2.commandline import Simple + return Simple def _makeOne(self, options): @@ -87,155 +93,146 @@ def test___call___w_skip_and_value_None_no_fields(self): self._no_output(simple, meta) def test___call___w_skip_and_value_empty_tuple_explicit_fields(self): - simple = self._makeOne(_Options(fields=('foo',), skip=True)) - meta = _Meta(foo=(), bar='Bar') + simple = self._makeOne(_Options(fields=("foo",), skip=True)) + meta = _Meta(foo=(), bar="Bar") self._no_output(simple, meta) def test___call___w_skip_but_values_explicit_fields(self): - simple = self._makeOne(_Options(fields=('foo',), skip=True)) - meta = _Meta(foo='Foo') + simple = self._makeOne(_Options(fields=("foo",), skip=True)) + meta = _Meta(foo="Foo") output = self._capture_output(simple, meta) - self.assertEqual(output, 'foo: Foo\n') + self.assertEqual(output, "foo: Foo\n") -class SingleLineTests(unittest.TestCase, _FormatterBase): +class SingleLineTests(unittest.TestCase, _FormatterBase): def _getTargetClass(self): from pkginfo2.commandline import SingleLine + return SingleLine def _makeOne(self, options): return self._getTargetClass()(options) def test___init___(self): - single = self._makeOne( - _Options(fields=None, item_delim='I', sequence_delim='S')) - self.assertEqual(single._item_delim, 'I') - self.assertEqual(single._sequence_delim, 'S') + single = self._makeOne(_Options(fields=None, item_delim="I", sequence_delim="S")) + self.assertEqual(single._item_delim, "I") + self.assertEqual(single._sequence_delim, "S") def test___call__wo_fields_wo_list(self): single = self._makeOne( - _Options(fields=(), item_delim='|', - sequence_delim=object())) # raise if used - meta = _Meta(foo='Foo', bar='Bar') + _Options(fields=(), item_delim="|", sequence_delim=object()) + ) # raise if used + meta = _Meta(foo="Foo", bar="Bar") output = self._capture_output(single, meta) - self.assertEqual(output, 'Bar|Foo\n') + self.assertEqual(output, "Bar|Foo\n") def test___call__w_fields_w_list(self): - single = self._makeOne( - _Options(fields=('foo', 'bar'), item_delim='|', - sequence_delim='*')) - meta = _Meta(foo='Foo', bar=['Bar1', 'Bar2'], baz='Baz') + single = self._makeOne(_Options(fields=("foo", "bar"), item_delim="|", sequence_delim="*")) + meta = _Meta(foo="Foo", bar=["Bar1", "Bar2"], baz="Baz") output = self._capture_output(single, meta) - self.assertEqual(output, 'Foo|Bar1*Bar2\n') + self.assertEqual(output, "Foo|Bar1*Bar2\n") -class CSVTests(unittest.TestCase, _FormatterBase): +class CSVTests(unittest.TestCase, _FormatterBase): def _getTargetClass(self): from pkginfo2.commandline import CSV + return CSV def _makeOne(self, options): return self._getTargetClass()(options) def test___init___(self): - csv = self._makeOne( - _Options(fields=None, sequence_delim='S')) - self.assertEqual(csv._sequence_delim, 'S') + csv = self._makeOne(_Options(fields=None, sequence_delim="S")) + self.assertEqual(csv._sequence_delim, "S") def test___call__wo_fields_wo_list(self): - meta = _Meta(foo='Foo', bar='Bar') - csv = self._makeOne( - _Options(fields=None, - sequence_delim=object())) # raise if used + meta = _Meta(foo="Foo", bar="Bar") + csv = self._makeOne(_Options(fields=None, sequence_delim=object())) # raise if used output = self._capture_output(csv, meta) - self.assertEqual(output, 'bar,foo\r\nBar,Foo\r\n') + self.assertEqual(output, "bar,foo\r\nBar,Foo\r\n") def test___call__w_fields_w_list(self): - meta = _Meta(foo='Foo', bar=['Bar1', 'Bar2'], baz='Baz') - csv = self._makeOne( - _Options(fields=('foo', 'bar'), item_delim='|', - sequence_delim='*')) + meta = _Meta(foo="Foo", bar=["Bar1", "Bar2"], baz="Baz") + csv = self._makeOne(_Options(fields=("foo", "bar"), item_delim="|", sequence_delim="*")) output = self._capture_output(csv, meta) - self.assertEqual(output, 'foo,bar\r\nFoo,Bar1*Bar2\r\n') + self.assertEqual(output, "foo,bar\r\nFoo,Bar1*Bar2\r\n") -class INITests(unittest.TestCase, _FormatterBase): +class INITests(unittest.TestCase, _FormatterBase): def _getTargetClass(self): from pkginfo2.commandline import INI + return INI def _makeOne(self, options): return self._getTargetClass()(options) def test___call___duplicate(self): - ini = self._makeOne(_Options(fields=('foo',))) - meta = _Meta(name='foo', version='0.1', foo='Foo') - ini._parser.add_section('foo-0.1') + ini = self._makeOne(_Options(fields=("foo",))) + meta = _Meta(name="foo", version="0.1", foo="Foo") + ini._parser.add_section("foo-0.1") self.assertRaises(ValueError, ini, meta) def test___call___wo_fields_wo_list(self): ini = self._makeOne(_Options(fields=None)) - meta = _Meta(name='foo', version='0.1', foo='Foo') + meta = _Meta(name="foo", version="0.1", foo="Foo") ini(meta) cp = ini._parser - self.assertEqual(cp.sections(), ['foo-0.1']) - self.assertEqual(sorted(cp.options('foo-0.1')), - ['foo', 'name', 'version']) - self.assertEqual(cp.get('foo-0.1', 'name'), 'foo') - self.assertEqual(cp.get('foo-0.1', 'version'), '0.1') - self.assertEqual(cp.get('foo-0.1', 'foo'), 'Foo') + self.assertEqual(cp.sections(), ["foo-0.1"]) + self.assertEqual(sorted(cp.options("foo-0.1")), ["foo", "name", "version"]) + self.assertEqual(cp.get("foo-0.1", "name"), "foo") + self.assertEqual(cp.get("foo-0.1", "version"), "0.1") + self.assertEqual(cp.get("foo-0.1", "foo"), "Foo") def test___call___w_fields_w_list(self): - ini = self._makeOne(_Options(fields=('foo', 'bar'))) - meta = _Meta(name='foo', version='0.1', - foo='Foo', bar=['Bar1', 'Bar2'], baz='Baz') + ini = self._makeOne(_Options(fields=("foo", "bar"))) + meta = _Meta(name="foo", version="0.1", foo="Foo", bar=["Bar1", "Bar2"], baz="Baz") ini(meta) cp = ini._parser - self.assertEqual(cp.sections(), ['foo-0.1']) - self.assertEqual(sorted(cp.options('foo-0.1')), ['bar', 'foo']) - self.assertEqual(cp.get('foo-0.1', 'foo'), 'Foo') - self.assertEqual(cp.get('foo-0.1', 'bar'), 'Bar1\n\tBar2') + self.assertEqual(cp.sections(), ["foo-0.1"]) + self.assertEqual(sorted(cp.options("foo-0.1")), ["bar", "foo"]) + self.assertEqual(cp.get("foo-0.1", "foo"), "Foo") + self.assertEqual(cp.get("foo-0.1", "bar"), "Bar1\n\tBar2") -class JSONtests(unittest.TestCase, _FormatterBase): +class JSONtests(unittest.TestCase, _FormatterBase): def _getTargetClass(self): from pkginfo2.commandline import JSON + return JSON def _makeOne(self, options): return self._getTargetClass()(options) def test___call___duplicate_with_meta_and_fields(self): - json = self._makeOne(_Options(fields=('name',))) - meta = _Meta(name='foo', version='0.1', foo='Foo') - json._mapping['name'] = 'foo' + json = self._makeOne(_Options(fields=("name",))) + meta = _Meta(name="foo", version="0.1", foo="Foo") + json._mapping["name"] = "foo" self.assertRaises(ValueError, json, meta) def test___call___duplicate_with_meta_wo_fields(self): json = self._makeOne(_Options(fields=None)) - meta = _Meta(name='foo', version='0.1', foo='Foo') - json._mapping['name'] = 'foo' + meta = _Meta(name="foo", version="0.1", foo="Foo") + json._mapping["name"] = "foo" self.assertRaises(ValueError, json, meta) def test___call___wo_fields_wo_list(self): from collections import OrderedDict json = self._makeOne(_Options(fields=None)) - meta = _Meta(name='foo', version='0.1', foo='Foo') + meta = _Meta(name="foo", version="0.1", foo="Foo") json(meta) - expected = OrderedDict([ - ('foo', 'Foo'), ('name', 'foo'), ('version', '0.1')]) + expected = OrderedDict([("foo", "Foo"), ("name", "foo"), ("version", "0.1")]) self.assertEqual(expected, json._mapping) def test___call___w_fields_w_list(self): from collections import OrderedDict - json = self._makeOne(_Options(fields=('foo', 'bar'))) - meta = _Meta(name='foo', version='0.1', - foo='Foo', bar=['Bar1', 'Bar2'], baz='Baz') + json = self._makeOne(_Options(fields=("foo", "bar"))) + meta = _Meta(name="foo", version="0.1", foo="Foo", bar=["Bar1", "Bar2"], baz="Baz") json(meta) - expected = OrderedDict([ - ('foo', 'Foo'), ('bar', ['Bar1', 'Bar2'])]) + expected = OrderedDict([("foo", "Foo"), ("bar", ["Bar1", "Bar2"])]) self.assertEqual(expected, json._mapping) def test___call___output(self): @@ -243,20 +240,19 @@ def test___call___output(self): import json as json_parser json = self._makeOne(_Options(fields=None)) - meta = _Meta(name='foo', version='0.1', foo='Foo') + meta = _Meta(name="foo", version="0.1", foo="Foo") json(meta) output = self._capture_output(json.finish) - output = json_parser.loads( - output, object_pairs_hook=OrderedDict) - expected = OrderedDict([ - ('foo', 'Foo'), ('name', 'foo'), ('version', '0.1')]) + output = json_parser.loads(output, object_pairs_hook=OrderedDict) + expected = OrderedDict([("foo", "Foo"), ("name", "foo"), ("version", "0.1")]) self.assertEqual(expected, output) -class Test_main(unittest.TestCase): - def _callFUT(self, args, monkey='simple'): +class Test_main(unittest.TestCase): + def _callFUT(self, args, monkey="simple"): from pkginfo2.commandline import main from pkginfo2.commandline import _FORMATTERS + before = _FORMATTERS[monkey] dummy = _Formatter() _FORMATTERS[monkey] = lambda *options: dummy @@ -268,51 +264,57 @@ def _callFUT(self, args, monkey='simple'): def test_w_mising_dist(self): from pkginfo2 import commandline as MUT + def _get_metadata(path_or_module, md_version): - self.assertEqual(path_or_module, 'foo') + self.assertEqual(path_or_module, "foo") self.assertEqual(md_version, None) return None + with _Monkey(MUT, get_metadata=_get_metadata): - formatter = self._callFUT(['foo']) + formatter = self._callFUT(["foo"]) self.assertEqual(formatter._called_with, []) self.assertTrue(formatter._finished) def test_w_dist_wo_download_url(self): from pkginfo2 import commandline as MUT + meta = _Meta(download_url=None) + def _get_metadata(path_or_module, md_version): - self.assertEqual(path_or_module, '/path/to/foo') + self.assertEqual(path_or_module, "/path/to/foo") self.assertEqual(md_version, None) return meta + with _Monkey(MUT, get_metadata=_get_metadata): - formatter = self._callFUT( - ['-d', 'http://example.com', '/path/to/foo']) + formatter = self._callFUT(["-d", "http://example.com", "/path/to/foo"]) self.assertEqual(formatter._called_with, [meta]) self.assertTrue(formatter._finished) - self.assertEqual(meta.download_url, 'http://example.com/foo') + self.assertEqual(meta.download_url, "http://example.com/foo") def test_w_dist_w_download_url(self): from pkginfo2 import commandline as MUT - meta = _Meta(download_url='http://example.com/dist/foo') + + meta = _Meta(download_url="http://example.com/dist/foo") + def _get_metadata(path_or_module, md_version): - self.assertEqual(path_or_module, '/path/to/foo') + self.assertEqual(path_or_module, "/path/to/foo") self.assertEqual(md_version, None) return meta + with _Monkey(MUT, get_metadata=_get_metadata): - formatter = self._callFUT( - ['-d', 'http://example.com', '/path/to/foo']) + formatter = self._callFUT(["-d", "http://example.com", "/path/to/foo"]) self.assertEqual(formatter._called_with, [meta]) self.assertTrue(formatter._finished) - self.assertEqual(meta.download_url, 'http://example.com/dist/foo') + self.assertEqual(meta.download_url, "http://example.com/dist/foo") -class _Options(object): +class _Options(object): def __init__(self, **kw): for k in kw: self.__dict__[k] = kw[k] -class _Meta(object): +class _Meta(object): def __init__(self, **kw): for k in kw: self.__dict__[k] = kw[k] @@ -320,6 +322,7 @@ def __init__(self, **kw): def __iter__(self): return iter(sorted(self.__dict__)) + class _Monkey(object): # context-manager for replacing module names in the scope of a test. @@ -336,8 +339,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): for key, value in self.to_restore.items(): setattr(self.module, key, value) -class _Formatter(object): +class _Formatter(object): _finished = False def __init__(self): diff --git a/pkginfo2/tests/test_develop.py b/tests/test_develop.py similarity index 67% rename from pkginfo2/tests/test_develop.py rename to tests/test_develop.py index 8b588a1..536d9a0 100644 --- a/pkginfo2/tests/test_develop.py +++ b/tests/test_develop.py @@ -1,25 +1,21 @@ import unittest class DevelopTests(unittest.TestCase): - def _getTargetClass(self): from pkginfo2.develop import Develop + return Develop def _makeOne(self, dirname=None): return self._getTargetClass()(dirname) - def test_ctor_w_path(self): - from pkginfo2.tests import _checkSample - develop = self._makeOne('.') - _checkSample(self, develop) - def test_ctor_w_invalid_path(self): - import warnings + import warnings + old_filters = warnings.filters[:] - warnings.filterwarnings('ignore') + warnings.filterwarnings("ignore") try: - develop = self._makeOne('/nonesuch') + develop = self._makeOne("/nonesuch") self.assertEqual(develop.metadata_version, None) self.assertEqual(develop.name, None) self.assertEqual(develop.version, None) diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..e5ad69c --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,456 @@ +import unittest + + +class DistributionTests(unittest.TestCase): + def _getTargetClass(self): + from pkginfo2.distribution import Distribution + + return Distribution + + def _makeOne(self, metadata_version="1.0"): + dist = self._getTargetClass()() + if metadata_version is not None: + dist.metadata_version = metadata_version + return dist + + def test_ctor_defaults(self): + sdist = self._makeOne(None) + self.assertEqual(sdist.metadata_version, None) + # version 1.0 + self.assertEqual(sdist.name, None) + self.assertEqual(sdist.version, None) + self.assertEqual(sdist.platforms, ()) + self.assertEqual(sdist.supported_platforms, ()) + self.assertEqual(sdist.summary, None) + self.assertEqual(sdist.description, None) + self.assertEqual(sdist.keywords, None) + self.assertEqual(sdist.home_page, None) + self.assertEqual(sdist.download_url, None) + self.assertEqual(sdist.author, None) + self.assertEqual(sdist.author_email, None) + self.assertEqual(sdist.license, None) + # version 1.1 + self.assertEqual(sdist.classifiers, ()) + self.assertEqual(sdist.requires, ()) + self.assertEqual(sdist.provides, ()) + self.assertEqual(sdist.obsoletes, ()) + # version 1.2 + self.assertEqual(sdist.maintainer, None) + self.assertEqual(sdist.maintainer_email, None) + self.assertEqual(sdist.requires_python, None) + self.assertEqual(sdist.requires_external, ()) + self.assertEqual(sdist.requires_dist, ()) + self.assertEqual(sdist.provides_dist, ()) + self.assertEqual(sdist.obsoletes_dist, ()) + self.assertEqual(sdist.project_urls, ()) + # version 2.1 + self.assertEqual(sdist.provides_extras, ()) + self.assertEqual(sdist.description_content_type, None) + # version 2.2 + self.assertEqual(sdist.dynamic, ()) + + def test_extractMetadata_raises_NotImplementedError(self): + # 'extractMetadata' calls 'read', which subclasses must override. + dist = self._makeOne(None) + self.assertRaises(NotImplementedError, dist.extractMetadata) + + def test_read_raises_NotImplementedError(self): + # Subclasses must override 'read'. + dist = self._makeOne(None) + self.assertRaises(NotImplementedError, dist.read) + + def test_parse_given_unicode(self): + from pkginfo2._compat import u + + dist = self._makeOne() + dist.parse(u("Metadata-Version: 1.0\nName: lp722928_c3")) # no raise + + def test_parse_Metadata_Version_1_0(self): + from pkginfo2.distribution import HEADER_ATTRS_1_0 + + dist = self._makeOne(None) + dist.parse("Metadata-Version: 1.0") + self.assertEqual(dist.metadata_version, "1.0") + self.assertEqual(list(dist), [x[1] for x in HEADER_ATTRS_1_0]) + + def test_parse_Metadata_Version_1_1(self): + from pkginfo2.distribution import HEADER_ATTRS_1_1 + + dist = self._makeOne(None) + dist.parse("Metadata-Version: 1.1") + self.assertEqual(dist.metadata_version, "1.1") + self.assertEqual(list(dist), [x[1] for x in HEADER_ATTRS_1_1]) + + def test_parse_Metadata_Version_1_2(self): + from pkginfo2.distribution import HEADER_ATTRS_1_2 + + dist = self._makeOne(None) + dist.parse("Metadata-Version: 1.2") + self.assertEqual(dist.metadata_version, "1.2") + self.assertEqual(list(dist), [x[1] for x in HEADER_ATTRS_1_2]) + + def test_parse_Metadata_Version_2_1(self): + from pkginfo2.distribution import HEADER_ATTRS_2_1 + + dist = self._makeOne(None) + dist.parse("Metadata-Version: 2.1") + self.assertEqual(dist.metadata_version, "2.1") + self.assertEqual(list(dist), [x[1] for x in HEADER_ATTRS_2_1]) + + def test_parse_Metadata_Version_2_2(self): + from pkginfo2.distribution import HEADER_ATTRS_2_2 + + dist = self._makeOne(None) + dist.parse("Metadata-Version: 2.2") + self.assertEqual(dist.metadata_version, "2.2") + self.assertEqual(list(dist), [x[1] for x in HEADER_ATTRS_2_2]) + + def test_parse_Metadata_Version_unknown(self): + dist = self._makeOne(None) + dist.parse("Metadata-Version: 1.3") + self.assertEqual(dist.metadata_version, "1.3") + + def test_parse_Metadata_Version_override(self): + dist = self._makeOne("1.2") + dist.parse("Metadata-Version: 1.0") + self.assertEqual(dist.metadata_version, "1.2") + + def test_parse_Name(self): + dist = self._makeOne() + dist.parse("Name: foobar") + self.assertEqual(dist.name, "foobar") + + def test_parse_Version(self): + dist = self._makeOne() + dist.parse("Version: 2.1.3b5") + self.assertEqual(dist.version, "2.1.3b5") + + def test_parse_Platform_single(self): + dist = self._makeOne() + dist.parse("Platform: Plan9") + self.assertEqual(list(dist.platforms), ["Plan9"]) + + def test_parse_Platform_multiple(self): + dist = self._makeOne() + dist.parse("Platform: Plan9\nPlatform: AIX") + self.assertEqual(list(dist.platforms), ["Plan9", "AIX"]) + + def test_parse_Supported_Platform_single(self): + dist = self._makeOne() + dist.parse("Supported-Platform: Plan9") + self.assertEqual(list(dist.supported_platforms), ["Plan9"]) + + def test_parse_Supported_Platform_multiple(self): + dist = self._makeOne() + dist.parse("Supported-Platform: i386-win32\nSupported-Platform: RedHat 7.2") + self.assertEqual(list(dist.supported_platforms), ["i386-win32", "RedHat 7.2"]) + + def test_parse_Summary(self): + dist = self._makeOne() + dist.parse("Summary: Package for foo") + self.assertEqual(dist.summary, "Package for foo") + + def test_parse_Description(self): + dist = self._makeOne() + dist.parse("Description: This package enables integration with foo servers.") + self.assertEqual(dist.description, "This package enables integration with foo servers.") + + def test_parse_Description_multiline(self): + dist = self._makeOne() + dist.parse("Description: This package enables integration with\n foo servers.") + self.assertEqual(dist.description, "This package enables integration with\nfoo servers.") + + def test_parse_Description_in_payload(self): + dist = self._makeOne() + dist.parse("Foo: Bar\n\nThis package enables integration with\nfoo servers.") + self.assertEqual(dist.description, "This package enables integration with\nfoo servers.") + + def test_parse_Keywords(self): + dist = self._makeOne() + dist.parse("Keywords: bar foo qux") + self.assertEqual(dist.keywords, "bar foo qux") + + def test_parse_Home_page(self): + dist = self._makeOne() + dist.parse("Home-page: http://example.com/package") + self.assertEqual(dist.home_page, "http://example.com/package") + + def test_parse_Author(self): + dist = self._makeOne() + dist.parse("Author: J. Phredd Bloggs") + self.assertEqual(dist.author, "J. Phredd Bloggs") + + def test_parse_Author_Email(self): + dist = self._makeOne() + dist.parse("Author-email: phreddy@example.com") + self.assertEqual(dist.author_email, "phreddy@example.com") + + def test_parse_License(self): + dist = self._makeOne() + dist.parse("License: Poetic") + self.assertEqual(dist.license, "Poetic") + + # Metadata version 1.1, defined in PEP 314. + def test_parse_Classifier_single(self): + dist = self._makeOne("1.1") + dist.parse("Classifier: Some :: Silly Thing") + self.assertEqual(list(dist.classifiers), ["Some :: Silly Thing"]) + + def test_parse_Classifier_multiple(self): + dist = self._makeOne("1.1") + dist.parse("Classifier: Some :: Silly Thing\nClassifier: Or :: Other") + self.assertEqual(list(dist.classifiers), ["Some :: Silly Thing", "Or :: Other"]) + + def test_parse_Download_URL(self): + dist = self._makeOne("1.1") + dist.parse("Download-URL: http://example.com/package/mypackage-0.1.zip") + self.assertEqual(dist.download_url, "http://example.com/package/mypackage-0.1.zip") + + def test_parse_Requires_single_wo_version(self): + dist = self._makeOne("1.1") + dist.parse("Requires: SpanishInquisition") + self.assertEqual(list(dist.requires), ["SpanishInquisition"]) + + def test_parse_Requires_single_w_version(self): + dist = self._makeOne("1.1") + dist.parse("Requires: SpanishInquisition (>=1.3)") + self.assertEqual(list(dist.requires), ["SpanishInquisition (>=1.3)"]) + + def test_parse_Requires_multiple(self): + dist = self._makeOne("1.1") + dist.parse( + "Requires: SpanishInquisition\n" + "Requires: SillyWalks (1.4)\n" + "Requires: kniggits (>=2.3,<3.0)" + ) + self.assertEqual( + list(dist.requires), + [ + "SpanishInquisition", + "SillyWalks (1.4)", + "kniggits (>=2.3,<3.0)", + ], + ) + + def test_parse_Provides_single_wo_version(self): + dist = self._makeOne("1.1") + dist.parse("Provides: SillyWalks") + self.assertEqual(list(dist.provides), ["SillyWalks"]) + + def test_parse_Provides_single_w_version(self): + dist = self._makeOne("1.1") + dist.parse("Provides: SillyWalks (1.4)") + self.assertEqual(list(dist.provides), ["SillyWalks (1.4)"]) + + def test_parse_Provides_multiple(self): + dist = self._makeOne("1.1") + dist.parse("Provides: SillyWalks\nProvides: DeadlyJoke (3.1.4)") + self.assertEqual( + list(dist.provides), + [ + "SillyWalks", + "DeadlyJoke (3.1.4)", + ], + ) + + def test_parse_Obsoletes_single_no_version(self): + dist = self._makeOne("1.1") + dist.parse("Obsoletes: SillyWalks") + self.assertEqual(list(dist.obsoletes), ["SillyWalks"]) + + def test_parse_Obsoletes_single_w_version(self): + dist = self._makeOne("1.1") + dist.parse("Obsoletes: SillyWalks (<=1.3)") + self.assertEqual(list(dist.obsoletes), ["SillyWalks (<=1.3)"]) + + def test_parse_Obsoletes_multiple(self): + dist = self._makeOne("1.1") + dist.parse("Obsoletes: kniggits\nObsoletes: SillyWalks (<=2.0)") + self.assertEqual( + list(dist.obsoletes), + [ + "kniggits", + "SillyWalks (<=2.0)", + ], + ) + + # Metadata version 1.2, defined in PEP 345. + def test_parse_Maintainer(self): + dist = self._makeOne(metadata_version="1.2") + dist.parse("Maintainer: J. Phredd Bloggs") + self.assertEqual(dist.maintainer, "J. Phredd Bloggs") + + def test_parse_Maintainer_Email(self): + dist = self._makeOne(metadata_version="1.2") + dist.parse("Maintainer-email: phreddy@example.com") + self.assertEqual(dist.maintainer_email, "phreddy@example.com") + + def test_parse_Requires_Python_single_spec(self): + dist = self._makeOne("1.2") + dist.parse("Requires-Python: >2.4") + self.assertEqual(dist.requires_python, ">2.4") + + def test_parse_Requires_External_single_wo_version(self): + dist = self._makeOne("1.2") + dist.parse("Requires-External: libfoo") + self.assertEqual(list(dist.requires_external), ["libfoo"]) + + def test_parse_Requires_External_single_w_version(self): + dist = self._makeOne("1.2") + dist.parse("Requires-External: libfoo (>=1.3)") + self.assertEqual(list(dist.requires_external), ["libfoo (>=1.3)"]) + + def test_parse_Requires_External_multiple(self): + dist = self._makeOne("1.2") + dist.parse( + "Requires-External: libfoo\n" + "Requires-External: libbar (1.4)\n" + "Requires-External: libbaz (>=2.3,<3.0)" + ) + self.assertEqual( + list(dist.requires_external), + [ + "libfoo", + "libbar (1.4)", + "libbaz (>=2.3,<3.0)", + ], + ) + + def test_parse_Requires_Dist_single_wo_version(self): + dist = self._makeOne("1.2") + dist.parse("Requires-Dist: SpanishInquisition") + self.assertEqual(list(dist.requires_dist), ["SpanishInquisition"]) + + def test_parse_Requires_Dist_single_w_version(self): + dist = self._makeOne("1.2") + dist.parse("Requires-Dist: SpanishInquisition (>=1.3)") + self.assertEqual(list(dist.requires_dist), ["SpanishInquisition (>=1.3)"]) + + def test_parse_Requires_Dist_single_w_env_marker(self): + dist = self._makeOne("1.2") + dist.parse("Requires-Dist: SpanishInquisition; python_version == '1.4'") + self.assertEqual(list(dist.requires_dist), ["SpanishInquisition; python_version == '1.4'"]) + + def test_parse_Requires_Dist_multiple(self): + dist = self._makeOne("1.2") + dist.parse( + "Requires-Dist: SpanishInquisition\n" + "Requires-Dist: SillyWalks; python_version == '1.4'\n" + "Requires-Dist: kniggits (>=2.3,<3.0)" + ) + self.assertEqual( + list(dist.requires_dist), + [ + "SpanishInquisition", + "SillyWalks; python_version == '1.4'", + "kniggits (>=2.3,<3.0)", + ], + ) + + def test_parse_Provides_Dist_single_wo_version(self): + dist = self._makeOne("1.2") + dist.parse("Provides-Dist: SillyWalks") + self.assertEqual(list(dist.provides_dist), ["SillyWalks"]) + + def test_parse_Provides_Dist_single_w_version(self): + dist = self._makeOne("1.2") + dist.parse("Provides-Dist: SillyWalks (1.4)") + self.assertEqual(list(dist.provides_dist), ["SillyWalks (1.4)"]) + + def test_parse_Provides_Dist_single_w_env_marker(self): + dist = self._makeOne("1.2") + dist.parse("Provides-Dist: SillyWalks; sys.platform == 'os2'") + self.assertEqual(list(dist.provides_dist), ["SillyWalks; sys.platform == 'os2'"]) + + def test_parse_Provides_Dist_multiple(self): + dist = self._makeOne("1.2") + dist.parse( + "Provides-Dist: SillyWalks\n" + "Provides-Dist: SpanishInquisition; sys.platform == 'os2'\n" + "Provides-Dist: DeadlyJoke (3.1.4)" + ) + self.assertEqual( + list(dist.provides_dist), + [ + "SillyWalks", + "SpanishInquisition; sys.platform == 'os2'", + "DeadlyJoke (3.1.4)", + ], + ) + + def test_parse_Obsoletes_Dist_single_no_version(self): + dist = self._makeOne("1.2") + dist.parse("Obsoletes-Dist: SillyWalks") + self.assertEqual(list(dist.obsoletes_dist), ["SillyWalks"]) + + def test_parse_Obsoletes_Dist_single_w_version(self): + dist = self._makeOne("1.2") + dist.parse("Obsoletes-Dist: SillyWalks (<=1.3)") + self.assertEqual(list(dist.obsoletes_dist), ["SillyWalks (<=1.3)"]) + + def test_parse_Obsoletes_Dist_single_w_env_marker(self): + dist = self._makeOne("1.2") + dist.parse("Obsoletes-Dist: SillyWalks; sys.platform == 'os2'") + self.assertEqual(list(dist.obsoletes_dist), ["SillyWalks; sys.platform == 'os2'"]) + + def test_parse_Obsoletes_Dist_multiple(self): + dist = self._makeOne("1.2") + dist.parse( + "Obsoletes-Dist: kniggits\n" + "Obsoletes-Dist: SillyWalks; sys.platform == 'os2'\n" + "Obsoletes-Dist: DeadlyJoke (<=2.0)\n" + ) + self.assertEqual( + list(dist.obsoletes_dist), + [ + "kniggits", + "SillyWalks; sys.platform == 'os2'", + "DeadlyJoke (<=2.0)", + ], + ) + + def test_parse_Project_URL_single_no_version(self): + dist = self._makeOne("1.2") + dist.parse("Project-URL: Bug tracker, http://bugs.example.com/grail") + self.assertEqual(list(dist.project_urls), ["Bug tracker, http://bugs.example.com/grail"]) + + def test_parse_Project_URL_multiple(self): + dist = self._makeOne("1.2") + dist.parse( + "Project-URL: Bug tracker, http://bugs.example.com/grail\n" + "Project-URL: Repository, http://svn.example.com/grail" + ) + self.assertEqual( + list(dist.project_urls), + [ + "Bug tracker, http://bugs.example.com/grail", + "Repository, http://svn.example.com/grail", + ], + ) + + # Metadata version 2.1, defined in PEP 566. + def test_parse_Provides_Extra_single(self): + dist = self._makeOne("2.1") + dist.parse("Provides-Extra: pdf") + self.assertEqual(list(dist.provides_extras), ["pdf"]) + + def test_parse_Provides_Extra_multiple(self): + dist = self._makeOne("2.1") + dist.parse("Provides-Extra: pdf\nProvides-Extra: tex") + self.assertEqual(list(dist.provides_extras), ["pdf", "tex"]) + + def test_parse_Provides_Extra_single(self): + dist = self._makeOne("2.1") + dist.parse("Description-Content-Type: text/plain") + self.assertEqual(dist.description_content_type, "text/plain") + + # Metadata version 2.2, defined in PEP 643. + def test_parse_Dynamic_single(self): + dist = self._makeOne("2.2") + dist.parse("Dynamic: Platforms") + self.assertEqual(list(dist.dynamic), ["Platforms"]) + + def test_parse_Dynamic_multiple(self): + dist = self._makeOne("2.2") + dist.parse("Dynamic: Platforms\nDynamic: Supported-Platforms") + self.assertEqual(list(dist.dynamic), ["Platforms", "Supported-Platforms"]) diff --git a/pkginfo2/tests/test_index.py b/tests/test_index.py similarity index 70% rename from pkginfo2/tests/test_index.py rename to tests/test_index.py index 85793d0..ad8205b 100644 --- a/pkginfo2/tests/test_index.py +++ b/tests/test_index.py @@ -1,9 +1,10 @@ import unittest -class IndexTests(unittest.TestCase): +class IndexTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.index import Index + return Index def _makeOne(self): @@ -18,47 +19,51 @@ def test_empty(self): def _makeDummy(self): from pkginfo2.distribution import Distribution + class DummyDistribution(Distribution): - name = 'dummy' - version = '1.0' + name = "dummy" + version = "1.0" return DummyDistribution() def test___getitem___miss(self): index = self._makeOne() - self.assertRaises(KeyError, index.__getitem__, 'nonesuch') + self.assertRaises(KeyError, index.__getitem__, "nonesuch") def test___setitem___value_not_dist(self): class NotDistribution: - name = 'dummy' - version = '1.0' + name = "dummy" + version = "1.0" + dummy = NotDistribution() index = self._makeOne() - self.assertRaises(ValueError, index.__setitem__, 'dummy-1.0', dummy) + self.assertRaises(ValueError, index.__setitem__, "dummy-1.0", dummy) def test___setitem___bad_key(self): index = self._makeOne() dummy = self._makeDummy() - self.assertRaises(ValueError, index.__setitem__, 'nonesuch', dummy) + self.assertRaises(ValueError, index.__setitem__, "nonesuch", dummy) def test___setitem___valid_key(self): index = self._makeOne() dummy = self._makeDummy() - index['dummy-1.0'] = dummy - self.assertTrue(index['dummy-1.0'] is dummy) + index["dummy-1.0"] = dummy + self.assertTrue(index["dummy-1.0"] is dummy) self.assertEqual(len(index), 1) self.assertEqual(len(index.keys()), 1) - self.assertEqual(list(index.keys())[0], 'dummy-1.0') + self.assertEqual(list(index.keys())[0], "dummy-1.0") self.assertEqual(len(index.values()), 1) self.assertEqual(list(index.values())[0], dummy) self.assertEqual(len(index.items()), 1) - self.assertEqual(list(index.items())[0], ('dummy-1.0', dummy)) + self.assertEqual(list(index.items())[0], ("dummy-1.0", dummy)) def test_add_not_dist(self): index = self._makeOne() + class NotDistribution: - name = 'dummy' - version = '1.0' + name = "dummy" + version = "1.0" + dummy = NotDistribution() self.assertRaises(ValueError, index.add, dummy) @@ -66,11 +71,11 @@ def test_add_valid_dist(self): index = self._makeOne() dummy = self._makeDummy() index.add(dummy) - self.assertTrue(index['dummy-1.0'] is dummy) + self.assertTrue(index["dummy-1.0"] is dummy) self.assertEqual(len(index), 1) self.assertEqual(len(index.keys()), 1) - self.assertEqual(list(index.keys())[0], 'dummy-1.0') + self.assertEqual(list(index.keys())[0], "dummy-1.0") self.assertEqual(len(index.values()), 1) self.assertEqual(list(index.values())[0], dummy) self.assertEqual(len(index.items()), 1) - self.assertEqual(list(index.items())[0], ('dummy-1.0', dummy)) + self.assertEqual(list(index.items())[0], ("dummy-1.0", dummy)) diff --git a/pkginfo2/tests/test_installed.py b/tests/test_installed.py similarity index 62% rename from pkginfo2/tests/test_installed.py rename to tests/test_installed.py index bfbcf6e..b5ab52c 100644 --- a/pkginfo2/tests/test_installed.py +++ b/tests/test_installed.py @@ -1,9 +1,10 @@ import unittest -class InstalledTests(unittest.TestCase): +class InstalledTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.installed import Installed + return Installed def _makeOne(self, filename=None, metadata_version=None): @@ -13,12 +14,12 @@ def _makeOne(self, filename=None, metadata_version=None): def test_ctor_w_egg_info_as_file(self): import os + where, _ = os.path.split(__file__) - funny = os.path.join(where, 'funny') + funny = os.path.join(where, "funny") installed = self._makeOne(funny) - self.assertEqual(installed.metadata_version, '1.0') - self.assertTrue(installed.package.endswith('funny')) - self.assertEqual(installed.name , 'funny') - self.assertEqual(installed.package_name , 'funny') - self.assertEqual(installed.version, '0.1') - + self.assertEqual(installed.metadata_version, "1.0") + self.assertTrue(installed.package.endswith("funny")) + self.assertEqual(installed.name, "funny") + self.assertEqual(installed.package_name, "funny") + self.assertEqual(installed.version, "0.1") diff --git a/pkginfo2/tests/test_sdist.py b/tests/test_sdist.py similarity index 69% rename from pkginfo2/tests/test_sdist.py rename to tests/test_sdist.py index f23073e..e00839d 100644 --- a/pkginfo2/tests/test_sdist.py +++ b/tests/test_sdist.py @@ -2,10 +2,11 @@ import tempfile import unittest -class SDistTests(unittest.TestCase): +class SDistTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.sdist import SDist + return SDist def _makeOne(self, filename=None, metadata_version=None): @@ -15,95 +16,108 @@ def _makeOne(self, filename=None, metadata_version=None): def _checkSample(self, sdist, filename): self.assertEqual(sdist.filename, filename) - self.assertEqual(sdist.name, 'mypackage') - self.assertEqual(sdist.version, '0.1') + self.assertEqual(sdist.name, "mypackage") + self.assertEqual(sdist.version, "0.1") self.assertEqual(sdist.keywords, None) self.assertEqual(list(sdist.supported_platforms), []) def _checkClassifiers(self, sdist): - self.assertEqual(list(sdist.classifiers), - ['Development Status :: 4 - Beta', - 'Environment :: Console (Text Based)', - ]) + self.assertEqual( + list(sdist.classifiers), + [ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)", + ], + ) def test_ctor_w_invalid_filename(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nonesuch-0.1.tar.gz' % d + filename = "%s/../tests/examples/nonesuch-0.1.tar.gz" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_wo_PKG_INFO(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nopkginfo-0.1.zip' % d + filename = "%s/../tests/examples/nopkginfo-0.1.zip" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_w_tar(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar' % d + filename = "%s/../tests/examples/mypackage-0.1.tar" % d sdist = self._makeOne(filename) - self.assertEqual(sdist.metadata_version, '1.0') + self.assertEqual(sdist.metadata_version, "1.0") self._checkSample(sdist, filename) def test_ctor_w_gztar(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.gz' % d + filename = "%s/../tests/examples/mypackage-0.1.tar.gz" % d sdist = self._makeOne(filename) - self.assertEqual(sdist.metadata_version, '1.0') + self.assertEqual(sdist.metadata_version, "1.0") self._checkSample(sdist, filename) def test_ctor_w_gztar_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.gz' % d - sdist = self._makeOne(filename, metadata_version='1.1') + filename = "%s/../tests/examples/mypackage-0.1.tar.gz" % d + sdist = self._makeOne(filename, metadata_version="1.1") self._checkSample(sdist, filename) - self.assertEqual(sdist.metadata_version, '1.1') + self.assertEqual(sdist.metadata_version, "1.1") self._checkClassifiers(sdist) def test_ctor_w_bztar(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.bz2' % d + filename = "%s/../tests/examples/mypackage-0.1.tar.bz2" % d sdist = self._makeOne(filename) - self.assertEqual(sdist.metadata_version, '1.0') + self.assertEqual(sdist.metadata_version, "1.0") self._checkSample(sdist, filename) def test_ctor_w_bztar_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.bz2' % d - sdist = self._makeOne(filename, metadata_version='1.1') - self.assertEqual(sdist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1.tar.bz2" % d + sdist = self._makeOne(filename, metadata_version="1.1") + self.assertEqual(sdist.metadata_version, "1.1") self._checkSample(sdist, filename) self._checkClassifiers(sdist) def test_ctor_w_zip(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d + filename = "%s/../tests/examples/mypackage-0.1.zip" % d sdist = self._makeOne(filename) - self.assertEqual(sdist.metadata_version, '1.0') + self.assertEqual(sdist.metadata_version, "1.0") self._checkSample(sdist, filename) def test_ctor_w_zip_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d - sdist = self._makeOne(filename, metadata_version='1.1') - self.assertEqual(sdist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1.zip" % d + sdist = self._makeOne(filename, metadata_version="1.1") + self.assertEqual(sdist.metadata_version, "1.1") self._checkSample(sdist, filename) self._checkClassifiers(sdist) def test_ctor_w_bogus(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.bogus' % d + filename = "%s/../tests/examples/mypackage-0.1.bogus" % d with self.assertRaises(ValueError): - self._makeOne(filename, metadata_version='1.1') + self._makeOne(filename, metadata_version="1.1") class UnpackedMixin(object): @@ -117,10 +131,12 @@ def tearDown(self): def _getTargetClass(self): from pkginfo2.sdist import UnpackedSDist + return UnpackedSDist def _getTopDirectory(self): import os + topnames = os.listdir(self.__tmpdir) if len(topnames) == 1: return os.path.join(self.__tmpdir, topnames[0]) @@ -131,7 +147,6 @@ def _getLoadFilename(self): return self._getTopDirectory() def _makeOne(self, filename=None, metadata_version=None): - archive, _, _ = self._getTargetClass()._get_archive(filename) try: archive.extractall(self.__tmpdir) @@ -152,7 +167,9 @@ def _checkSample(self, sdist, filename): class UnpackedSDistGivenDirectoryTests(UnpackedMixin, SDistTests): pass + class UnpackedSDistGivenFileSDistTests(UnpackedMixin, SDistTests): def _getLoadFilename(self): import os - return os.path.join(self._getTopDirectory(), 'setup.py') + + return os.path.join(self._getTopDirectory(), "setup.py") diff --git a/pkginfo2/tests/test_utils.py b/tests/test_utils.py similarity index 55% rename from pkginfo2/tests/test_utils.py rename to tests/test_utils.py index 1dc4a4a..b670856 100644 --- a/pkginfo2/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,120 +1,133 @@ import unittest -class Test_get_metadata(unittest.TestCase): +class Test_get_metadata(unittest.TestCase): def _callFUT(self, path, metadata_version=None): from pkginfo2.utils import get_metadata + if metadata_version is not None: return get_metadata(path, metadata_version) return get_metadata(path) def _checkMyPackage(self, dist, filename): self.assertEqual(dist.filename, filename) - self.assertEqual(dist.name, 'mypackage') - self.assertEqual(dist.version, '0.1') + self.assertEqual(dist.name, "mypackage") + self.assertEqual(dist.version, "0.1") self.assertEqual(dist.keywords, None) self.assertEqual(list(dist.supported_platforms), []) def _checkClassifiers(self, dist): - self.assertEqual(list(dist.classifiers), - ['Development Status :: 4 - Beta', - 'Environment :: Console (Text Based)', - ]) + self.assertEqual( + list(dist.classifiers), + [ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)", + ], + ) def test_w_gztar(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.gz' % d + filename = "%s/../tests/examples/mypackage-0.1.tar.gz" % d dist = self._callFUT(filename) - self.assertEqual(dist.metadata_version, '1.0') + self.assertEqual(dist.metadata_version, "1.0") self._checkMyPackage(dist, filename) def test_w_gztar_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.gz' % d - dist = self._callFUT(filename, metadata_version='1.1') - self.assertEqual(dist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1.tar.gz" % d + dist = self._callFUT(filename, metadata_version="1.1") + self.assertEqual(dist.metadata_version, "1.1") self._checkMyPackage(dist, filename) self._checkClassifiers(dist) def test_w_bztar(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.bz2' % d + filename = "%s/../tests/examples/mypackage-0.1.tar.bz2" % d dist = self._callFUT(filename) - self.assertEqual(dist.metadata_version, '1.0') + self.assertEqual(dist.metadata_version, "1.0") self._checkMyPackage(dist, filename) def test_w_bztar_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.tar.bz2' % d - dist = self._callFUT(filename, metadata_version='1.1') - self.assertEqual(dist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1.tar.bz2" % d + dist = self._callFUT(filename, metadata_version="1.1") + self.assertEqual(dist.metadata_version, "1.1") self._checkMyPackage(dist, filename) self._checkClassifiers(dist) def test_w_zip(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d + filename = "%s/../tests/examples/mypackage-0.1.zip" % d dist = self._callFUT(filename) - self.assertEqual(dist.metadata_version, '1.0') + self.assertEqual(dist.metadata_version, "1.0") self._checkMyPackage(dist, filename) def test_w_zip_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d - dist = self._callFUT(filename, metadata_version='1.1') - self.assertEqual(dist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1.zip" % d + dist = self._callFUT(filename, metadata_version="1.1") + self.assertEqual(dist.metadata_version, "1.1") self._checkMyPackage(dist, filename) self._checkClassifiers(dist) def test_w_egg(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1-py2.6.egg' % d + filename = "%s/../tests/examples/mypackage-0.1-py2.6.egg" % d dist = self._callFUT(filename) - self.assertEqual(dist.metadata_version, '1.0') + self.assertEqual(dist.metadata_version, "1.0") self._checkMyPackage(dist, filename) def test_w_egg_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1-py2.6.egg' % d - dist = self._callFUT(filename, metadata_version='1.1') - self.assertEqual(dist.metadata_version, '1.1') + filename = "%s/../tests/examples/mypackage-0.1-py2.6.egg" % d + dist = self._callFUT(filename, metadata_version="1.1") + self.assertEqual(dist.metadata_version, "1.1") self._checkMyPackage(dist, filename) self._checkClassifiers(dist) def test_w_wheel(self): import os + d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'mypackage-0.1-cp26-none-linux_x86_64.whl') % d + filename = ("%s/../tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl") % d dist = self._callFUT(filename) - self.assertEqual(dist.metadata_version, '2.0') + self.assertEqual(dist.metadata_version, "2.0") self._checkMyPackage(dist, filename) def test_w_wheel_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'mypackage-0.1-cp26-none-linux_x86_64.whl') % d - dist = self._callFUT(filename, metadata_version='1.1') - self.assertEqual(dist.metadata_version, '1.1') + filename = ("%s/../tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl") % d + dist = self._callFUT(filename, metadata_version="1.1") + self.assertEqual(dist.metadata_version, "1.1") self._checkMyPackage(dist, filename) self._checkClassifiers(dist) def test_w_directory_no_EGG_INFO(self): import os import warnings + dir, name = os.path.split(__file__) - subdir = os.path.join(dir, 'funny') + subdir = os.path.join(dir, "funny") old_filters = warnings.filters[:] - warnings.filterwarnings('ignore') + warnings.filterwarnings("ignore") try: dist = self._callFUT(subdir) self.assertEqual(dist.path, subdir) @@ -125,18 +138,20 @@ def test_w_directory_no_EGG_INFO(self): def test_w_directory(self): import os + dir, name = os.path.split(__file__) - subdir = os.path.join(dir, 'silly') + subdir = os.path.join(dir, "silly") dist = self._callFUT(subdir) - self.assertEqual(dist.metadata_version, '1.0') - self.assertEqual(dist.name, 'silly') - self.assertEqual(dist.version, '0.1') + self.assertEqual(dist.metadata_version, "1.0") + self.assertEqual(dist.name, "silly") + self.assertEqual(dist.version, "0.1") def test_w_directory_and_metadata_version(self): import os + dir, name = os.path.split(__file__) - subdir = os.path.join(dir, 'silly') - dist = self._callFUT(subdir, metadata_version='1.2') - self.assertEqual(dist.metadata_version, '1.2') - self.assertEqual(dist.name, 'silly') - self.assertEqual(dist.version, '0.1') + subdir = os.path.join(dir, "silly") + dist = self._callFUT(subdir, metadata_version="1.2") + self.assertEqual(dist.metadata_version, "1.2") + self.assertEqual(dist.name, "silly") + self.assertEqual(dist.version, "0.1") diff --git a/pkginfo2/tests/test_wheel.py b/tests/test_wheel.py similarity index 63% rename from pkginfo2/tests/test_wheel.py rename to tests/test_wheel.py index ca78fe5..945fcbb 100644 --- a/pkginfo2/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -1,9 +1,10 @@ import unittest -class WheelTests(unittest.TestCase): +class WheelTests(unittest.TestCase): def _getTargetClass(self): from pkginfo2.wheel import Wheel + return Wheel def _makeOne(self, filename=None, metadata_version=None): @@ -13,72 +14,78 @@ def _makeOne(self, filename=None, metadata_version=None): def _checkSample(self, wheel, filename): self.assertEqual(wheel.filename, filename) - self.assertEqual(wheel.name, 'mypackage') - self.assertEqual(wheel.version, '0.1') + self.assertEqual(wheel.name, "mypackage") + self.assertEqual(wheel.version, "0.1") self.assertEqual(wheel.keywords, None) def _checkClassifiers(self, wheel): - self.assertEqual(list(wheel.classifiers), - ['Development Status :: 4 - Beta', - 'Environment :: Console (Text Based)', - ]) + self.assertEqual( + list(wheel.classifiers), + [ + "Development Status :: 4 - Beta", + "Environment :: Console (Text Based)", + ], + ) self.assertEqual(list(wheel.supported_platforms), []) def test_ctor_w_bogus_filename(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nonesuch-0.1-any.whl' % d + filename = "%s/../tests/examples/nonesuch-0.1-any.whl" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_w_non_wheel(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/mypackage-0.1.zip' % d + filename = "%s/../tests/examples/mypackage-0.1.zip" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_wo_dist_info(self): import os + d, _ = os.path.split(__file__) - filename = '%s/../../docs/examples/nodistinfo-0.1-any.whl' % d + filename = "%s/../tests/examples/nodistinfo-0.1-any.whl" % d self.assertRaises(ValueError, self._makeOne, filename) def test_ctor_w_valid_wheel(self): import os + d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'mypackage-0.1-cp26-none-linux_x86_64.whl') % d + filename = ("%s/../tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl") % d wheel = self._makeOne(filename) - self.assertEqual(wheel.metadata_version, '2.0') + self.assertEqual(wheel.metadata_version, "2.0") self._checkSample(wheel, filename) self._checkClassifiers(wheel) def test_ctor_w_installed_wheel(self): import os + d, _ = os.path.split(__file__) - filename = ( - '%s/../../docs/examples/mypackage-0.1.dist-info') % d + filename = ("%s/../tests/examples/mypackage-0.1.dist-info") % d wheel = self._makeOne(filename) - self.assertEqual(wheel.metadata_version, '2.0') + self.assertEqual(wheel.metadata_version, "2.0") self._checkSample(wheel, filename) self._checkClassifiers(wheel) def test_ctor_w_valid_wheel_and_metadata_version(self): import os + d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'mypackage-0.1-cp26-none-linux_x86_64.whl') % d - wheel = self._makeOne(filename, metadata_version='1.1') - self.assertEqual(wheel.metadata_version, '1.1') + filename = ("%s/../tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl") % d + wheel = self._makeOne(filename, metadata_version="1.1") + self.assertEqual(wheel.metadata_version, "1.1") self._checkSample(wheel, filename) self._checkClassifiers(wheel) def test_ctor_w_valid_wheel_w_description_header(self): import os + d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'distlib-0.3.1-py2.py3-none-any.whl') % d - wheel = self._makeOne(filename, metadata_version='1.1') - self.assertEqual(wheel.metadata_version, '1.1') + filename = ("%s/../tests/examples/distlib-0.3.1-py2.py3-none-any.whl") % d + wheel = self._makeOne(filename, metadata_version="1.1") + self.assertEqual(wheel.metadata_version, "1.1") self.assertTrue(wheel.description) def test_ctor_w_valid_installed_wheel(self): @@ -88,8 +95,7 @@ def test_ctor_w_valid_installed_wheel(self): import zipfile d, _ = os.path.split(__file__) - filename = ('%s/../../docs/examples/' - 'mypackage-0.1-cp26-none-linux_x86_64.whl') % d + filename = ("%s/../tests/examples/mypackage-0.1-cp26-none-linux_x86_64.whl") % d try: # note: we mock a wheel installation by unzipping @@ -97,7 +103,7 @@ def test_ctor_w_valid_installed_wheel(self): with zipfile.ZipFile(filename) as zipf: zipf.extractall(test_dir) wheel = self._makeOne(filename) - self.assertEqual(wheel.metadata_version, '2.0') + self.assertEqual(wheel.metadata_version, "2.0") self._checkSample(wheel, filename) self._checkClassifiers(wheel) finally: diff --git a/pkginfo2/tests/manky/namespaced.manky-0.1.egg-info/PKG-INFO b/tests/wonky/EGG-INFO/PKG-INFO similarity index 100% rename from pkginfo2/tests/manky/namespaced.manky-0.1.egg-info/PKG-INFO rename to tests/wonky/EGG-INFO/PKG-INFO diff --git a/pkginfo2/tests/wonky/NOT-A-PACKAGE.txt b/tests/wonky/NOT-A-PACKAGE.txt similarity index 100% rename from pkginfo2/tests/wonky/NOT-A-PACKAGE.txt rename to tests/wonky/NOT-A-PACKAGE.txt diff --git a/pkginfo2/tests/manky/namespaced/__init__.py b/tests/wonky/namespaced/__init__.py similarity index 99% rename from pkginfo2/tests/manky/namespaced/__init__.py rename to tests/wonky/namespaced/__init__.py index 2e2033b..6d83202 100644 --- a/pkginfo2/tests/manky/namespaced/__init__.py +++ b/tests/wonky/namespaced/__init__.py @@ -1,7 +1,9 @@ # this is a namespace package try: import pkg_resources + pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/pkginfo2/tests/wonky/namespaced/wonky/__init__.py b/tests/wonky/namespaced/wonky/__init__.py similarity index 100% rename from pkginfo2/tests/wonky/namespaced/wonky/__init__.py rename to tests/wonky/namespaced/wonky/__init__.py diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b13b252..0000000 --- a/tox.ini +++ /dev/null @@ -1,37 +0,0 @@ -[tox] -envlist = - py27,pypy,py36,py37,py38,py39,py310,pypy3,cover2,cover3,docs - -[testenv] -usedevelop = true -commands = - python setup.py test -q - -[testenv:cover2] -basepython = - python2.7 -commands = - python setup.py nosetests --with-xunit --with-xcoverage -deps = - nose - coverage - nosexcover - -[testenv:cover3] -basepython = - python3.7 -commands = - python setup.py nosetests --with-xunit --with-xcoverage -deps = - nose - coverage - nosexcover - -[testenv:docs] -basepython = - python3.7 -commands = - sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html - sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest -deps = - Sphinx