diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 00000000..18a44aa0 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,37 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.9] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Give permission to run scripts + run: chmod +x ./docs/scripts/doc8_style_check.sh + + - name: Install Dependencies + run: pip install -e .[docs] + + - name: Check Sphinx Documentation build minimally + working-directory: ./docs + run: sphinx-build -E -W source build + + - name: Check for documentation style errors + working-directory: ./docs + run: ./scripts/doc8_style_check.sh + + diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..3a4fe279 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,27 @@ +name: Release library as a PyPI wheel and sdist on GH release creation + +on: + release: + types: [created] + +jobs: + build-and-publish-to-pypi: + name: Build and publish library to PyPI + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@master + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Install pypa/build + run: python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ + . + - name: Publish distribution to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1c26ef7a..a4c03b07 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,14 @@ +============================== +Changelog +============================== + +2022-03-21 + Release 7.0.2 + + * Relax dependency constraints + * Use latest skeleton and settings + + 2022-03-21 Release 7.0.1 diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e8..8424cbea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,7 @@ include *.rst include setup.* include configure* include requirements* -include .git* +include .giti* global-exclude *.py[co] __pycache__ *.*~ diff --git a/README.rst b/README.rst index 58785245..42170087 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,4 @@ +================= AboutCode Toolkit ================= diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 24d963e1..6ca19c4d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -31,13 +31,13 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml + - template: etc/ci/azure-posix.yml parameters: - job_name: win2016_cpython - image_name: vs2017-win2016 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: macos11_cpython + image_name: macos-11 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: - all: venv\Scripts\pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -47,3 +47,10 @@ jobs: test_suites: all: venv\Scripts\pytest -n 2 -vvs + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.7', '3.8', '3.9', '3.10'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index 557c5633..a52f539e 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ set -e # Source this script for initial configuration # Use configure --help for details # +# NOTE: please keep in sync with Windows script configure.bat +# # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default ################################ @@ -28,19 +30,13 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv -# Cleanable files and directories with the --clean option -CLEANABLE=" - build - tmp - bin - Lib - Scripts - include - venv" +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -55,11 +51,13 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin -# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " + +################################ +# Thirdparty package locations and index handling +# Find packages from the local thirdparty directory +if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" ################################ @@ -70,56 +68,22 @@ fi ################################ -# find a proper Python to run +# Find a proper Python to run # Use environment variables or a file if available. # Otherwise the latest Python by default. -if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE - if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then - PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") - else - PYTHON_EXECUTABLE=python3 +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi fi -fi - - -################################ -cli_help() { - echo An initial configuration script - echo " usage: ./configure [options]" - echo - echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. - echo - echo The options are: - echo " --clean: clean built and installed files and exit." - echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." - echo " --help: display this help message and exit." - echo - echo By default, the python interpreter version found in the path is used. - echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to - echo configure another Python executable interpreter to use. If this is not - echo set, a file named PYTHON_EXECUTABLE containing a single line with the - echo path of the Python executable to use will be checked last. - set +e - exit -} - - -clean() { - # Remove cleanable file and directories and files from the root dir. - echo "* Cleaning ..." - for cln in $CLEANABLE; - do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; - done - set +e - exit } +################################ create_virtualenv() { # create a virtualenv for Python # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -150,6 +114,7 @@ create_virtualenv() { } +################################ install_packages() { # install requirements in virtualenv # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -166,11 +131,43 @@ install_packages() { } +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} + + ################################ # Main command line entry point -CFG_DEV_MODE=0 CFG_REQUIREMENTS=$REQUIREMENTS -NO_INDEX="--no-index" # We are using getopts to parse option arguments that start with "-" while getopts :-: optchar; do @@ -178,17 +175,19 @@ while getopts :-: optchar; do -) case "${OPTARG}" in help ) cli_help;; - clean ) clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - init ) NO_INDEX="";; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" +find_python create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" . "$CFG_BIN_DIR/activate" + set +e diff --git a/configure.bat b/configure.bat index ed061613..41547cc5 100644 --- a/configure.bat +++ b/configure.bat @@ -14,6 +14,8 @@ @rem # Source this script for initial configuration @rem # Use configure --help for details +@rem # NOTE: please keep in sync with POSIX script configure + @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ @@ -26,6 +28,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -49,10 +52,10 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ @@ -64,9 +67,7 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point -set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" -set "NO_INDEX=--no-index" :again if not "%1" == "" ( @@ -74,20 +75,19 @@ if not "%1" == "" ( if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 ) - if "%1" EQU "--init" ( - set "NO_INDEX= " + if "%1" EQU "--docs" ( + set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" ) shift goto again ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" @rem ################################ -@rem # find a proper Python to run +@rem # Find a proper Python to run @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @@ -99,6 +99,8 @@ if not defined PYTHON_EXECUTABLE ( ) ) + +@rem ################################ :create_virtualenv @rem # create a virtualenv for Python @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -143,6 +145,7 @@ if %ERRORLEVEL% neq 0 ( ) +@rem ################################ :install_packages @rem # install requirements in virtualenv @rem # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -157,6 +160,9 @@ if %ERRORLEVEL% neq 0 ( %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% + +@rem ################################ +:create_bin_junction @rem # Create junction to bin to have the same directory between linux and windows if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" @@ -171,20 +177,15 @@ exit /b 0 @rem ################################ - :cli_help echo An initial configuration script echo " usage: configure [options]" echo " " echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo " " echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo " " echo By default, the python interpreter version found in the path is used. @@ -195,6 +196,7 @@ exit /b 0 exit /b 0 +@rem ################################ :clean @rem # Remove cleanable file and directories and files from the root dir. echo "* Cleaning ..." @@ -202,4 +204,4 @@ for %%F in (%CLEANABLE%) do ( rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) -exit /b 0 \ No newline at end of file +exit /b 0 diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst new file mode 100644 index 00000000..cde23dcd --- /dev/null +++ b/docs/source/skeleton-usage.rst @@ -0,0 +1,160 @@ +Usage +===== +A brand new project +------------------- +.. code-block:: bash + + git init my-new-repo + cd my-new-repo + git pull git@github.com:nexB/skeleton + + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + +From here, you can make the appropriate changes to the files for your specific project. + +Update an existing project +--------------------------- +.. code-block:: bash + + cd my-existing-project + git remote add skeleton git@github.com:nexB/skeleton + git fetch skeleton + git merge skeleton/main --allow-unrelated-histories + +This is also the workflow to use when updating the skeleton files in any given repository. + +Customizing +----------- + +You typically want to perform these customizations: + +- remove or update the src/README.rst and tests/README.rst files +- set project info and dependencies in setup.cfg +- check the configure and configure.bat defaults + +Initializing a project +---------------------- + +All projects using the skeleton will be expected to pull all of it dependencies +from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using +requirements.txt and/or requirements-dev.txt to determine what version of a +package to collect. By default, PyPI will not be used to find and collect +packages from. + +In the case where we are starting a new project where we do not have +requirements.txt and requirements-dev.txt and whose dependencies are not yet on +thirdparty.aboutcode.org/pypi, we run the following command after adding and +customizing the skeleton files to your project: + +.. code-block:: bash + + ./configure + +This will initialize the virtual environment for the project, pull in the +dependencies from PyPI and add them to the virtual environment. + + +Generating requirements.txt and requirements-dev.txt +---------------------------------------------------- + +After the project has been initialized, we can generate the requirements.txt and +requirements-dev.txt files. + +Ensure the virtual environment is enabled. + +.. code-block:: bash + + source venv/bin/activate + +To generate requirements.txt: + +.. code-block:: bash + + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ + +Replace \ with the version number of the Python being used, for example: +``venv/lib/python3.6/site-packages/`` + +To generate requirements-dev.txt after requirements.txt has been generated: + +.. code-block:: bash + + ./configure --dev + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ + +Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` + +.. code-block:: bash + + python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ + .\configure --dev + python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + + +Collecting and generating ABOUT files for dependencies +------------------------------------------------------ + +Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: + +.. code-block:: bash + + pip install -r etc/scripts/requirements.txt + +Once we have requirements.txt and requirements-dev.txt, we can fetch the project +dependencies as wheels and generate ABOUT files for them: + +.. code-block:: bash + + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt + +There may be issues with the generated ABOUT files, which will have to be +corrected. You can check to see if your corrections are valid by running: + +.. code-block:: bash + + python etc/scripts/check_thirdparty.py -d thirdparty + +Once the wheels are collected and the ABOUT files are generated and correct, +upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT +files from the thirdparty directory to the pypi directory at +https://github.com/nexB/thirdparty-packages + + +Usage after project initialization +---------------------------------- + +Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated +and the project dependencies and their ABOUT files have been uploaded to +thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically +when you update dependencies or use a new checkout. + +If the virtual env for the project becomes polluted, or you would like to remove +it, use the ``--clean`` option: + +.. code-block:: bash + + ./configure --clean + +Then you can run ``./configure`` again to set up the project virtual environment. + +To set up the project for development use: + +.. code-block:: bash + + ./configure --dev + +To update the project dependencies (adding, removing, updating packages, etc.), +update the dependencies in ``setup.cfg``, then run: + +.. code-block:: bash + + ./configure --clean # Remove existing virtual environment + source venv/bin/activate # Ensure virtual environment is activated + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt + pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files + +Ensure that the generated ABOUT files are valid, then take the dependency wheels +and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 7a9acff4..9fdc7f15 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -13,10 +13,8 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} @@ -24,19 +22,18 @@ jobs: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - python3 --version - python$(python_version) --version - echo "python$(python_version)" > PYTHON_EXECUTABLE - ./configure --dev - displayName: 'Run Configure' + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 03d89274..26b41116 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -13,27 +13,27 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --dev - displayName: 'Run Configure' + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index d8b00f98..edf82e44 100644 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -15,10 +15,10 @@ Pre-requisites * To generate or update pip requirement files, you need to start with a clean virtualenv as instructed below (This is to avoid injecting requirements - specific to the tools here in the main requirements). + specific to the tools used here in the main requirements). * For other usages, the tools here can run either in their own isolated - virtualenv best or in the the main configured development virtualenv. + virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: pip install --requirement etc/release/requirements.txt @@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files Scripts ~~~~~~~ -* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and - NOTICE files to populate a local a thirdparty directory strictly from our - remote repo and using only pinned packages listed in one or more pip - requirements file(s). Fetch only requirements for specific python versions and - operating systems. Optionally fetch the corresponding source distributions. - -* **publish_files.py** will upload/sync a thirdparty directory of files to our - remote repo. Requires a GitHub personal access token. - -* **build_wheels.py** will build a package binary wheel for multiple OS and - python versions. Optionally wheels that contain native code are built - remotely. Dependent wheels are optionally included. Requires Azure credentials - and tokens if building wheels remotely on multiple operatin systems. - -* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to - add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE - and .LICENSE files. Requires Azure credentials and tokens if requesting the - build of missing wheels remotely on multiple operatin systems. +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. * **check_thirdparty.py** will check a thirdparty directory for errors. -* **bootstrap.py** will bootstrap a thirdparty directory from a requirements - file(s) to add or build missing wheels, sources archives and create .ABOUT, - .NOTICE and .LICENSE files. Requires Azure credentials and tokens if - requesting the build of missing wheels remotely on multiple operatin systems. - - - -Usage -~~~~~ - -See each command line --help option for details. - -* (TODO) **add_package.py** will add or update a Python package including wheels, - sources and ABOUT files and this for multiple Python version and OSes(for use - with upload_packages.py afterwards) You will need an Azure personal access - token for buidling binaries and an optional DejaCode API key to post and fetch - new package versions there. TODO: explain how we use romp - Upgrade virtualenv app ---------------------- diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py deleted file mode 100644 index 31f2f553..00000000 --- a/etc/scripts/bootstrap.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import itertools - -import click - -import utils_thirdparty -from utils_thirdparty import Environment -from utils_thirdparty import PypiPackage - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file(s) to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built and " - "sources, ABOUT and LICENSE files fetched.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version(s) to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS(ses) to use for this build: one of linux, mac or windows.", -) -@click.option( - "-l", - "--latest-version", - is_flag=True, - help="Get the latest version of all packages, ignoring version specifiers.", -) -@click.option( - "--sync-dejacode", - is_flag=True, - help="Synchronize packages with DejaCode.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.help_option("-h", "--help") -def bootstrap( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_deps, - latest_version, - sync_dejacode, - build_remotely=False, -): - """ - Boostrap a thirdparty Python packages directory from pip requirements. - - Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for - the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible - with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` - OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and - .LICENSE files. - - Optionally ignore version specifiers and use the ``--latest-version`` - of everything. - - Sources and wheels are fetched with attempts first from PyPI, then our remote repository. - If missing wheels are built as needed. - """ - # rename variables for clarity since these are lists - requirements_files = requirements_file - python_versions = python_version - operating_systems = operating_system - - # create the environments we need - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - - # collect all packages to process from requirements files - # this will fail with an exception if there are packages we cannot find - - required_name_versions = set() - - for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) - required_name_versions.update(nvs) - if latest_version: - required_name_versions = set((name, None) for name, _ver in required_name_versions) - - print( - f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" - ) - - # fetch all available wheels, keep track of missing - # start with local, then remote, then PyPI - - print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") - # list of all the wheel filenames either pre-existing, fetched or built - # updated as we progress - available_wheel_filenames = [] - - local_packages_by_namever = { - (p.name, p.version): p - for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) - } - - # list of (name, version, environment) not local and to fetch - name_version_envt_to_fetch = [] - - # start with a local check - for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get( - ( - name, - version, - ) - ) - if local_pack: - supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) - if supported_wheels: - available_wheel_filenames.extend(w.filename for w in supported_wheels) - print( - f"====> No fetch or build needed. " - f"Local wheel already available for {name}=={version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - continue - - name_version_envt_to_fetch.append( - ( - name, - version, - envt, - ) - ) - - print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") - - # list of (name, version, environment) not fetch and to build - name_version_envt_to_build = [] - - # then check if the wheel can be fetched without building from remote and Pypi - for name, version, envt in name_version_envt_to_fetch: - - fetched_fwn = utils_thirdparty.fetch_package_wheel( - name=name, - version=version, - environment=envt, - dest_dir=thirdparty_dir, - ) - - if fetched_fwn: - available_wheel_filenames.append(fetched_fwn) - else: - name_version_envt_to_build.append( - ( - name, - version, - envt, - ) - ) - - # At this stage we have all the wheels we could obtain without building - for name, version, envt in name_version_envt_to_build: - print( - f"====> Need to build wheels for {name}=={version} on os: " - f"{envt.operating_system} for Python: {envt.python_version}" - ) - - packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build - ] - - print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") - - package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( - packages_and_envts=packages_and_envts_to_build, - build_remotely=build_remotely, - with_deps=with_deps, - dest_dir=thirdparty_dir, - ) - if wheel_filenames_built: - available_wheel_filenames.extend(available_wheel_filenames) - - for pack, envt in package_envts_not_built: - print( - f"====> FAILED to build any wheel for {pack.name}=={pack.version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - - print(f"==> FETCHING SOURCE DISTRIBUTIONS") - # fetch all sources, keep track of missing - # This is a list of (name, version) - utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - print(f"==> FETCHING ABOUT AND LICENSE FILES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - ############################################################################ - if sync_dejacode: - print(f"==> SYNC WITH DEJACODE") - # try to fetch from DejaCode any missing ABOUT - # create all missing DejaCode packages - pass - - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py deleted file mode 100644 index 8a28176e..00000000 --- a/etc/scripts/build_wheels.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-n", - "--name", - type=str, - metavar="PACKAGE_NAME", - required=True, - help="Python package name to add or build.", -) -@click.option( - "-v", - "--version", - type=str, - default=None, - metavar="VERSION", - help="Python package version to add or build.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def build_wheels( - name, - version, - thirdparty_dir, - python_version, - operating_system, - with_deps, - build_remotely, - remote_build_log_file, - verbose, -): - """ - Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and - optional VERSION. Build wheels compatible with all the `--python-version` - PYVER(s) and `--operating_system` OS(s). - - Build native wheels remotely if needed when `--build-remotely` and include - all dependencies with `--with-deps`. - """ - utils_thirdparty.add_or_upgrade_built_wheels( - name=name, - version=version, - python_versions=python_version, - operating_systems=operating_system, - dest_dir=thirdparty_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - -if __name__ == "__main__": - build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 4fea16c5..b052f25b 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,17 +16,39 @@ @click.command() @click.option( "-d", - "--thirdparty-dir", + "--dest", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def check_thirdparty_dir( + dest, + wheels, + sdists, +): """ - Check a thirdparty directory for problems. + Check a thirdparty directory for problems and print these on screen. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) if __name__ == "__main__": diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py deleted file mode 100644 index a78861e9..00000000 --- a/etc/scripts/fetch_built_wheels.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "--remote-build-log-file", - type=click.Path(readable=True), - metavar="LOG-FILE", - help="Path to a remote builds log file.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory to save built wheels.", -) -@click.option( - "--no-wait", - is_flag=True, - default=False, - help="Do not wait for build completion.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def fetch_remote_wheels( - remote_build_log_file, - thirdparty_dir, - no_wait, - verbose, -): - """ - Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines - build log file. - """ - utils_thirdparty.fetch_remotely_built_wheels( - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - no_wait=no_wait, - verbose=verbose, - ) - - -if __name__ == "__main__": - fetch_remote_wheels() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py deleted file mode 100644 index 9da9ce96..00000000 --- a/etc/scripts/fetch_requirements.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools - -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="INT", - multiple=True, - default=["36"], - show_default=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - multiple=True, - default=["linux"], - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "-s", - "--with-sources", - is_flag=True, - help="Fetch the corresponding source distributions.", -) -@click.option( - "-a", - "--with-about", - is_flag=True, - help="Fetch the corresponding ABOUT and LICENSE files.", -) -@click.option( - "--allow-unpinned", - is_flag=True, - help="Allow requirements without pinned versions.", -) -@click.option( - "-s", - "--only-sources", - is_flag=True, - help="Fetch only the corresponding source distributions.", -) -@click.option( - "-u", - "--remote-links-url", - type=str, - metavar="URL", - default=utils_thirdparty.REMOTE_LINKS_URL, - show_default=True, - help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", -) -@click.help_option("-h", "--help") -def fetch_requirements( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_sources, - with_about, - allow_unpinned, - only_sources, - remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, -): - """ - Fetch and save to THIRDPARTY_DIR all the required wheels for pinned - dependencies found in the `--requirement` FILE requirements file(s). Only - fetch wheels compatible with the provided `--python-version` and - `--operating-system`. - Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together - with a virtualenv.pyz app. - - Use exclusively wheel not from PyPI but rather found in the PyPI-like link - repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` - as a local directory path to a wheels directory if this is not a a URL. - """ - - # fetch wheels - python_versions = python_version - operating_systems = operating_system - requirements_files = requirements_file - - if not only_sources: - envs = itertools.product(python_versions, operating_systems) - envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) - - for env, reqf in itertools.product(envs, requirements_files): - - for package, error in utils_thirdparty.fetch_wheels( - environment=env, - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch wheel:", package, ":", error) - - # optionally fetch sources - if with_sources or only_sources: - - for reqf in requirements_files: - for package, error in utils_thirdparty.fetch_sources( - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch source:", package, ":", error) - - if with_about: - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - utils_thirdparty.find_problems( - dest_dir=thirdparty_dir, - report_missing_sources=with_sources or only_sources, - report_missing_wheels=not only_sources, - ) - - -if __name__ == "__main__": - fetch_requirements() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 00000000..89d17ded --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import os +import sys + +import click + +import utils_thirdparty +import utils_requirements + +TRACE = False +TRACE_DEEP = False + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEX_URLS, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, + use_cached_index, +): + """ + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. + + Download from the provided PyPI simple --index-url INDEX(s) URLs. + """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_required_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + + # create the environments matrix we need for wheels + environments = None + if wheels: + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_fetched = [] + wheels_not_found = [] + + sdists_fetched = [] + sdists_not_found = [] + + for name, version in sorted(required_name_versions): + nv = name, version + print(f"Processing: {name} @ {version}") + if wheels: + for environment in environments: + if TRACE: + print(f" ==> Fetching wheel for envt: {environment}") + fwfns = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + repos=repos, + ) + if fwfns: + wheels_fetched.extend(fwfns) + else: + wheels_not_found.append(f"{name}=={version} for: {environment}") + if TRACE: + print(f" NOT FOUND") + + if sdists: + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if fetched: + sdists_fetched.append(fetched) + else: + sdists_not_found.append(f"{name}=={version}") + if TRACE: + print(f" NOT FOUND") + + if wheels and wheels_not_found: + print(f"==> MISSING WHEELS") + for wh in wheels_not_found: + print(f" {wh}") + + if sdists and sdists_not_found: + print(f"==> MISSING SDISTS") + for sd in sdists_not_found: + print(f" {sd}") + + print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py deleted file mode 100644 index d664c9c4..00000000 --- a/etc/scripts/fix_thirdparty.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to fix.", -) -@click.option( - "--build-wheels", - is_flag=True, - help="Build all missing wheels .", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--strip-classifiers", - is_flag=True, - help="Remove danglingf classifiers", -) -@click.help_option("-h", "--help") -def fix_thirdparty_dir( - thirdparty_dir, - build_wheels, - build_remotely, - remote_build_log_file, - strip_classifiers, -): - """ - Fix a thirdparty directory of dependent package wheels and sdist. - - Multiple fixes are applied: - - fetch or build missing binary wheels - - fetch missing source distributions - - derive, fetch or add missing ABOUT files - - fetch missing .LICENSE and .NOTICE files - - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files - - Optionally build missing binary wheels for all supported OS and Python - version combos locally or remotely. - """ - if strip_classifiers: - print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - else: - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" - ) - - print("***FIND PROBLEMS***") - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 53db9b0a..03312ab3 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -5,86 +5,46 @@ # Copyright (c) 2010 David Wolever . All rights reserved. # originally from https://github.com/wolever/pip2pi +import hashlib import os import re import shutil - +from collections import defaultdict from html import escape from pathlib import Path +from typing import NamedTuple """ -name: pip compatibility tags -version: 20.3.1 -download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py -copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) -license_expression: mit -notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py - -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Generate a PyPI simple index froma directory. """ -get_wheel_from_filename = re.compile( - r"""^(?P(?P.+?)-(?P.*?)) - ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl)$""", - re.VERBOSE, -).match - -sdist_exts = ( - ".tar.gz", - ".tar.bz2", - ".zip", - ".tar.xz", -) -wheel_ext = ".whl" -app_ext = ".pyz" -dist_exts = sdist_exts + (wheel_ext, app_ext) class InvalidDistributionFilename(Exception): pass -def get_package_name_from_filename(filename, normalize=True): +def get_package_name_from_filename(filename): """ - Return the package name extracted from a package ``filename``. - Optionally ``normalize`` the name according to distribution name rules. + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 'foo' - >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") 'foo-bar' >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 'cython' >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 'python-ldap' - >>> get_package_name_from_filename("foo.whl") - Traceback (most recent call last): - ... - InvalidDistributionFilename: ... - >>> get_package_name_from_filename("foo.png") - Traceback (most recent call last): - ... - InvalidFilePackageName: ... + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass """ if not filename or not filename.endswith(dist_exts): raise InvalidDistributionFilename(filename) @@ -132,18 +92,99 @@ def get_package_name_from_filename(filename, normalize=True): if not name: raise InvalidDistributionFilename(filename) - if normalize: - name = name.lower().replace("_", "-") + name = normalize_name(name) return name -def build_pypi_index(directory, write_index=False): +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = f""" + + + Links for all packages + + """ + document.append(header) + + for _name, packages in packages_by_package_name.items(): + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI simple - directory index at ``directory``/simple/ populated with the proper PyPI simple - index directory structure crafted using symlinks. + Using a ``directory`` directory of wheels and sdists, create the a PyPI + simple directory index at ``directory``/simple/ populated with the proper + PyPI simple index directory structure crafted using symlinks. WARNING: The ``directory``/simple/ directory is removed if it exists. + NOTE: in addition to the a PyPI simple index.html there is also a links.html + index file generated which is suitable to use with pip's --find-links """ directory = Path(directory) @@ -153,14 +194,15 @@ def build_pypi_index(directory, write_index=False): shutil.rmtree(str(index_dir), ignore_errors=True) index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) - if write_index: - simple_html_index = [ - "PyPI Simple Index", - "", - ] + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '' '', + ] - package_names = set() for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name @@ -172,23 +214,99 @@ def build_pypi_index(directory, write_index=False): ): continue - pkg_name = get_package_name_from_filename(pkg_filename) + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) pkg_index_dir = index_dir / pkg_name pkg_index_dir.mkdir(parents=True, exist_ok=True) pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename pkg_indexed_file.symlink_to(link_target) - if write_index and pkg_name not in package_names: + if pkg_name not in packages_by_package_name: esc_name = escape(pkg_name) simple_html_index.append(f'{esc_name}
') - package_names.add(pkg_name) - if write_index: - simple_html_index.append("") - index_html = index_dir / "index.html" - index_html.write_text("\n".join(simple_html_index)) + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) if __name__ == "__main__": import sys diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 6f17a75f..07e26f77 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -8,37 +8,48 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-r", - "--requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements.txt", - show_default=True, - help="Path to the requirements file to update or create.", -) -@click.help_option("-h", "--help") -def gen_requirements(site_packages_dir, requirements_file): - """ + +def gen_requirements(): + description = """ Create or replace the `--requirements-file` file FILE requirements file with all locally installed Python packages.all Python packages found installed in `--site-packages-dir` """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + utils_requirements.lock_requirements( - requirements_file=requirements_file, - site_packages_dir=site_packages_dir, + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, ) diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ef804554..12cc06d3 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -8,51 +8,59 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-d", - "--dev-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements-dev.txt", - show_default=True, - help="Path to the dev requirements file to update or create.", -) -@click.option( - "-r", - "--main-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - default="requirements.txt", - metavar="FILE", - show_default=True, - help="Path to the main requirements file. Its requirements will be excluded " - "from the generated dev requirements.", -) -@click.help_option("-h", "--help") -def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): - """ + +def gen_dev_requirements(): + description = """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with all Python packages found installed in `--site-packages-dir`. Exclude package names also listed in the --main-requirements-file pip requirements FILE (that are assume to the production requirements and therefore to always be present in addition to the development requirements). """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + utils_requirements.lock_dev_requirements( - dev_requirements_file=dev_requirements_file, - main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir, + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, ) diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py deleted file mode 100644 index 86693631..00000000 --- a/etc/scripts/publish_files.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import hashlib -import os -import sys - -from pathlib import Path - -import click -import requests -import utils_thirdparty - -from github_release_retry import github_release_retry as grr - -""" -Create GitHub releases and upload files there. -""" - - -def get_files(location): - """ - Return an iterable of (filename, Path, md5) tuples for files in the `location` - directory tree recursively. - """ - for top, _dirs, files in os.walk(location): - for filename in files: - pth = Path(os.path.join(top, filename)) - with open(pth, "rb") as fi: - md5 = hashlib.md5(fi.read()).hexdigest() - yield filename, pth, md5 - - -def get_etag_md5(url): - """ - Return the cleaned etag of URL `url` or None. - """ - headers = utils_thirdparty.get_remote_headers(url) - headers = {k.lower(): v for k, v in headers.items()} - etag = headers.get("etag") - if etag: - etag = etag.strip('"').lower() - return etag - - -def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, -): - """ - Create or update a GitHub release at https://github.com// for - `tag_name` tag using the optional `description` for this release. - Use the provided `token` as a GitHub token for API calls authentication. - Upload all files found in the `directory` tree to that GitHub release. - Retry API calls up to `retry_limit` time to work around instability the - GitHub API. - - Remote files that are not the same as the local files are deleted and re- - uploaded. - """ - release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" - - # scrape release page HTML for links - urls_by_filename = { - os.path.basename(l): l - for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) - } - - # compute what is new, modified or unchanged - print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") - - new_to_upload = [] - unchanged_to_skip = [] - modified_to_delete_and_reupload = [] - for filename, pth, md5 in get_files(directory): - url = urls_by_filename.get(filename) - if not url: - print(f"{filename} content is NEW, will upload") - new_to_upload.append(pth) - continue - - out_of_date = get_etag_md5(url) != md5 - if out_of_date: - print(f"{url} content is CHANGED based on md5 etag, will re-upload") - modified_to_delete_and_reupload.append(pth) - else: - # print(f'{url} content is IDENTICAL, skipping upload based on Etag') - unchanged_to_skip.append(pth) - print(".") - - ghapi = grr.GithubApi( - github_api_url="https://api.github.com", - user=user, - repo=repo, - token=token, - retry_limit=retry_limit, - ) - - # yank modified - print( - f"Unpublishing {len(modified_to_delete_and_reupload)} published but " - f"locally modified files in {release_homepage_url}" - ) - - release = ghapi.get_release_by_tag(tag_name) - - for pth in modified_to_delete_and_reupload: - filename = os.path.basename(pth) - asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print(f" Unpublishing file: {filename}).") - response = ghapi.delete_asset(asset_id) - if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f"failed asset deletion: {response}") - - # finally upload new and modified - to_upload = new_to_upload + modified_to_delete_and_reupload - print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") - release = grr.Release(tag_name=tag_name, body=description) - grr.make_release(ghapi, release, to_upload) - - -TOKEN_HELP = ( - "The Github personal acess token is used to authenticate API calls. " - "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " - "See for details: https://github.com/settings/tokens and " - "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" -) - - -@click.command() -@click.option( - "--user-repo-tag", - help="The GitHub qualified repository user/name/tag in which " - "to create the release such as in nexB/thirdparty/pypi", - type=str, - required=True, -) -@click.option( - "-d", - "--directory", - help="The directory that contains files to upload to the release.", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, -) -@click.option( - "--token", - help=TOKEN_HELP, - default=os.environ.get("GITHUB_TOKEN", None), - type=str, - required=False, -) -@click.option( - "--description", - help="Text description for the release. Ignored if the release exists.", - default=None, - type=str, - required=False, -) -@click.option( - "--retry_limit", - help="Number of retries when making failing GitHub API calls. " - "Retrying helps work around transient failures of the GitHub API.", - type=int, - default=10, -) -@click.help_option("-h", "--help") -def publish_files( - user_repo_tag, - directory, - retry_limit=10, - token=None, - description=None, -): - """ - Publish all the files in DIRECTORY as assets to a GitHub release. - Either create or update/replace remote files' - """ - if not token: - click.secho("--token required option is missing.") - click.secho(TOKEN_HELP) - sys.exit(1) - - user, repo, tag_name = user_repo_tag.split("/") - - create_or_update_release_and_upload_directory( - user=user, - repo=repo, - tag_name=tag_name, - description=description, - retry_limit=retry_limit, - token=token, - directory=directory, - ) - - -if __name__ == "__main__": - publish_files() diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index 6591e49c..ebb404b7 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -1,12 +1,11 @@ aboutcode_toolkit -github-release-retry2 attrs commoncode click requests saneyaml -romp pip setuptools twine -wheel \ No newline at end of file +wheel +build \ No newline at end of file diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7753ea02..7c99a33b 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -14,95 +14,63 @@ """ Utilities to manage requirements files and call pip. NOTE: this should use ONLY the standard library and not import anything else -becasue this is used for boostrapping. +because this is used for boostrapping with no requirements installed. """ -def load_requirements(requirements_file="requirements.txt", force_pinned=True): +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): """ Yield package (name, version) tuples for each requirement in a `requirement` - file. Every requirement versions must be pinned if `force_pinned` is True. - Otherwise un-pinned requirements are returned with a None version + file. Only accept requirements pinned to an exact version. """ with open(requirements_file) as reqs: req_lines = reqs.read().splitlines(False) - return get_required_name_versions(req_lines, force_pinned) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) -def get_required_name_versions( - requirement_lines, - force_pinned=True, -): +def get_required_name_versions(requirement_lines, with_unpinned=False): """ Yield required (name, version) tuples given a`requirement_lines` iterable of - requirement text lines. Every requirement versions must be pinned if - `force_pinned` is True. Otherwise un-pinned requirements are returned with a - None version. - + requirement text lines. Only accept requirements pinned to an exact version. """ + for req_line in requirement_lines: req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if force_pinned: - if "==" not in req_line: - raise Exception(f"Requirement version is not pinned: {req_line}") - name = req_line - version = None - else: - if req_line.startswith("-"): - print(f"Requirement skipped, is not supported: {req_line}") - - if "==" in req_line: - name, _, version = req_line.partition("==") - version = version.lower().strip() - else: - # FIXME: we do not support unpinned requirements yet! - name = strip_reqs(req_line) - version = None - - name = name.lower().strip() - yield name, version - - -def strip_reqs(line): - """ - Return a name given a pip reuirement text ``line` striping version and - requirements. - - For example:: - - >>> s = strip_reqs("foo <=12, >=13,!=12.6") - >>> assert s == "foo" - """ - if "--" in line: - raise Exception(f"Unsupported requirement style: {line}") - - line = line.strip() - - ops = ">>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> try: + ... assert not get_required_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) """ - requires = [c for c in requires.splitlines(False) if c] - if not requires: - return [] - - requires = ["".join(r.split()) for r in requires if r and r.strip()] - return sorted(requires) + requirement = requirement and "".join(requirement.lower().split()) + assert requirement, f"specifier is required is empty:{requirement!r}" + name, operator, version = split_req(requirement) + assert name, f"Name is required: {requirement}" + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + return name, version def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): @@ -139,8 +107,49 @@ def lock_dev_requirements( def get_installed_reqs(site_packages_dir): """ - Return the installed pip requirements as text found in `site_packages_dir` as a text. + Return the installed pip requirements as text found in `site_packages_dir` + as a text. """ - # Also include these packages in the output with --all: wheel, distribute, setuptools, pip + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + assert req + # do not allow multiple constraints and tags + assert not any(c in req for c in ",;") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index a2fbe4e5..2d6f3e46 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -8,47 +8,45 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -from collections import defaultdict import email import itertools -import operator import os import re import shutil import subprocess -import tarfile import tempfile import time import urllib +from collections import defaultdict +from urllib.parse import quote_plus import attr import license_expression import packageurl import requests import saneyaml -import utils_pip_compatibility_tags -import utils_pypi_supported_tags - from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version -from utils_requirements import load_requirements + +import utils_pip_compatibility_tags """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. -- update pip requirement files from installed packages for prod. and dev. -- build and save wheels for all required packages -- also build variants for wheels with native code for all each supported - operating systems (Linux, macOS, Windows) and Python versions (3.x) - combinations using remote Ci jobs -- collect source distributions for all required packages -- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like - repository (using GitHub) -- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development Approach @@ -56,35 +54,65 @@ The processing is organized around these key objects: -- A PyPiPackage represents a PyPI package with its name and version. It tracks - the downloadable Distribution objects for that version: +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: - - one Sdist source Distribution object - - a list of Wheel binary Distribution objects + - one Sdist source Distribution + - a list of Wheel binary Distribution - A Distribution (either a Wheel or Sdist) is identified by and created from its - filename. It also has the metadata used to populate an .ABOUT file and - document origin and license. A Distribution can be fetched from Repository. - Metadata can be loaded from and dumped to ABOUT files and optionally from - DejaCode package data. + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" -- An Environment is a combination of a Python version and operating system. - A Wheel Distribution also has Python/OS tags is supports and these can be - supported in a given Environment. +""" +Wheel downloader + +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} -- Paths or URLs to "filenames" live in a Repository, either a plain - LinksRepository (an HTML page listing URLs or a local directory) or a - PypiRepository (a PyPI simple index where each package name has an HTML page - listing URLs to all distribution types and versions). - Repositories and Distributions are related through filenames. + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message - The Wheel models code is partially derived from the mit-licensed pip and the - Distribution/Wheel/Sdist design has been heavily inspired by the packaging- - dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung """ TRACE = False +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False # Supported environments PYTHON_VERSIONS = "36", "37", "38", "39", "310" @@ -106,16 +134,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m"], - "37": ["cp37", "cp37m"], - "38": ["cp38", "cp38m"], - "39": ["cp39", "cp39m"], - "310": ["cp310", "cp310m"], - "36": ["cp36", "abi3"], - "37": ["cp37", "abi3"], - "38": ["cp38", "abi3"], - "39": ["cp39", "abi3"], - "310": ["cp310", "abi3"], + "36": ["cp36", "cp36m", "abi3"], + "37": ["cp37", "cp37m", "abi3"], + "38": ["cp38", "cp38m", "abi3"], + "39": ["cp39", "cp39m", "abi3"], + "310": ["cp310", "cp310m", "abi3"], } PLATFORMS_BY_OS = { @@ -144,6 +167,16 @@ def get_python_dot_version(version): "macosx_10_15_x86_64", "macosx_11_0_x86_64", "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", # 'macosx_11_0_arm64', ], "windows": [ @@ -154,12 +187,19 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" +################################################################################ + +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( ".tar.gz", - ".tar.bz2", ".zip", ".tar.xz", ) @@ -171,169 +211,105 @@ def get_python_dot_version(version): ) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = "https://pypi.org/simple" - LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() -# time to wait build for in seconds, as a string -# 0 measn no wait -DEFAULT_ROMP_BUILD_WAIT = "5" +collect_urls = re.compile('href="([^"]+)"').findall ################################################################################ -# -# Fetch remote wheels and sources locally -# +# Fetch wheels and sources locally ################################################################################ -def fetch_wheels( - environment=None, - requirements_file="requirements.txt", - allow_unpinned=False, - dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, -): - """ - Download all of the wheel of packages listed in the ``requirements_file`` - requirements file into ``dest_dir`` directory. - - Only get wheels for the ``environment`` Enviromnent constraints. If the - provided ``environment`` is None then the current Python interpreter - environment is used implicitly. +class DistributionNotFound(Exception): + pass - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. - Use exclusively direct downloads from a remote repo at URL - ``remote_links_url``. If ``remote_links_url`` is a path, use this as a - directory of links instead of a URL. +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): + """ + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. - Yield tuples of (PypiPackage, error) where is None on success. + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. """ - missed = [] + if TRACE_DEEP: + print(f" download_wheel: {name}=={version} for envt: {environment}") - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + if not repos: + repos = DEFAULT_PYPI_REPOS - try: - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) - except Exception as e: - raise Exception( - dict( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) from e + fetched_wheel_filenames = [] - fetched_filenames = set() - for name, version, package in rrp: + for repo in repos: + package = repo.get_package_version(name=name, version=version) if not package: - missed.append( - ( - name, - version, + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_wheels: Missing package in remote repo: {nv}" + continue - else: - fetched_filename = package.fetch_wheel( - environment=environment, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) - if fetched_filename: - fetched_filenames.add(fetched_filename) - error = None - else: - if fetched_filename in fetched_filenames: - error = None - else: - error = f"Failed to fetch" - yield package, error - - if missed: - rr = get_remote_repo() - print() - print(f"===> fetch_wheels: Missed some packages") - for n, v in missed: - nv = f"{n}=={v}" if v else n - print(f"Missed package {nv} in remote repo, has only:") - for pv in rr.get_versions(n): - print(" ", pv) - raise Exception("Missed some packages in remote repo") - - -def fetch_sources( - requirements_file="requirements.txt", - allow_unpinned=False, - dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, -): - """ - Download all of the dependent package sources listed in the - ``requirements_file`` requirements file into ``dest_dir`` destination - directory. + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break - Use direct downloads to achieve this (not pip download). Use exclusively the - packages found from a remote repo at URL ``remote_links_url``. If - ``remote_links_url`` is a path, use this as a directory of links instead of - a URL. + return fetched_wheel_filenames - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. - Yield tuples of (PypiPackage, error message) for each package where error - message will empty on success. +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ - missed = [] + Download the sdist source distribution of package ``name`` and ``version`` + into the ``dest_dir`` directory. Return a fetched filename or None. - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. + """ + if TRACE: + print(f" download_sdist: {name}=={version}") - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) + if not repos: + repos = DEFAULT_PYPI_REPOS + + fetched_sdist_filename = None + + for repo in repos: + package = repo.get_package_version(name=name, version=version) - for name, version, package in rrp: if not package: - missed.append( - ( - name, - name, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_sources: Missing package in remote repo: {nv}" + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue - elif not package.sdist: - yield package, f"Missing sdist in links" + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) - else: - fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f"Failed to fetch" if not fetched else None - yield package, error - if missed: - raise Exception(f"Missing source packages in {remote_links_url}", missed) + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break + + return fetched_sdist_filename ################################################################################ @@ -367,17 +343,6 @@ def normalize_name(name): """ return name and re.sub(r"[-_.]+", "-", name).lower() or name - @staticmethod - def standardize_name(name): - """ - Return a standardized package name, e.g. lowercased and using - not _ - """ - return name and re.sub(r"[-_]+", "-", name).lower() or name - - @property - def name_ver(self): - return f"{self.name}-{self.version}" - def sortable_name_version(self): """ Return a tuple of values to sort by name, then version. @@ -387,13 +352,13 @@ def sortable_name_version(self): @classmethod def sorted(cls, namevers): - return sorted(namevers, key=cls.sortable_name_version) + return sorted(namevers or [], key=cls.sortable_name_version) @attr.attributes class Distribution(NameVer): - # field names that can be updated from another dist of mapping + # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", "copyright", @@ -415,7 +380,7 @@ class Distribution(NameVer): repr=False, type=str, default="", - metadata=dict(help="Path or download URL."), + metadata=dict(help="Path or URL"), ) sha256 = attr.ib( @@ -542,25 +507,79 @@ def package_url(self): """ Return a Package URL string of self. """ - return str(packageurl.PackageURL(**self.purl_identifiers())) + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith("https://"): - return self.path_or_url - else: - return self.get_best_download_url() + return self.get_best_download_url() + + def get_best_download_url(self, repos=tuple()): + """ + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. + """ + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE_DEEP: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + # FIXME: + fetch_and_save( + path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, + as_text=False, + ) + return self.filename @property def about_filename(self): return f"{self.filename}.ABOUT" - def has_about_file(self, dest_dir=THIRDPARTY_DIR): - return os.path.exists(os.path.join(dest_dir, self.about_filename)) - @property def about_download_url(self): - return self.build_remote_download_url(self.about_filename) + return f"{ABOUT_BASE_URL}/{self.about_filename}" @property def notice_filename(self): @@ -568,13 +587,13 @@ def notice_filename(self): @property def notice_download_url(self): - return self.build_remote_download_url(self.notice_filename) + return f"{ABOUT_BASE_URL}/{self.notice_filename}" @classmethod def from_path_or_url(cls, path_or_url): """ Return a distribution built from the data found in the filename of a - `path_or_url` string. Raise an exception if this is not a valid + ``path_or_url`` string. Raise an exception if this is not a valid filename. """ filename = os.path.basename(path_or_url.strip("/")) @@ -601,122 +620,10 @@ def from_filename(cls, filename): Return a distribution built from the data found in a `filename` string. Raise an exception if this is not a valid filename """ + filename = os.path.basename(filename.strip("/")) clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - @classmethod - def from_data(cls, data, keep_extra=False): - """ - Return a distribution built from a `data` mapping. - """ - filename = data["filename"] - dist = cls.from_filename(filename) - dist.update(data, keep_extra=keep_extra) - return dist - - @classmethod - def from_dist(cls, data, dist): - """ - Return a distribution built from a `data` mapping and update it with data - from another dist Distribution. Return None if it cannot be created - """ - # We can only create from a dist of the same package - has_same_key_fields = all( - data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") - ) - if not has_same_key_fields: - print( - f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) - if not has_key_field_values: - print( - f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - data = dict(data) - # do not overwrite the data with the other dist - # only supplement - data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)}) - return cls.from_data(data) - - @classmethod - def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): - """ - Return a direct download URL for a file in our remote repo - """ - return f"{base_url}/{filename}" - - def get_best_download_url(self): - """ - Return the best download URL for this distribution where best means that - PyPI is better and our own remote repo URLs are second. - If none is found, return a synthetic remote URL. - """ - name = self.normalized_name - version = self.version - filename = self.filename - - pypi_package = get_pypi_package(name=name, version=version) - if pypi_package: - pypi_url = pypi_package.get_url_for_filename(filename) - if pypi_url: - return pypi_url - - remote_package = get_remote_package(name=name, version=version) - if remote_package: - remote_url = remote_package.get_url_for_filename(filename) - if remote_url: - return remote_url - else: - # the package may not have been published yet, so we craft a URL - # using our remote base URL - return self.build_remote_download_url(self.filename) - - def purl_identifiers(self, skinny=False): - """ - Return a mapping of non-empty identifier name/values for the purl - fields. If skinny is True, only inlucde type, namespace and name. - """ - identifiers = dict( - type=self.type, - namespace=self.namespace, - name=self.name, - ) - - if not skinny: - identifiers.update( - version=self.version, - subpath=self.subpath, - qualifiers=self.qualifiers, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - - def identifiers(self, purl_as_fields=True): - """ - Return a mapping of non-empty identifier name/values. - Return each purl fields separately if purl_as_fields is True. - Otherwise return a package_url string for the purl. - """ - if purl_as_fields: - identifiers = self.purl_identifiers() - else: - identifiers = dict(package_url=self.package_url) - - identifiers.update( - download_url=self.download_url, - filename=self.filename, - md5=self.md5, - sha1=self.sha1, - package_url=self.package_url, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. @@ -781,9 +688,11 @@ def save_if_modified(location, content): fo.write(content) return True + as_about = self.to_about() + save_if_modified( location=os.path.join(dest_dir, self.about_filename), - content=saneyaml.dump(self.to_about()), + content=saneyaml.dump(as_about), ) notice_text = self.notice_text and self.notice_text.strip() @@ -844,7 +753,10 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url) + about_text = CACHE.get( + path_or_url=self.about_download_url, + as_text=True, + ) except RemoteNotFetchedException: return False @@ -855,7 +767,10 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) + notice_text = CACHE.get( + path_or_url=self.notice_download_url, + as_text=True, + ) if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: @@ -892,26 +807,23 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): return False return True - def get_pip_hash(self): - """ - Return a pip hash option string as used in requirements for this dist. - """ - assert self.sha256, f"Missinh SHA256 for dist {self}" - return f"--hash=sha256:{self.sha256}" - def get_license_keys(self): try: - keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) except license_expression.ExpressionParseError: return ["unknown"] return keys - def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ - Fetch license files is missing in `dest_dir`. + Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - paths_or_urls = get_remote_repo().links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -924,12 +836,12 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try remotely first - lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -939,10 +851,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -960,9 +872,17 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = "zip" if self.filename.endswith(".whl") else None - dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right @@ -983,7 +903,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f"!!!!PKG-INFO not found in {self.filename}") + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) @@ -1075,10 +995,102 @@ def update(self, data, overwrite=False, keep_extra=True): return updated +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + class InvalidDistributionFilename(Exception): pass +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + @attr.attributes class Sdist(Distribution): @@ -1095,21 +1107,11 @@ def from_filename(cls, filename): Return a Sdist object built from a filename. Raise an exception if this is not a valid sdist filename """ - name_ver = None - extension = None - - for ext in EXTENSIONS_SDIST: - if filename.endswith(ext): - name_ver, extension, _ = filename.rpartition(ext) - break - - if not extension or not name_ver: + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition("-") - - if not name or not version: - raise InvalidDistributionFilename(filename) + name, version, extension = name_ver_ext return cls( type="pypi", @@ -1243,15 +1245,12 @@ def is_supported_by_tags(self, tags): """ Return True is this wheel is compatible with one of a list of PEP 425 tags. """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) return not self.tags.isdisjoint(tags) - def is_supported_by_environment(self, environment): - """ - Return True if this wheel is compatible with the Environment - `environment`. - """ - return not self.is_supported_by_tags(environment.tags) - def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be @@ -1300,14 +1299,14 @@ def is_pure_wheel(filename): @attr.attributes class PypiPackage(NameVer): """ - A Python package with its "distributions", e.g. wheels and source - distribution , ABOUT files and licenses or notices. + A Python package contains one or more wheels and one source distribution + from a repository. """ sdist = attr.ib( repr=False, - type=str, - default="", + type=Sdist, + default=None, metadata=dict(help="Sdist source distribution for this package."), ) @@ -1318,32 +1317,14 @@ class PypiPackage(NameVer): metadata=dict(help="List of Wheel for this package"), ) - @property - def specifier(self): - """ - A requirement specifier for this package - """ - if self.version: - return f"{self.name}=={self.version}" - else: - return self.name - - @property - def specifier_with_hashes(self): - """ - Return a requirement specifier for this package with --hash options for - all its distributions - """ - items = [self.specifier] - items += [d.get_pip_hashes() for d in self.get_distributions()] - return " \\\n ".join(items) - - def get_supported_wheels(self, environment): + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the Environment `environment`. """ envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) for wheel in self.wheels: if wheel.is_supported_by_tags(envt_tags): yield wheel @@ -1369,6 +1350,8 @@ def package_from_dists(cls, dists): >>> assert package.wheels == [w1, w2] """ dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") if not dists: return @@ -1379,13 +1362,21 @@ def package_from_dists(cls, dists): package = PypiPackage(name=normalized_name, version=version) for dist in dists: - if dist.normalized_name != normalized_name or dist.version != version: + if dist.normalized_name != normalized_name: if TRACE: print( - f" Skipping inconsistent dist name and version: {dist} " - f'Expected instead package name: {normalized_name} and version: "{version}"' + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" ) continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue if isinstance(dist, Sdist): package.sdist = dist @@ -1396,174 +1387,50 @@ def package_from_dists(cls, dists): else: raise Exception(f"Unknown distribution type: {dist}") + if TRACE_DEEP: + print(f"package_from_dists: {package}") + return package @classmethod - def packages_from_one_path_or_url(cls, path_or_url): + def packages_from_dir(cls, directory): """ - Yield PypiPackages built from files found in at directory path or the - URL to an HTML page (that will be fetched). + Yield PypiPackages built from files found in at directory path. """ - extracted_paths_or_urls = get_paths_or_urls(path_or_url) - return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls) + base = os.path.abspath(directory) + + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. """ - dists = cls.get_dists(paths_or_urls) + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( dists, key=NameVer.sortable_name_version, ): - yield PypiPackage.package_from_dists(dists_of_package) - - @classmethod - def get_versions_from_path_or_url(cls, name, path_or_url): - """ - Return a subset list from a list of PypiPackages version at `path_or_url` - that match PypiPackage `name`. - """ - packages = cls.packages_from_one_path_or_url(path_or_url) - return cls.get_versions(name, packages) + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package @classmethod - def get_versions(cls, name, packages): + def dists_from_paths_or_urls(cls, paths_or_urls): """ - Return a subset list of package versions from a list of `packages` that - match PypiPackage `name`. - The list is sorted by version from oldest to most recent. - """ - norm_name = NameVer.normalize_name(name) - versions = [p for p in packages if p.normalized_name == norm_name] - return cls.sorted(versions) - - @classmethod - def get_latest_version(cls, name, packages): - """ - Return the latest version of PypiPackage `name` from a list of `packages`. - """ - versions = cls.get_versions(name, packages) - if not versions: - return - return versions[-1] - - @classmethod - def get_outdated_versions(cls, name, packages): - """ - Return all versions except the latest version of PypiPackage `name` from a - list of `packages`. - """ - versions = cls.get_versions(name, packages) - return versions[:-1] - - @classmethod - def get_name_version(cls, name, version, packages): - """ - Return the PypiPackage with `name` and `version` from a list of `packages` - or None if it is not found. - If `version` is None, return the latest version found. - """ - if version is None: - return cls.get_latest_version(name, packages) - - nvs = [p for p in cls.get_versions(name, packages) if p.version == version] - - if not nvs: - return - - if len(nvs) == 1: - return nvs[0] - - raise Exception(f"More than one PypiPackage with {name}=={version}") - - def fetch_wheel( - self, - environment=None, - fetched_filenames=None, - dest_dir=THIRDPARTY_DIR, - ): - """ - Download a binary wheel of this package matching the ``environment`` - Enviromnent constraints into ``dest_dir`` directory. - - Return the wheel filename if it was fetched, None otherwise. - - If the provided ``environment`` is None then the current Python - interpreter environment is used implicitly. Do not refetch wheel if - their name is in a provided ``fetched_filenames`` set. - """ - fetched_wheel_filename = None - if fetched_filenames is not None: - fetched_filenames = fetched_filenames - else: - fetched_filenames = set() - - supported_wheels = list(self.get_supported_wheels(environment)) - for wheel in supported_wheels: - - if wheel.filename not in fetched_filenames: - fetch_and_save_path_or_url( - filename=wheel.filename, - path_or_url=wheel.path_or_url, - dest_dir=dest_dir, - as_text=False, - ) - fetched_filenames.add(wheel.filename) - fetched_wheel_filename = wheel.filename - - # TODO: what if there is more than one? - break - - return fetched_wheel_filename - - def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): - """ - Download the source distribution into `dest_dir` directory. Return the - fetched filename if it was fetched, False otherwise. - """ - if self.sdist: - assert self.sdist.filename - if TRACE: - print("Fetching source for package:", self.name, self.version) - fetch_and_save_path_or_url( - filename=self.sdist.filename, - dest_dir=dest_dir, - path_or_url=self.sdist.path_or_url, - as_text=False, - ) - if TRACE: - print(" --> file:", self.sdist.filename) - return self.sdist.filename - else: - print(f"Missing sdist for: {self.name}=={self.version}") - return False - - def delete_files(self, dest_dir=THIRDPARTY_DIR): - """ - Delete all PypiPackage files from `dest_dir` including wheels, sdist and - their ABOUT files. Note that we do not delete licenses since they can be - shared by several packages: therefore this would be done elsewhere in a - function that is aware of all used licenses. - """ - for to_delete in self.wheels + [self.sdist]: - if not to_delete: - continue - tdfn = to_delete.filename - for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: - target = os.path.join(dest_dir, deletable) - if os.path.exists(target): - print(f"Deleting outdated {target}") - fileutils.delete(target) - - @classmethod - def get_dists(cls, paths_or_urls): - """ - Return a list of Distribution given a list of - `paths_or_urls` to wheels or source distributions. + Return a list of Distribution given a list of + ``paths_or_urls`` to wheels or source distributions. Each Distribution receives two extra attributes: - the path_or_url it was created from @@ -1574,34 +1441,46 @@ def get_dists(cls, paths_or_urls): ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl - ... httsp://example.com/bar/bitarray-0.8.1.tar.gz - ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.get_dists(paths_or_urls)) + ... https://example.com/bar/bitarray-0.8.1.tar.gz + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: - ... r.filename = '' - ... r.path_or_url = '' - >>> expected = [ - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['linux_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['win_amd64']), - ... Sdist(name='bitarray', version='0.8.1') - ... ] - >>> assert expected == result - """ + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 + """ + dists = [] + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: - yield Distribution.from_path_or_url(path_or_url) + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, + ) except InvalidDistributionFilename: - if TRACE: - print(f"Skipping invalid distribution from: {path_or_url}") + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") continue + return dists def get_distributions(self): """ @@ -1626,10 +1505,11 @@ class Environment: """ An Environment describes a target installation environment with its supported Python version, ABI, platform, implementation and related - attributes. We can use these to pass as `pip download` options and force - fetching only the subset of packages that match these Environment - constraints as opposed to the current running Python interpreter - constraints. + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. """ python_version = attr.ib( @@ -1648,18 +1528,21 @@ class Environment: type=str, default="cp", metadata=dict(help="Python implementation supported by this environment."), + repr=False, ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help="List of ABI tags supported by this environment."), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, ) platforms = attr.ib( type=list, default=attr.Factory(list), metadata=dict(help="List of platform tags supported by this environment."), + repr=False, ) @classmethod @@ -1677,18 +1560,20 @@ def from_pyver_and_os(cls, python_version, operating_system): def get_pip_cli_options(self): """ - Return a list of pip command line options for this environment. + Return a list of pip download command line options for this environment. """ options = [ "--python-version", self.python_version, "--implementation", self.implementation, - "--abi", - self.abi, ] + for abi in self.abis: + options.extend(["--abi", abi]) + for platform in self.platforms: options.extend(["--platform", platform]) + return options def tags(self): @@ -1713,58 +1598,125 @@ def tags(self): @attr.attributes -class Repository: +class PypiSimpleRepository: """ - A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc. + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. """ - packages_by_normalized_name = attr.ib( - type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), ) - packages_by_normalized_name_version = attr.ib( + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( type=dict, - default=attr.Factory(dict), - metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), + ) + + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), ) - def get_links(self, *args, **kwargs): - raise NotImplementedError() + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), + ) - def get_versions(self, name): + def _get_package_versions_map(self, name): """ - Return a list of all available PypiPackage version for this package name. - The list may be empty. + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest """ - raise NotImplementedError() + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") - def get_package(self, name, version): + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions + + def get_package_versions(self, name): + """ + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. + """ + return dict(self._get_package_versions_map(name)) + + def get_package_version(self, name, version=None): """ Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. """ - raise NotImplementedError() + if not version: + versions = list(self._get_package_versions_map(name).values()) + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) - def get_latest_version(self, name): + def fetch_links(self, normalized_name): """ - Return the latest PypiPackage version for this package name or None. + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. """ - raise NotImplementedError() + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + return links + + +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} @attr.attributes -class LinksRepository(Repository): +class LinksRepository: """ - Represents a simple links repository which is either a local directory with - Python wheels and sdist or a remote URL to an HTML with links to these. - (e.g. suitable for use with pip --find-links). + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. """ - path_or_url = attr.ib( + url = attr.ib( type=str, default="", - metadata=dict(help="Package directory path or URL"), + metadata=dict(help="Links directory URL"), ) links = attr.ib( @@ -1773,82 +1725,76 @@ class LinksRepository(Repository): metadata=dict(help="List of links available in this repo"), ) + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), + ) + def __attrs_post_init__(self): if not self.links: - self.links = get_paths_or_urls(links_url=self.path_or_url) - if not self.packages_by_normalized_name: - for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links): - normalized_name = p.normalized_name - self.packages_by_normalized_name[normalized_name].append(p) - self.packages_by_normalized_name_version[(normalized_name, p.version)] = p - - def get_links(self, *args, **kwargs): - return self.links or [] - - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - return self.packages_by_normalized_name.get(name, []) - - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) - - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + self.links = self.find_links() + def find_links(self, _CACHE=[]): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + if _CACHE: + return _CACHE -@attr.attributes -class PypiRepository(Repository): - """ - Represents the public PyPI simple index. - It is populated lazily based on requested packages names - """ + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() - simple_url = attr.ib( - type=str, - default=PYPI_SIMPLE_URL, - metadata=dict(help="Base PyPI simple URL for this index."), - ) + if TRACE_DEEP: + print(f"Base URL {base_url}") - links_by_normalized_name = attr.ib( - type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [links]} available in this repo"), - ) + text = CACHE.get( + path_or_url=links_url, + as_text=True, + force=not self.use_cached_index, + ) - def _fetch_links(self, name): - name = name and NameVer.normalize_name(name) - return find_pypi_links(name=name, simple_url=self.simple_url) + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - if name in self.links_by_normalized_name: - return + plink = urllib.parse.urlsplit(link) - links = self._fetch_links(name) - self.links_by_normalized_name[name] = links + if plink.scheme: + # full URL kept as-is + url = link - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - self.packages_by_normalized_name[name] = packages + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p + else: + # relative link + url = f"{links_url}/{link}" - def get_links(self, name, *args, **kwargs): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + if TRACE_DEEP: + print(f"Adding URL: {url}") - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.packages_by_normalized_name.get(name, []) + links.append(url) - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if TRACE: + print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) + return links - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) + return _LINKS_REPO[url] ################################################################################ @@ -1862,51 +1808,7 @@ def get_local_packages(directory=THIRDPARTY_DIR): Return the list of all PypiPackage objects built from a local directory. Return an empty list if the package cannot be found. """ - return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory)) - - -def get_local_repo(directory=THIRDPARTY_DIR): - return LinksRepository(path_or_url=directory) - - -_REMOTE_REPO = None - - -def get_remote_repo(remote_links_url=REMOTE_LINKS_URL): - global _REMOTE_REPO - if not _REMOTE_REPO: - _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url) - return _REMOTE_REPO - - -def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): - """ - Return a PypiPackage or None. - """ - try: - return get_remote_repo(remote_links_url).get_package(name, version) - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - -_PYPI_REPO = None - - -def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL): - global _PYPI_REPO - if not _PYPI_REPO: - _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url) - return _PYPI_REPO - - -def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): - """ - Return a PypiPackage or None. - """ - try: - return get_pypi_repo(pypi_simple_url).get_package(name, version) - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") + return list(PypiPackage.packages_from_dir(directory=directory)) ################################################################################ @@ -1928,34 +1830,31 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def clear(self): - shutil.rmtree(self.directory) - - def get(self, path_or_url, as_text=True): + def get(self, path_or_url, as_text=True, force=False): """ - Get a file from a `path_or_url` through the cache. - `path_or_url` can be a path or a URL to a file. + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. """ - filename = os.path.basename(path_or_url.strip("/")) - cached = os.path.join(self.directory, filename) + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) - if not os.path.exists(cached): + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") return get_local_file_content(path=cached, as_text=as_text) - def put(self, filename, content): - """ - Put in the cache the `content` of `filename`. - """ - cached = os.path.join(self.directory, filename) - wmode = "wb" if isinstance(content, bytes) else "w" - with open(cached, wmode) as fo: - fo.write(content) + +CACHE = Cache() def get_file_content(path_or_url, as_text=True): @@ -1963,17 +1862,17 @@ def get_file_content(path_or_url, as_text=True): Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if path_or_url.startswith("file://") or ( - path_or_url.startswith("/") and os.path.exists(path_or_url) - ): - return get_local_file_content(path=path_or_url, as_text=as_text) - - elif path_or_url.startswith("https://"): - if TRACE: + if path_or_url.startswith("https://"): + if TRACE_DEEP: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + else: raise Exception(f"Unsupported URL scheme: {path_or_url}") @@ -2016,6 +1915,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header + print(f" DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -2039,102 +1939,22 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def get_url_content_if_modified( - url, - md5, - _delay=0, -): - """ - Return fetched content bytes at `url` or None if the md5 has not changed. - Retries multiple times to fetch if there is a HTTP 429 throttling response - and this with an increasing delay. - """ - time.sleep(_delay) - headers = None - if md5: - etag = f'"{md5}"' - headers = {"If-None-Match": f"{etag}"} - - # using a GET with stream=True ensure we get the the final header from - # several redirects and that we can ignore content there. A HEAD request may - # not get us this last header - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: - status = response.status_code - if status == requests.codes.too_many_requests and _delay < 20: # NOQA - # too many requests: start waiting with some exponential delay - _delay = (_delay * 2) or 1 - return get_url_content_if_modified(url=url, md5=md5, _delay=_delay) - - elif status == requests.codes.not_modified: # NOQA - # all is well, the md5 is the same - return None - - elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") - - return response.content - - -def get_remote_headers(url): - """ - Fetch and return a mapping of HTTP headers of `url`. - """ - headers, _content = get_remote_file_content(url, headers_only=True) - return headers - - -def fetch_and_save_filename_from_paths_or_urls( +def fetch_and_save( + path_or_url, + dest_dir, filename, - paths_or_urls, - dest_dir=THIRDPARTY_DIR, as_text=True, ): """ - Return the content from fetching the `filename` file name found in the - `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an - Exception on errors. Treats the content as text if `as_text` is True - otherwise as binary. + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. """ - path_or_url = get_link_for_filename( - filename=filename, - paths_or_urls=paths_or_urls, - ) - - return fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + content = CACHE.get( path_or_url=path_or_url, as_text=as_text, ) - - -def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()): - """ - Return the content from fetching at path or URL. Raise an Exception on - errors. Treats the content as text if as_text is True otherwise as treat as - binary. Use the provided file cache. This is the main entry for using the - cache. - - Note: the `cache` argument is a global, though it does not really matter - since it does not hold any state which is only kept on disk. - """ - if cache: - return cache.get(path_or_url=path_or_url, as_text=as_text) - else: - return get_file_content(path_or_url=path_or_url, as_text=as_text) - - -def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True): - """ - Return the content from fetching the `filename` file name at URL or path - and save to `dest_dir`. Raise an Exception on errors. Treats the content as - text if as_text is True otherwise as treat as binary. - """ - if through_cache: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text) - else: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) - output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" with open(output, wmode) as fo: @@ -2144,397 +1964,29 @@ def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, th ################################################################################ # -# Sync and fix local thirdparty directory for various issues and gaps +# Functions to update or fetch ABOUT and license files # ################################################################################ -def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): - """ - Given a thirdparty dir, fetch missing source distributions from our remote - repo or PyPI. Return a list of (name, version) tuples for source - distribution that were not found - """ - not_found = [] - local_packages = get_local_packages(directory=dest_dir) - remote_repo = get_remote_repo() - pypi_repo = get_pypi_repo() - - for package in local_packages: - if not package.sdist: - print(f"Finding sources for: {package.name}=={package.version}: ", end="") - try: - pypi_package = pypi_repo.get_package(name=package.name, version=package.version) - - if pypi_package and pypi_package.sdist: - print(f"Fetching sources from Pypi") - pypi_package.fetch_sdist(dest_dir=dest_dir) - continue - else: - remote_package = remote_repo.get_package( - name=package.name, version=package.version - ) - - if remote_package and remote_package.sdist: - print(f"Fetching sources from Remote") - remote_package.fetch_sdist(dest_dir=dest_dir) - continue - - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - print(f"No sources found") - not_found.append( - ( - package.name, - package.version, - ) - ) - - return not_found - - -def fetch_missing_wheels( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, +def clean_about_files( dest_dir=THIRDPARTY_DIR, ): """ - Given a thirdparty dir fetch missing wheels for all known combos of Python - versions and OS. Return a list of tuple (Package, Environment) for wheels - that were not found locally or remotely. + Given a thirdparty dir, clean ABOUT files """ local_packages = get_local_packages(directory=dest_dir) - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - packages_and_envts = itertools.product(local_packages, environments) - - not_fetched = [] - fetched_filenames = set() - for package, envt in packages_and_envts: - - filename = package.fetch_wheel( - environment=envt, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) - - if filename: - fetched_filenames.add(filename) - else: - not_fetched.append( - ( - package, - envt, - ) - ) - - return not_fetched - - -def build_missing_wheels( - packages_and_envts, - build_remotely=False, - with_deps=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, -): - """ - Build all wheels in a list of tuple (Package, Environment) and save in - `dest_dir`. Return a list of tuple (Package, Environment), and a list of - built wheel filenames. - """ - - not_built = [] - built_filenames = [] - - packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) - - for package, pkg_envts in packages_and_envts: - - envts = [envt for _pkg, envt in pkg_envts] - python_versions = sorted(set(e.python_version for e in envts)) - operating_systems = sorted(set(e.operating_system for e in envts)) - built = None - try: - built = build_wheels( - requirements_specifier=package.specifier, - with_deps=with_deps, - build_remotely=build_remotely, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=TRACE, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - print(".") - except Exception as e: - import traceback - - print("#############################################################") - print("############# WHEEL BUILD FAILED ######################") - traceback.print_exc() - print() - print("#############################################################") - - if not built: - for envt in pkg_envts: - not_built.append((package, envt)) - else: - for bfn in built: - print(f" --> Built wheel: {bfn}") - built_filenames.append(bfn) - - return not_built, built_filenames - - -################################################################################ -# -# Functions to handle remote or local repo used to "find-links" -# -################################################################################ - - -def get_paths_or_urls(links_url): - if links_url.startswith("https:"): - paths_or_urls = find_links_from_release_url(links_url) - else: - paths_or_urls = find_links_from_dir(links_url) - return paths_or_urls - - -def find_links_from_dir(directory=THIRDPARTY_DIR): - """ - Return a list of path to files in `directory` for any file that ends with - any of the extension in the list of `extensions` strings. - """ - base = os.path.abspath(directory) - files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] - return files - - -get_links = re.compile('href="([^"]+)"').findall - - -def find_links_from_release_url(links_url=REMOTE_LINKS_URL): - """ - Return a list of download link URLs found in the HTML page at `links_url` - URL that starts with the `prefix` string and ends with any of the extension - in the list of `extensions` strings. Use the `base_url` to prefix the links. - """ - if TRACE: - print(f"Finding links for {links_url}") - - plinks_url = urllib.parse.urlparse(links_url) - - base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - - if TRACE: - print(f"Base URL {base_url}") - - _headers, text = get_remote_file_content(links_url) - links = [] - for link in get_links(text): - if not link.endswith(EXTENSIONS): - continue - - plink = urllib.parse.urlsplit(link) - - if plink.scheme: - # full URL kept as-is - url = link - - if plink.path.startswith("/"): - # absolute link - url = f"{base_url}{link}" - - else: - # relative link - url = f"{links_url}/{link}" - - if TRACE: - print(f"Adding URL: {url}") - - links.append(url) - - if TRACE: - print(f"Found {len(links)} links at {links_url}") - return links - - -def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): - """ - Return a list of download link URLs found in a PyPI simple index for package name. - with the list of `extensions` strings. Use the `simple_url` PyPI url. - """ - if TRACE: - print(f"Finding links for {simple_url}") - - name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip("/") - simple_url = f"{simple_url}/{name}" - - _headers, text = get_remote_file_content(simple_url) - links = get_links(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - links = [l for l in links if l.endswith(EXTENSIONS)] - return links - - -def get_link_for_filename(filename, paths_or_urls): - """ - Return a link for `filename` found in the `links` list of URLs or paths. Raise an - exception if no link is found or if there are more than one link for that - file name. - """ - path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] - if not path_or_url: - raise Exception(f"Missing link to file: {filename}") - if not len(path_or_url) == 1: - raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) - return path_or_url[0] - - -################################################################################ -# -# Requirements processing -# -################################################################################ - - -class MissingRequirementException(Exception): - pass - - -def get_required_packages(required_name_versions): - """ - Return a tuple of (remote packages, PyPI packages) where each is a mapping - of {(name, version): PypiPackage} for packages listed in the - `required_name_versions` list of (name, version) tuples. Raise a - MissingRequirementException with a list of missing (name, version) if a - requirement cannot be satisfied remotely or in PyPI. - """ - remote_repo = get_remote_repo() - - remote_packages = { - (name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions - } - - pypi_repo = get_pypi_repo() - pypi_packages = { - (name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions - } - - # remove any empty package (e.g. that do not exist in some place) - remote_packages = {nv: p for nv, p in remote_packages.items() if p} - pypi_packages = {nv: p for nv, p in pypi_packages.items() if p} - - # check that we are not missing any - repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys()) - missing_name_versions = required_name_versions.difference(repos_name_versions) - if missing_name_versions: - raise MissingRequirementException(sorted(missing_name_versions)) - - return remote_packages, pypi_packages - - -def get_required_remote_packages( - requirements_file="requirements.txt", - force_pinned=True, - remote_links_url=REMOTE_LINKS_URL, -): - """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI-like link repo - ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a - local directory path to a wheels directory if this is not a a URL. - """ - required_name_versions = load_requirements( - requirements_file=requirements_file, - force_pinned=force_pinned, - ) - - if remote_links_url.startswith("https://"): - repo = get_remote_repo(remote_links_url=remote_links_url) - else: - # a local path - assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" - repo = get_local_repo(directory=remote_links_url) - - for name, version in required_name_versions: - if version: - yield name, version, repo.get_package(name, version) - else: - yield name, version, repo.get_latest_version(name) - - -def update_requirements(name, version=None, requirements_file="requirements.txt"): - """ - Upgrade or add `package_name` with `new_version` to the `requirements_file` - requirements file. Write back requirements sorted with name and version - canonicalized. Note: this cannot deal with hashed or unpinned requirements. - Do nothing if the version already exists as pinned. - """ - normalized_name = NameVer.normalize_name(name) - - is_updated = False - updated_name_versions = [] - for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False): - - existing_normalized_name = NameVer.normalize_name(existing_name) - - if normalized_name == existing_normalized_name: - if version != existing_version: - is_updated = True - updated_name_versions.append( - ( - existing_normalized_name, - existing_version, - ) - ) - - if is_updated: - updated_name_versions = sorted(updated_name_versions) - nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - - with open(requirements_file, "w") as fo: - fo.write(nvs) - - -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): - """ - Hash all the requirements found in the `requirements_file` - requirements file based on distributions available in `dest_dir` - """ - local_repo = get_local_repo(directory=dest_dir) - packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version - hashed = [] - for name, version in load_requirements(requirements_file, force_pinned=True): - package = packages_by_normalized_name_version.get((name, version)) - if not package: - raise Exception(f"Missing required package {name}=={version}") - hashed.append(package.specifier_with_hashes) - - with open(requirements_file, "w") as fo: - fo.write("\n".join(hashed)) - + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) -################################################################################ -# -# Functions to update or fetch ABOUT and license files -# -################################################################################ + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) -def add_fetch_or_update_about_and_license_files( - dest_dir=THIRDPARTY_DIR, - include_remote=True, - strip_classifiers=False, -): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2544,36 +1996,33 @@ def add_fetch_or_update_about_and_license_files( - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files - - TODO: make API calls to fetch package data from DejaCode - The process consists in load and iterate on every package distributions, - collect data and then acsk to save. + Use available existing on-disk cached index if use_cached_index is True. """ - local_packages = get_local_packages(directory=dest_dir) - local_repo = get_local_repo(directory=dest_dir) - - remote_repo = get_remote_repo() - def get_other_dists(_package, _dist): """ - Return a list of all the dists from package that are not the `dist` object + Return a list of all the dists from `_package` that are not the `_dist` + object """ return [d for d in _package.get_distributions() if d != _dist] + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + for local_package in local_packages: for local_dist in local_package.get_distributions(): local_dist.load_about_data(dest_dir=dest_dir) local_dist.set_checksums(dest_dir=dest_dir) - if strip_classifiers and "classifiers" in local_dist.extra_data: - local_dist.extra_data.pop("classifiers", None) - local_dist.save_about_and_notice_files(dest_dir) - # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2585,19 +2034,18 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue - # try to get a latest version of the same package that is not our version + # try to get another version of the same package that is not our version other_local_packages = [ p - for p in local_repo.get_versions(local_package.name) + for p in packages_by_name[local_package.name] if p.version != local_package.version ] - - latest_local_version = other_local_packages and other_local_packages[-1] - if latest_local_version: - latest_local_dists = list(latest_local_version.get_distributions()) + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2612,46 +2060,51 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue - if include_remote: - # lets try to fetch remotely - local_dist.load_remote_about_data() + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + + other_remote_packages = [ + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) continue - # try to get a latest version of the same package that is not our version - other_remote_packages = [ - p - for p in remote_repo.get_versions(local_package.name) - if p.version != local_package.version - ] - - latest_version = other_remote_packages and other_remote_packages[-1] - if latest_version: - latest_dists = list(latest_version.get_distributions()) - for remote_dist in latest_dists: - remote_dist.load_remote_about_data() - if not remote_dist.has_key_metadata(): - # there is not much value to get other data if we are missing the key ones - continue - else: - local_dist.update_from_other_dist(remote_dist) - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - break - - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) - continue - # try to get data from pkginfo (no license though) local_dist.load_pkginfo_data(dest_dir=dest_dir) @@ -2659,9 +2112,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files(dest_dir) - - # TODO: try to get data from dejacode + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2680,9 +2131,9 @@ def get_other_dists(_package, _dist): def call(args, verbose=TRACE): """ Call args in a subprocess and display output on the fly if ``trace`` is True. - Return or raise stdout, stderr, returncode + Return a tuple of (returncode, stdout, stderr) """ - if TRACE: + if TRACE_DEEP: print("Calling:", " ".join(args)) with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" @@ -2700,462 +2151,90 @@ def call(args, verbose=TRACE): stdout, stderr = process.communicate() if not stdout.strip(): stdout = "\n".join(stdouts) + return process.returncode, stdout, stderr - returncode = process.returncode - - if returncode == 0: - return returncode, stdout, stderr - else: - raise Exception(returncode, stdout, stderr) - - -def add_or_upgrade_built_wheels( - name, - version=None, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=TRACE, - remote_build_log_file=None, -): - """ - Add or update package `name` and `version` as a binary wheel saved in - `dest_dir`. Use the latest version if `version` is None. Return the a list - of the collected, fetched or built wheel file names or an empty list. - - Use the provided lists of `python_versions` (e.g. "36", "39") and - `operating_systems` (e.g. linux, windows or macos) to decide which specific - wheel to fetch or build. - - Include wheels for all dependencies if `with_deps` is True. - Build remotely is `build_remotely` is True. - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - assert name, "Name is required" - ver = version and f"=={version}" or "" - print(f"\nAdding wheels for package: {name}{ver}") - - if verbose: - print("python_versions:", python_versions) - print("operating_systems:", operating_systems) - - wheel_filenames = [] - # a mapping of {req specifier: {mapping build_wheels kwargs}} - wheels_to_build = {} - for python_version, operating_system in itertools.product(python_versions, operating_systems): - print( - f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" - ) - environment = Environment.from_pyver_and_os(python_version, operating_system) - - # Check if requested wheel already exists locally for this version - local_repo = get_local_repo(directory=dest_dir) - local_package = local_repo.get_package(name=name, version=version) - - has_local_wheel = False - if version and local_package: - for wheel in local_package.get_supported_wheels(environment): - has_local_wheel = True - wheel_filenames.append(wheel.filename) - break - if has_local_wheel: - print(f" local wheel exists: {wheel.filename}") - continue - - if not version: - pypi_package = get_pypi_repo().get_latest_version(name) - version = pypi_package.version - - # Check if requested wheel already exists remotely or in Pypi for this version - wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir - ) - if verbose: - print(" fetching package wheel:", wheel_filename) - if wheel_filename: - wheel_filenames.append(wheel_filename) - - # the wheel is not available locally, remotely or in Pypi - # we need to build binary from sources - requirements_specifier = f"{name}=={version}" - to_build = wheels_to_build.get(requirements_specifier) - if to_build: - to_build["python_versions"].append(python_version) - to_build["operating_systems"].append(operating_system) - else: - wheels_to_build[requirements_specifier] = dict( - requirements_specifier=requirements_specifier, - python_versions=[python_version], - operating_systems=[operating_system], - dest_dir=dest_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - for build_wheels_kwargs in wheels_to_build.values(): - bwheel_filenames = build_wheels(**build_wheels_kwargs) - wheel_filenames.extend(bwheel_filenames) - - return sorted(set(wheel_filenames)) - - -def build_wheels( - requirements_specifier, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=False, - remote_build_log_file=None, -): - """ - Given a pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) for all - `python_versions` and `operating_systems` combinations and save them - back in `dest_dir` and return a list of built wheel file names. - - Include wheels for all dependencies if `with_deps` is True. - First try to build locally to process pure Python wheels, and fall back to - build remotey on all requested Pythons and operating systems. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - all_pure, builds = build_wheels_locally_if_pure_python( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - verbose=verbose, - dest_dir=dest_dir, - ) - for local_build in builds: - print(f"Built wheel: {local_build}") - - if all_pure: - return builds - - if build_remotely: - remote_builds = build_wheels_remotely_on_multiple_platforms( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=verbose, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - builds.extend(remote_builds) - - return builds - - -def build_wheels_remotely_on_multiple_platforms( - requirements_specifier, - with_deps=False, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - verbose=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) including wheels for - all dependencies for all `python_versions` and `operating_systems` - combinations and save them back in `dest_dir` and return a list of built - wheel file names. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - check_romp_is_configured() - pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = "" if with_deps else "--no-deps" - verbose = "--verbose" if verbose else "" - - if remote_build_log_file: - # zero seconds, no wait, log to file instead - wait_build_for = "0" - else: - wait_build_for = DEFAULT_ROMP_BUILD_WAIT - - romp_args = [ - "romp", - "--interpreter", - "cpython", - "--architecture", - "x86_64", - "--check-period", - wait_build_for, # in seconds - ] - - if remote_build_log_file: - romp_args += ["--build-log-file", remote_build_log_file] - - romp_args += pyos_options + [ - "--artifact-paths", - "*.whl", - "--artifact", - "artifacts.tar.gz", - "--command", - f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " - f"python -m pip {verbose} wheel {deps} {requirements_specifier}", - ] - - if verbose: - romp_args.append("--verbose") - - print(f"Building wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(romp_args)) - call(romp_args) - wheel_filenames = [] - if not remote_build_log_file: - wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) - for wfn in wheel_filenames: - print(f" built wheel: {wfn}") - return wheel_filenames - - -def fetch_remotely_built_wheels( - remote_build_log_file, +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, dest_dir=THIRDPARTY_DIR, - no_wait=False, - verbose=False, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, ): """ - Given a ``remote_build_log_file`` file path with a JSON lines log of a - remote build, fetch the built wheels and move them to ``dest_dir``. Return a - list of built wheel file names. - """ - wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT # in seconds - - romp_args = [ - "romp-fetch", - "--build-log-file", - remote_build_log_file, - "--check-period", - wait, + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", ] - if verbose: - romp_args.append("--verbose") - - print(f"Fetching built wheels from log file: {remote_build_log_file}") - print(f"Using command:", " ".join(romp_args)) - - call(romp_args, verbose=verbose) - - wheel_filenames = [] - - for art in os.listdir(os.getcwd()): - if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art): - continue - - print(f" Processing artifact archive: {art}") - wheel_fns = extract_tar(art, dest_dir) - for wfn in wheel_fns: - print(f" Retrieved built wheel: {wfn}") - wheel_filenames.extend(wheel_fns) - return wheel_filenames - - -def get_romp_pyos_options( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, -): - """ - Return a list of CLI options for romp - For example: - >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', - ... '--version', '3.9', '--version', '3.10', '--platform', 'linux', - ... '--platform', 'macos', '--platform', 'windows'] - >>> assert get_romp_pyos_options() == expected - """ - python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))] - pyos_options = list( - itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) - ) - - pyos_options += list( - itertools.chain.from_iterable( - ("--platform", plat) for plat in sorted(set(operating_systems)) - ) - ) - - return pyos_options - - -def check_romp_is_configured(): - # these environment variable must be set before - has_envt = ( - os.environ.get("ROMP_BUILD_REQUEST_URL") - and os.environ.get("ROMP_DEFINITION_ID") - and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") - and os.environ.get("ROMP_USERNAME") - ) - - if not has_envt: - raise Exception( - "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " - "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " - "are required enironment variables." - ) - - -def build_wheels_locally_if_pure_python( - requirements_specifier, - with_deps=False, - verbose=False, - dest_dir=THIRDPARTY_DIR, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) locally. - - If all these are "pure" Python wheels that run on all Python 3 versions and - operating systems, copy them back in `dest_dir` if they do not exists there - - Return a tuple of (True if all wheels are "pure", list of built wheel file names) - """ - deps = [] if with_deps else ["--no-deps"] - verbose = ["--verbose"] if verbose else [] - - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") - cli_args = ( - [ - "pip", - "wheel", - "--wheel-dir", - wheel_dir, - ] - + deps - + verbose - + [requirements_specifier] - ) - - print(f"Building local wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(cli_args)) - call(cli_args) - - built = os.listdir(wheel_dir) - if not built: - return [] - - all_pure = all(is_pure_wheel(bwfn) for bwfn in built) - - if not all_pure: - print(f" Some wheels are not pure") - - print(f" Copying local wheels") - pure_built = [] - for bwfn in built: - owfn = os.path.join(dest_dir, bwfn) - if not os.path.exists(owfn): - nwfn = os.path.join(wheel_dir, bwfn) - fileutils.copyfile(nwfn, owfn) - pure_built.append(bwfn) - print(f" Built local wheel: {bwfn}") - return all_pure, pure_built - - -# TODO: Use me -def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): - """ - Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its - tags for PyPI compatibility and making it smaller if possible. Return the - name of the new wheel if renamed or the existing new name otherwise. - """ - if is_pure_wheel(wheel_filename): - print(f"Pure wheel: {wheel_filename}, nothing to do.") - return wheel_filename - - original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") - awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] - call(awargs) - - audited = os.listdir(wheel_dir) - if not audited: - # cannot optimize wheel - return wheel_filename - - assert len(audited) == 1 - new_wheel_name = audited[0] - - new_wheel_loc = os.path.join(wheel_dir, new_wheel_name) - - # this needs to go now - os.remove(original_wheel_loc) - - if new_wheel_name == wheel_filename: - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel = Wheel.from_filename(new_wheel_name) - non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) - new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] - if not new_wheel.platforms: - print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel_cleaned_filename = new_wheel.to_filename() - new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename) - os.rename(new_wheel_loc, new_wheel_cleaned_loc) - return new_wheel_cleaned_filename + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) -def extract_tar( - location, - dest_dir=THIRDPARTY_DIR, -): - """ - Extract a tar archive at `location` in the `dest_dir` directory. Return a - list of extracted locations (either directories or files). - """ - with open(location, "rb") as fi: - with tarfile.open(fileobj=fi) as tar: - members = list(tar.getmembers()) - tar.extractall(dest_dir, members=members) + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) - return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") -def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): - """ - Fetch the binary wheel for package `name` and `version` and save in - `dest_dir`. Use the provided `environment` Environment to determine which - specific wheel to fetch. + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error - Return the fetched wheel file name on success or None if it was not fetched. - Trying fetching from our own remote repo, then from PyPI. - """ - wheel_filename = None - remote_package = get_remote_package(name=name, version=version) - if TRACE: - print(" remote_package:", remote_package) - if remote_package: - wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - if wheel_filename: - return wheel_filename - pypi_package = get_pypi_package(name=name, version=version) - if TRACE: - print(" pypi_package:", pypi_package) - if pypi_package: - wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - return wheel_filename +################################################################################ +# +# Functions to check for problems +# +################################################################################ def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -3195,6 +2274,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ if not declared_licenses: return try: diff --git a/requirements-dev.txt b/requirements-dev.txt index d8a9e806..97fe5b5f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,23 @@ -atomicwrites==1.4.0 +bleach==4.1.0 +build==0.7.0 +commonmark==0.9.1 +docutils==0.18.1 +et-xmlfile==1.1.0 execnet==1.9.0 iniconfig==1.1.1 -mock==4.0.3 -packaging==21.0 -pluggy==0.13.1 -py==1.10.0 -pyparsing==2.4.7 -pytest==6.2.5 -pytest-forked==1.3.0 -pytest-xdist==2.3.0 -toml==0.10.2 \ No newline at end of file +jeepney==0.7.1 +keyring==23.4.1 +openpyxl==3.0.9 +pep517==0.12.0 +pkginfo==1.8.2 +py==1.11.0 +pytest==7.0.1 +pytest-forked==1.4.0 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +rfc3986==1.5.0 +rich==12.3.0 +secretstorage==3.3.2 +tomli==1.2.3 +twine==3.8.0 diff --git a/requirements.txt b/requirements.txt index 5fbffae1..050ca43e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,80 @@ -attrs==21.2.0 +attrs==21.4.0 +banal==1.0.6 +beautifulsoup4==4.11.1 +binaryornot==0.4.4 boolean.py==3.8 -certifi==2021.5.30 -click==8.0.1 +certifi==2021.10.8 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.12 +click==8.0.4 colorama==0.4.4 -importlib-metadata==4.8.1 -Jinja2==3.0.1 +commoncode==30.2.0 +construct==2.10.68 +container-inspector==31.0.0 +cryptography==36.0.2 +debian-inspector==30.0.0 +dockerfile-parse==1.2.0 +dparse2==0.6.1 +extractcode==31.0.0 +extractcode-7z==16.5.210531 +extractcode-libarchive==3.5.1.210531 +fasteners==0.17.3 +fingerprints==1.0.3 +ftfy==6.0.3 +future==0.18.2 +gemfileparser==0.8.0 +html5lib==1.1 +idna==3.3 +importlib-metadata==4.8.3 +inflection==0.5.1 +intbitset==3.0.1 +isodate==0.6.1 +jaraco.functools==3.4.0 +javaproperties==0.8.1 +Jinja2==3.0.3 +jsonstreams==0.6.0 license-expression==21.6.14 +lxml==4.8.0 MarkupSafe==2.0.1 -openpyxl==3.0.9 -packageurl-python==0.9.4 -pip==21.2.4 +more-itertools==8.13.0 +normality==2.3.3 +packagedcode-msitools==0.101.210706 +packageurl-python==0.9.9 +packaging==21.3 +parameter-expansion-patched==0.3.1 +patch==1.16 +pdfminer-six==20220506 +pefile==2021.9.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 +pluggy==1.0.0 +plugincode==30.0.0 +ply==3.11 +publicsuffix2==2.20191221 +pyahocorasick==2.0.0b1 +pycparser==2.21 +pygmars==0.7.0 +Pygments==2.12.0 +pymaven-patch==0.3.0 +pyparsing==3.0.8 +pytz==2022.1 PyYAML==6.0 +rdflib==5.0.0 +regipy==2.3.1 +requests==2.27.1 +rpm-inspector-rpm==4.16.1.3.210404 saneyaml==0.5.2 -setuptools==58.1.0 -typing-extensions==3.10.0.2 -wheel==0.37.0 -zipp==3.5.0 +six==1.16.0 +soupsieve==2.3.1 +spdx-tools==0.7.0a3 +text-unidecode==1.3 +toml==0.10.2 +typecode==30.0.0 +typecode-libmagic==5.39.210531 +urllib3==1.26.9 +urlpy==0.5 +wcwidth==0.2.5 +webencodings==0.5.1 +xmltodict==0.12.0 +zipp==3.6.0 diff --git a/setup.cfg b/setup.cfg index 0421405f..474703a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,50 +1,65 @@ [metadata] -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst name = aboutcode-toolkit -author = nexB. Inc. and others -author_email = info@aboutcode.org license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = AboutCode-toolkit is a tool to document the provenance (origin and license) of third-party software using small text files. Collect inventories and generate attribution documentation. long_description = file:README.rst +long_description_content_type = text/x-rst url = https://github.com/nexB/aboutcode-toolkit + +author = nexB. Inc. and others +author_email = info@aboutcode.org + classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Software Development - Topic :: Software Development :: Documentation - Topic :: Software Development :: Quality Assurance - Topic :: System :: Software Distribution + Topic :: Software Development :: Documentation + Topic :: Software Development :: Quality Assurance + Topic :: System :: Software Distribution Topic :: Utilities + keywords = - license - about - metadata - package - copyright - attribution - software - inventory + license + about + metadata + package + copyright + attribution + software + inventory + open source + sca + SBOM + spdx + +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst [options] -package_dir= +package_dir = =src -packages=find: +packages = find: include_package_data = true zip_safe = false + +setup_requires = setuptools_scm[toml] >= 4 + +python_requires = >=3.6.* + install_requires = attrs - boolean.py >= 3.5, < 4.0 + boolean.py >= 3.5 certifi click jinja2 @@ -53,67 +68,23 @@ install_requires = packageurl_python >= 0.9.0 saneyaml -setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.6.*, <4 [options.packages.find] -where=src +where = src + [options.extras_require] testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - mock - colorama - py black -docs= - Sphinx>=3.3.1 - sphinx-rtd-theme>=0.5.0 - doc8>=0.8.1 + twine + +docs = + Sphinx >= 3.3.1 + sphinx-rtd-theme >= 0.5.0 + doc8 >= 0.8.1 [options.entry_points] console_scripts = about=attributecode.cmd:about - -[tool:pytest] -norecursedirs = - .git - .cache - .settings - bin - dist - dist - build - _build - docs - man - share - example - etc - Include - include - Lib - lib - local - Scripts - thirdparty - tmp - tests/testdata/* - tests/ignore - tests/Ignore - tests/ignore/ - -python_files = *.py - -python_classes=Test -python_functions=test - -addopts = - -rfEsxXw - --strict-markers - -s - -vv - --ignore docs/conf.py - --ignore setup.py - --doctest-modules diff --git a/tests/test_api.py b/tests/test_api.py index fcfbab56..d3efd071 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import unittest -import mock +import unittest +from unittest import mock from attributecode import api from attributecode import ERROR diff --git a/tests/test_model.py b/tests/test_model.py index 56cf1d67..0226e24a 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -20,8 +20,8 @@ import posixpath import shutil import unittest +from unittest import mock -import mock import saneyaml from attributecode import CRITICAL diff --git a/thirdparty/README.rst b/thirdparty/README.rst deleted file mode 100644 index b31482f8..00000000 --- a/thirdparty/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python dependency wheels to be vendored in this directory. -