diff --git a/configure b/configure index 05dcbd419ca..120627280e5 100755 --- a/configure +++ b/configure @@ -67,12 +67,17 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # scancode-specific: Thirdparty package locations and index handling -# Do we have thirdparty/files? use the mit.LICENSE as a proxy -if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then - LINKS=$CFG_ROOT_DIR/thirdparty +if [[ "$PYPI_LINKS" == "" ]]; then + # Do we have thirdparty/files? use the mit.LICENSE as a proxy + if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then + LINKS=$CFG_ROOT_DIR/thirdparty + else + LINKS=https://thirdparty.aboutcode.org/pypi + fi else - LINKS=https://thirdparty.aboutcode.org/pypi + LINKS=$PYPI_LINKS fi + PIP_EXTRA_ARGS="--no-index --find-links $LINKS" ################################ diff --git a/etc/release/fetch_requirements.py b/etc/release/fetch_requirements.py index 6cb73ed0b2c..dfd202a7f53 100755 --- a/etc/release/fetch_requirements.py +++ b/etc/release/fetch_requirements.py @@ -8,10 +8,11 @@ # See https://github.com/nexB/scancode-toolkit for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import itertools + import click import utils_thirdparty -import itertools @click.command() @@ -63,6 +64,15 @@ is_flag=True, help='Fetch only the corresponding source distributions.', ) +@click.option('-u', '--remote-links-url', + type=str, + metavar='URL', + default=utils_thirdparty.REMOTE_LINKS_URL, + show_default=True, + help='URL to a PyPI-like links web site. ' + 'Or local path to a directory with wheels.', +) + @click.help_option('-h', '--help') def fetch_requirements( requirements_file, @@ -73,6 +83,7 @@ def fetch_requirements( with_about, allow_unpinned, only_sources, + remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, ): """ Fetch and save to THIRDPARTY_DIR all the required wheels for pinned @@ -82,7 +93,9 @@ def fetch_requirements( Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together with a virtualenv.pyz app. - Use exclusively our remote repository (and not PyPI). + Use exclusively wheel not from PyPI but rather found in the PyPI-like link + repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` + as a local directory path to a wheels directory if this is not a a URL. """ # fetch wheels @@ -93,23 +106,28 @@ def fetch_requirements( if not only_sources: envs = itertools.product(python_versions, operating_systems) envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) + for env, reqf in itertools.product(envs, requirements_files): + for package, error in utils_thirdparty.fetch_wheels( environment=env, requirements_file=reqf, allow_unpinned=allow_unpinned, dest_dir=thirdparty_dir, + remote_links_url=remote_links_url, ): if error: print('Failed to fetch wheel:', package, ':', error) # optionally fetch sources if with_sources or only_sources: + for reqf in requirements_files: for package, error in utils_thirdparty.fetch_sources( requirements_file=reqf, allow_unpinned=allow_unpinned, dest_dir=thirdparty_dir, + remote_links_url=remote_links_url, ): if error: print('Failed to fetch source:', package, ':', error) diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh index 6cf8302df01..1960d4ffe00 100755 --- a/etc/release/scancode-create-release.sh +++ b/etc/release/scancode-create-release.sh @@ -8,6 +8,11 @@ # Create, test and publish release archives, wheels and sdists. # Use the --test to also run basic smoke tests of the built archives # +# To use a local checkout of https://github.com/nexB/thirdparty-packages/ rather +# than https://thirdparty.aboutcode.org/ set the variable PYPI_LINKS to point +# to your thirdparty-packages/pypi local directory (this speeds up the release +# creation and allow to work mostly offline. +# ################################################################################ # Supported current app Python version and OS @@ -202,51 +207,36 @@ function clean_build { clean_egg_info } -backup_previous_release -clean_build -mkdir release - -echo "## RELEASE: Setup environment" - -echo "## RELEASE: Clean and configure, then regen license index" -./configure --clean -./configure -source bin/activate -scancode --reindex-licenses - - - -echo "## RELEASE: Install release requirements" -# We do not need a full env for releasing -bin/pip install $QUIET -r etc/release/requirements.txt - - ################################ # PyPI wheels and sdist: these are not Python version- or OS-dependent ################################ -echo " " -echo "## RELEASE: Building a wheel and a source distribution" -clean_egg_info -bin/python setup.py $QUIET sdist bdist_wheel +function build_wheels { + # Build scancode wheels and dist for PyPI + # Arguments: -mv dist release/pypi + echo " " + echo "## RELEASE: Building a wheel and a source distribution" + clean_egg_info + bin/python setup.py $QUIET sdist bdist_wheel + mv dist release/pypi -echo " " -echo "## RELEASE: Building a mini wheel and a source distribution" -clean_egg_info -mv setup.cfg setup-full.cfg -cp setup-mini.cfg setup.cfg -rm -rf build -bin/python setup.py $QUIET sdist bdist_wheel -mv setup-full.cfg setup.cfg + echo " " + echo "## RELEASE: Building a mini wheel and a source distribution" + clean_egg_info + mv setup.cfg setup-full.cfg + cp setup-mini.cfg setup.cfg + rm -rf build + bin/python setup.py $QUIET sdist bdist_wheel + mv setup-full.cfg setup.cfg -cp dist/* release/pypi/ + cp dist/* release/pypi/ -clean_egg_info -echo "## RELEASE: full and mini, wheel and source distribution(s) built and ready for PyPI upload" -find release -ls + clean_egg_info + echo "## RELEASE: full and mini, wheel and source distribution(s) built and ready for PyPI upload" + find release -ls +} ################################ # Build OSes and Pythons-specific release archives @@ -268,34 +258,44 @@ function build_app_archive { echo " " echo "## RELEASE: Building archive for Python $python_app_version on operating system: $operating_system" - clean_build - mkdir -p thirdparty - if [ "$operating_system" == "windows" ]; then # create a zip only on Windows formats=zip - echo -n "py -$python_app_dot_version">PYTHON_EXECUTABLE + formats_ext=.zip else formats=xztar - echo -n "python$python_app_dot_version">PYTHON_EXECUTABLE + formats_ext=.tar.xz fi - # 1. Collect thirdparty deps only for the subset for this Python/operating_system - bin/python etc/release/fetch_requirements.py \ - --requirements-file=requirements.txt \ - --thirdparty-dir=thirdparty \ - --python-version=$python_app_version \ - --operating-system=$operating_system \ - --with-about + if [ ! -f release/archives/scancode-toolkit-*_py$python_app_version-$operating_system$formats_ext ]; then + clean_build + mkdir -p thirdparty + + if [ "$operating_system" == "windows" ]; then + echo -n "py -$python_app_dot_version">PYTHON_EXECUTABLE + else + echo -n "python$python_app_dot_version">PYTHON_EXECUTABLE + fi + + # 1. Collect thirdparty deps only for the subset for this Python/operating_system + bin/python etc/release/fetch_requirements.py \ + --requirements-file=requirements.txt \ + --thirdparty-dir=thirdparty \ + --python-version=$python_app_version \ + --operating-system=$operating_system \ + --with-about \ + --remote-links-url=$PYPI_LINKS + + # 2. Create tarball or zip. + # For now as a shortcut we use the Python setup.py sdist to create a tarball. + # This is hackish and we should instead use our own archiving code that + # would take a distutils manifest-like input + bin/python setup.py $QUIET sdist --formats=$formats + bin/python etc/release/scancode_rename_archives.py dist/ _py$python_app_version-$operating_system + mkdir -p release/archives + mv dist/* release/archives/ + fi - # 2. Create tarball or zip. - # For now as a shortcut we use the Python setup.py sdist to create a tarball. - # This is hackish and we should instead use our own archiving code that - # would take a distutils manifest-like input - bin/python setup.py $QUIET sdist --formats=$formats - bin/python etc/release/scancode_rename_archives.py dist/ _py$python_app_version-$operating_system - mkdir -p release/archives - mv dist/* release/archives/ } @@ -314,7 +314,8 @@ function build_source_archive { --requirements-file=requirements.txt \ --thirdparty-dir=thirdparty \ --with-about \ - --only-sources + --only-sources \ + --remote-links-url=$PYPI_LINKS # 2. Create tarball # For now as a shortcut we use the Python setup.py sdist to create a tarball. @@ -328,6 +329,31 @@ function build_source_archive { } +if [ "$CLI_ARGS" != "--continue" ]; then + echo "############# Reset release creation #############################" + backup_previous_release + clean_build + mkdir release + + echo "## RELEASE: Setup environment" + + echo "## RELEASE: Clean and configure, then regen license index" + ./configure --clean + PYPI_LINKS=$PYPI_LINKS ./configure --local + source bin/activate + scancode --reindex-licenses + + echo "## RELEASE: Install release requirements" + # We do not need a full env for releasing + bin/pip install $QUIET -r etc/release/requirements.txt +else + echo "############# Continuing previous release creation #############################" +fi + + +# wheels +build_wheels + # build the app combos on the current App Python for operating_system in $OPERATING_SYSTEMS do @@ -366,6 +392,5 @@ echo "### RELEASE is ready for publishing ###" # ping on chat and twitter # send email - set +e set +x diff --git a/etc/release/utils_thirdparty.py b/etc/release/utils_thirdparty.py index 549ea919dd0..360f07a6dde 100755 --- a/etc/release/utils_thirdparty.py +++ b/etc/release/utils_thirdparty.py @@ -146,20 +146,24 @@ def fetch_wheels( requirements_file='requirements.txt', allow_unpinned=False, dest_dir=THIRDPARTY_DIR, + remote_links_url=REMOTE_LINKS_URL, ): """ - Download all of the wheel of packages listed in the `requirements_file` - requirements file into `dest_dir` directory. + Download all of the wheel of packages listed in the ``requirements_file`` + requirements file into ``dest_dir`` directory. - Only get wheels for the `environment` Enviromnent constraints. If the - provided `environment` is None then the current Python interpreter + Only get wheels for the ``environment`` Enviromnent constraints. If the + provided ``environment`` is None then the current Python interpreter environment is used implicitly. - Only accept pinned requirements (e.g. with a version) unless allow_unpinned - is True. + Only accept pinned requirements (e.g. with a version) unless + ``allow_unpinned`` is True. - Use direct downloads from our remote repo exclusively. - Yield tuples of (PypiPackage, error) where is None on success + Use exclusively direct downloads from a remote repo at URL + ``remote_links_url``. If ``remote_links_url`` is a path, use this as a + directory of links instead of a URL. + + Yield tuples of (PypiPackage, error) where is None on success. """ missed = [] @@ -171,6 +175,7 @@ def fetch_wheels( rrp = list(get_required_remote_packages( requirements_file=requirements_file, force_pinned=force_pinned, + remote_links_url=remote_links_url, )) fetched_filenames = set() @@ -182,7 +187,11 @@ def fetch_wheels( else: fetched_filename = package.fetch_wheel( - environment=environment, fetched_filenames=fetched_filenames, dest_dir=dest_dir) + environment=environment, + fetched_filenames=fetched_filenames, + dest_dir=dest_dir, + ) + if fetched_filename: fetched_filenames.add(fetched_filename) error = None @@ -208,16 +217,20 @@ def fetch_sources( requirements_file='requirements.txt', allow_unpinned=False, dest_dir=THIRDPARTY_DIR, + remote_links_url=REMOTE_LINKS_URL, ): """ - Download all of the dependent package sources listed in the `requirement` - requirements file into `dest_dir` directory. + Download all of the dependent package sources listed in the + ``requirements_file`` requirements file into ``dest_dir`` destination + directory. - Use direct downloads to achieve this (not pip download). Use only the - packages found in our remote repo. + Use direct downloads to achieve this (not pip download). Use exclusively the + packages found from a remote repo at URL ``remote_links_url``. If + ``remote_links_url`` is a path, use this as a directory of links instead of + a URL. - Only accept pinned requirements (e.g. with a version) unless allow_unpinned - is True. + Only accept pinned requirements (e.g. with a version) unless + ``allow_unpinned`` is True. Yield tuples of (PypiPackage, error message) for each package where error message will empty on success. @@ -232,6 +245,7 @@ def fetch_sources( rrp = list(get_required_remote_packages( requirements_file=requirements_file, force_pinned=force_pinned, + remote_links_url=remote_links_url, )) for name, version, package in rrp: @@ -1297,7 +1311,7 @@ def packages_from_one_path_or_url(cls, path_or_url): @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ - Yield PypiPackages built from a list of of paths or URLs. + Yield PypiPackages built from a list of paths or URLs. """ dists = cls.get_dists(paths_or_urls) dists = NameVer.sorted(dists) @@ -1366,19 +1380,30 @@ def get_name_version(cls, name, version, packages): raise Exception(f'More than one PypiPackage with {name}=={version}') - def fetch_wheel(self, environment=None, fetched_filenames=None, dest_dir=THIRDPARTY_DIR): + def fetch_wheel( + self, + environment=None, + fetched_filenames=None, + dest_dir=THIRDPARTY_DIR, + ): """ - Download a binary wheel of this package matching the `environment` - Enviromnent constraints into `dest_dir` directory. + Download a binary wheel of this package matching the ``environment`` + Enviromnent constraints into ``dest_dir`` directory. Return the wheel filename if it was fetched, None otherwise. - If the provided `environment` is None then the current Python - interpreter environment is used implicitly. + If the provided ``environment`` is None then the current Python + interpreter environment is used implicitly. Do not refetch wheel if + their name is in a provided ``fetched_filenames`` set. """ fetched_wheel_filename = None - fetched_filenames = fetched_filenames if fetched_filenames is not None else set() + if fetched_filenames is not None: + fetched_filenames = fetched_filenames + else: + fetched_filenames = set() + for wheel in self.get_supported_wheels(environment): + if wheel.filename not in fetched_filenames: fetch_and_save_path_or_url( filename=wheel.filename, @@ -2272,20 +2297,34 @@ def get_required_packages(required_name_versions): return remote_packages, pypi_packages -def get_required_remote_packages(requirements_file='requirements.txt', force_pinned=True): +def get_required_remote_packages( + requirements_file='requirements.txt', + force_pinned=True, + remote_links_url=REMOTE_LINKS_URL, +): """ Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in our remote repo exclusively. + `requirements_file` requirements file and found in the PyPI-like link repo + ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a + local directory path to a wheels directory if this is not a a URL. """ required_name_versions = load_requirements( - requirements_file=requirements_file, force_pinned=force_pinned) + requirements_file=requirements_file, + force_pinned=force_pinned, + ) + + if remote_links_url.startswith('https://'): + repo = get_remote_repo(remote_links_url=remote_links_url) + else: + # a local path + assert os.path.exists(remote_links_url) + repo = get_local_repo(directory=remote_links_url) - remote_repo = get_remote_repo() for name, version in required_name_versions: if version: - yield name, version, remote_repo.get_package(name, version) + yield name, version, repo.get_package(name, version) else: - yield name, version, remote_repo.get_latest_version(name) + yield name, version, repo.get_latest_version(name) def update_requirements(name, version=None, requirements_file='requirements.txt'):