diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml new file mode 100644 index 00000000000..88dc3ac0fc4 --- /dev/null +++ b/.github/workflows/scancode-release.yml @@ -0,0 +1,388 @@ +name: Create ScanCode release archives, then test and publish to GH and PyPI + +# This is executed automatically on a tag + +# Summary of the steps: +# - build wheel and sdist for main and same for mini flavor +# - then upload all wheels and sdist to PyPI +# - build release app archives, one for each of linux, windows, macos on Python 3.8 +# - then create gh-release and upload app archives to release +# TODO: for each of linux, windows, macos: smoke test wheel, sdist and release archives +# TODO: add changelog to release text body + + +on: + workflow_dispatch: + push: + tags: + - "v*.*.*" + +jobs: + + build_scancode_for_pypi: + name: Build ScanCode PyPI archives + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + + - name: Build main wheel + run: | + venv/bin/python setup.py --quiet bdist_wheel + venv/bin/twine check dist/* + + - name: Collect built main wheel + uses: actions/upload-artifact@v3 + with: + name: main_wheel + path: dist/* + + - name: Build main sdist + run: | + rm -rf dist + venv/bin/python setup.py --quiet sdist + venv/bin/twine check dist/* + + - name: Collect built main sdist + uses: actions/upload-artifact@v3 + with: + name: main_sdist + path: dist/* + + + build_scancode_for_pypi_mini: + name: Build ScanCode PyPI archives mini + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build mini wheel + run: | + rm -rf build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + cp setup-mini.cfg setup.cfg + venv/bin/python setup.py --quiet bdist_wheel + venv/bin/twine check dist/* + + - name: Collect built mini wheel + uses: actions/upload-artifact@v3 + with: + name: mini_wheel + path: dist/* + + - name: Build mini sdist + run: | + rm -rf build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + cp setup-mini.cfg setup.cfg + venv/bin/python setup.py --quiet sdist + venv/bin/twine check dist/* + + - name: Collect built mini sdist + uses: actions/upload-artifact@v3 + with: + name: mini_sdist + path: dist/* + + + build_scancode_for_release_linux: + name: Build ScanCode Release archives for linux + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build linux app archive + run: | + source venv/bin/activate + formats=xztar + operating_system=linux + python_dot_version=3.8 + python_version=38 + echo -n "python$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built linux app + uses: actions/upload-artifact@v3 + with: + name: linux_app + path: dist/* + + + build_scancode_for_release_mac: + name: Build ScanCode Release archives for mac + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + + - name: Build mac app archive + run: | + source venv/bin/activate + formats=xztar + operating_system=macos + python_dot_version=3.8 + python_version=38 + echo -n "python$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built mac app + uses: actions/upload-artifact@v3 + with: + name: macos_app + path: dist/* + + + build_scancode_for_release_win: + name: Build ScanCode Release archives for windows + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build windows app archive + run: | + source venv/bin/activate + formats=zip + operating_system=windows + python_dot_version=3.8 + python_version=38 + echo -n "py -$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built windows app + uses: actions/upload-artifact@v3 + with: + name: windows_app + path: dist/* + + build_scancode_for_release_source: + name: Build ScanCode Release archives source + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + + - name: Build source archive with deps + run: | + source venv/bin/activate + formats=xztar + rm -rf thirdparty build dist src/licensedcode/data/cache + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --sdists + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _sources + + - name: Collect built source app tarball + uses: actions/upload-artifact@v3 + with: + name: source_app + path: dist/* + + + publish_to_pypi: + name: Publish to PyPI + needs: [build_scancode_for_pypi, build_scancode_for_pypi_mini] + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + dist_names: [main_wheel, main_sdist, mini_wheel, mini_sdist] + + steps: + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Download a single artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.dist_names }} + path: dist + + - name: Publish distributions to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + + publish_to_gh_release: + name: Publish to GH Release + needs: + - build_scancode_for_release_linux + - build_scancode_for_release_win + - build_scancode_for_release_mac + - build_scancode_for_release_source + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - name: Download a single artifact source_app + uses: actions/download-artifact@v3 + with: + name: source_app + path: dist + + - name: Download a single artifact macos_app + uses: actions/download-artifact@v3 + with: + name: macos_app + path: dist + + - name: Download a single artifact linux_app + uses: actions/download-artifact@v3 + with: + name: linux_app + path: dist + + - name: Download a single artifact windows_app + uses: actions/download-artifact@v3 + with: + name: windows_app + path: dist + + - name: Create release and publish archives + uses: softprops/action-gh-release@v1 + with: + draft: true + files: dist/* diff --git a/MANIFEST.in b/MANIFEST.in index 623a8575f6a..9334b861a6b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ graft src graft thirdparty -recursive-include etc configure.py graft etc/thirdparty recursive-include etc/thirdparty * @@ -14,7 +13,7 @@ include *.rst include setup.* include configure* include requirements* -include .git* +include .giti* include extractcode* include scancode* diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh index e1c6d072e8a..a99f13a7e81 100755 --- a/etc/release/scancode-create-release.sh +++ b/etc/release/scancode-create-release.sh @@ -339,7 +339,7 @@ function build_source_archive { # would take a distutils manifest-like input $BIN_DIR/python setup.py $QUIET sdist --formats=xztar - $BIN_DIR/python etc/release/scancode_rename_archives.py dist/ $src _sources + $BIN_DIR/python etc/release/scancode_rename_archives.py dist/ _sources mkdir -p release/archives mv dist/* release/archives/ } diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4c409693911..7a18f230137 100755 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -312,7 +312,7 @@ def download_sdist( raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") -@functools.cache +@functools.lru_cache(maxsize=None) def get_package_versions( name, version=None, @@ -2364,7 +2364,7 @@ def build_wheels_locally_if_pure_python( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") diff --git a/requirements-dev.txt b/requirements-dev.txt index c7487662ff8..cc54c189f30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,4 @@ pytest==7.0.1 pytest-forked==1.4.0 pytest-xdist==2.5.0 tomli==1.2.3 +twine \ No newline at end of file diff --git a/setup-mini.cfg b/setup-mini.cfg index 1e6edac91ce..7b8b9b2bad0 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -128,6 +128,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine docs = Sphinx >= 3.3.1 diff --git a/setup.cfg b/setup.cfg index cfae4518a6e..ddb9d172a2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine docs = Sphinx >= 3.3.1 diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index 2d32d3b08e0..931c5468ed6 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -167,7 +167,6 @@ def create_package_and_deps(codebase, **kwargs): handler = get_package_handler(package_data) if TRACE: logger_debug(' create_package_and_deps: handler:', handler) - click.echo(f' create_package_and_deps: handler: {handler}') items = handler.assemble( package_data=package_data, diff --git a/src/scancode_config.py b/src/scancode_config.py index 753dd9c85d2..e09947d2400 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -77,7 +77,7 @@ def _create_dir(location): # in case package is not installed or we do not have setutools/pkg_resources # on hand fall back to this version -__version__ = '31.0.0b3' +__version__ = '31.0.0b4' # used to warn user when the version is out of date __release_date__ = datetime.datetime(2022, 4, 30)