From c49681175f645ddb47f8012e801f0156ff5edf31 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 09:05:48 +0200 Subject: [PATCH 1/9] Remove debug message prints Reference: https://github.com/nexB/scancode-toolkit/issues/2937 Signed-off-by: Philippe Ombredanne --- src/packagedcode/plugin_package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index 2d32d3b08e0..931c5468ed6 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -167,7 +167,6 @@ def create_package_and_deps(codebase, **kwargs): handler = get_package_handler(package_data) if TRACE: logger_debug(' create_package_and_deps: handler:', handler) - click.echo(f' create_package_and_deps: handler: {handler}') items = handler.assemble( package_data=package_data, From e5526beea4321ec573e74eedd7d4e74224c75a7e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 09:25:50 +0200 Subject: [PATCH 2/9] Add twine to dev deps. Signed-off-by: Philippe Ombredanne --- setup-mini.cfg | 1 + setup.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/setup-mini.cfg b/setup-mini.cfg index 1e6edac91ce..7b8b9b2bad0 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -128,6 +128,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine docs = Sphinx >= 3.3.1 diff --git a/setup.cfg b/setup.cfg index cfae4518a6e..ddb9d172a2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine docs = Sphinx >= 3.3.1 From 549ef533635e2c7a156d494e0154ee8f7dbdb18b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 09:31:21 +0200 Subject: [PATCH 3/9] Bump version Signed-off-by: Philippe Ombredanne --- setup-mini.cfg | 2 +- setup.cfg | 2 +- src/scancode_config.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup-mini.cfg b/setup-mini.cfg index 7b8b9b2bad0..28dbbe9b652 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit-mini -version = 31.0.0b3 +version = 31.0.0b4 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/setup.cfg b/setup.cfg index ddb9d172a2b..2b2468054b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit -version = 31.0.0b3 +version = 31.0.0b4 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/src/scancode_config.py b/src/scancode_config.py index 753dd9c85d2..e09947d2400 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -77,7 +77,7 @@ def _create_dir(location): # in case package is not installed or we do not have setutools/pkg_resources # on hand fall back to this version -__version__ = '31.0.0b3' +__version__ = '31.0.0b4' # used to warn user when the version is out of date __release_date__ = datetime.datetime(2022, 4, 30) From 193aceeb2b590a1933c04d56cba14abce00591dd Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 3 May 2022 19:04:59 +0200 Subject: [PATCH 4/9] Add twine to development deps Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 1 + setup-mini.cfg | 1 + setup.cfg | 1 + 3 files changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index c7487662ff8..bb48ecef8b0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,4 @@ pytest==7.0.1 pytest-forked==1.4.0 pytest-xdist==2.5.0 tomli==1.2.3 +twine==4.0.0 \ No newline at end of file diff --git a/setup-mini.cfg b/setup-mini.cfg index 1e6edac91ce..c61f079c95f 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -128,6 +128,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine >= 4.0.0 docs = Sphinx >= 3.3.1 diff --git a/setup.cfg b/setup.cfg index cfae4518a6e..35d81017b2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,7 @@ testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 6.0.0 + twine >= 4.0.0 docs = Sphinx >= 3.3.1 From 422cc6efa92ed368bc0045f4c58f9594e4f83636 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 3 May 2022 22:43:17 +0200 Subject: [PATCH 5/9] Cleanup MANIFEST.in Remove old, missing files. Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 623a8575f6a..9334b861a6b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,6 @@ graft src graft thirdparty -recursive-include etc configure.py graft etc/thirdparty recursive-include etc/thirdparty * @@ -14,7 +13,7 @@ include *.rst include setup.* include configure* include requirements* -include .git* +include .giti* include extractcode* include scancode* From df739536deabbcb568c60b218cb813671a644f28 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 3 May 2022 22:44:15 +0200 Subject: [PATCH 6/9] Remove reference to non-existing variable Signed-off-by: Philippe Ombredanne --- etc/release/scancode-create-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh index e1c6d072e8a..a99f13a7e81 100755 --- a/etc/release/scancode-create-release.sh +++ b/etc/release/scancode-create-release.sh @@ -339,7 +339,7 @@ function build_source_archive { # would take a distutils manifest-like input $BIN_DIR/python setup.py $QUIET sdist --formats=xztar - $BIN_DIR/python etc/release/scancode_rename_archives.py dist/ $src _sources + $BIN_DIR/python etc/release/scancode_rename_archives.py dist/ _sources mkdir -p release/archives mv dist/* release/archives/ } From d57b91becebbd915bcd9f746d61858099812cce4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 4 May 2022 01:18:58 +0200 Subject: [PATCH 7/9] Work with Python 3.6+ functools.cache is only in Python 3.9 and is exactly functools.lru_cache(maxsize=None) Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4c409693911..7a18f230137 100755 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -312,7 +312,7 @@ def download_sdist( raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") -@functools.cache +@functools.lru_cache(maxsize=None) def get_package_versions( name, version=None, @@ -2364,7 +2364,7 @@ def build_wheels_locally_if_pure_python( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") From 83138d662f99edad85c0e72cbca281e6b3106f14 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 4 May 2022 01:20:42 +0200 Subject: [PATCH 8/9] Add new GH Action to automate release This workflow builds app and PyPI archives and then upload to PyPI and creates a GH release Signed-off-by: Philippe Ombredanne --- .github/workflows/scancode-release.yml | 388 +++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 .github/workflows/scancode-release.yml diff --git a/.github/workflows/scancode-release.yml b/.github/workflows/scancode-release.yml new file mode 100644 index 00000000000..88dc3ac0fc4 --- /dev/null +++ b/.github/workflows/scancode-release.yml @@ -0,0 +1,388 @@ +name: Create ScanCode release archives, then test and publish to GH and PyPI + +# This is executed automatically on a tag + +# Summary of the steps: +# - build wheel and sdist for main and same for mini flavor +# - then upload all wheels and sdist to PyPI +# - build release app archives, one for each of linux, windows, macos on Python 3.8 +# - then create gh-release and upload app archives to release +# TODO: for each of linux, windows, macos: smoke test wheel, sdist and release archives +# TODO: add changelog to release text body + + +on: + workflow_dispatch: + push: + tags: + - "v*.*.*" + +jobs: + + build_scancode_for_pypi: + name: Build ScanCode PyPI archives + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + + - name: Build main wheel + run: | + venv/bin/python setup.py --quiet bdist_wheel + venv/bin/twine check dist/* + + - name: Collect built main wheel + uses: actions/upload-artifact@v3 + with: + name: main_wheel + path: dist/* + + - name: Build main sdist + run: | + rm -rf dist + venv/bin/python setup.py --quiet sdist + venv/bin/twine check dist/* + + - name: Collect built main sdist + uses: actions/upload-artifact@v3 + with: + name: main_sdist + path: dist/* + + + build_scancode_for_pypi_mini: + name: Build ScanCode PyPI archives mini + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build mini wheel + run: | + rm -rf build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + cp setup-mini.cfg setup.cfg + venv/bin/python setup.py --quiet bdist_wheel + venv/bin/twine check dist/* + + - name: Collect built mini wheel + uses: actions/upload-artifact@v3 + with: + name: mini_wheel + path: dist/* + + - name: Build mini sdist + run: | + rm -rf build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + cp setup-mini.cfg setup.cfg + venv/bin/python setup.py --quiet sdist + venv/bin/twine check dist/* + + - name: Collect built mini sdist + uses: actions/upload-artifact@v3 + with: + name: mini_sdist + path: dist/* + + + build_scancode_for_release_linux: + name: Build ScanCode Release archives for linux + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build linux app archive + run: | + source venv/bin/activate + formats=xztar + operating_system=linux + python_dot_version=3.8 + python_version=38 + echo -n "python$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built linux app + uses: actions/upload-artifact@v3 + with: + name: linux_app + path: dist/* + + + build_scancode_for_release_mac: + name: Build ScanCode Release archives for mac + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + + - name: Build mac app archive + run: | + source venv/bin/activate + formats=xztar + operating_system=macos + python_dot_version=3.8 + python_version=38 + echo -n "python$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built mac app + uses: actions/upload-artifact@v3 + with: + name: macos_app + path: dist/* + + + build_scancode_for_release_win: + name: Build ScanCode Release archives for windows + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + ./scancode --reindex-licenses + + - name: Build windows app archive + run: | + source venv/bin/activate + formats=zip + operating_system=windows + python_dot_version=3.8 + python_version=38 + echo -n "py -$python_dot_version" > PYTHON_EXECUTABLE + rm -rf thirdparty build dist + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --python-version=$python_version \ + --operating-system=$operating_system \ + --wheels + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system + + - name: Collect built windows app + uses: actions/upload-artifact@v3 + with: + name: windows_app + path: dist/* + + build_scancode_for_release_source: + name: Build ScanCode Release archives source + runs-on: ubuntu-20.04 + + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install requirements and prepare index + run: | + ./configure --dev + + - name: Build source archive with deps + run: | + source venv/bin/activate + formats=xztar + rm -rf thirdparty build dist src/licensedcode/data/cache + rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info + mkdir -p thirdparty + venv/bin/python etc/scripts/fetch_thirdparty.py \ + --requirements=requirements.txt \ + --dest=thirdparty \ + --sdists + venv/bin/python setup.py --quiet sdist --formats=$formats + venv/bin/python etc/release/scancode_rename_archives.py dist/ _sources + + - name: Collect built source app tarball + uses: actions/upload-artifact@v3 + with: + name: source_app + path: dist/* + + + publish_to_pypi: + name: Publish to PyPI + needs: [build_scancode_for_pypi, build_scancode_for_pypi_mini] + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + dist_names: [main_wheel, main_sdist, mini_wheel, mini_sdist] + + steps: + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Download a single artifact + uses: actions/download-artifact@v3 + with: + name: ${{ matrix.dist_names }} + path: dist + + - name: Publish distributions to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + + publish_to_gh_release: + name: Publish to GH Release + needs: + - build_scancode_for_release_linux + - build_scancode_for_release_win + - build_scancode_for_release_mac + - build_scancode_for_release_source + runs-on: ubuntu-20.04 + defaults: + run: + shell: bash + strategy: + fail-fast: false + + steps: + - name: Download a single artifact source_app + uses: actions/download-artifact@v3 + with: + name: source_app + path: dist + + - name: Download a single artifact macos_app + uses: actions/download-artifact@v3 + with: + name: macos_app + path: dist + + - name: Download a single artifact linux_app + uses: actions/download-artifact@v3 + with: + name: linux_app + path: dist + + - name: Download a single artifact windows_app + uses: actions/download-artifact@v3 + with: + name: windows_app + path: dist + + - name: Create release and publish archives + uses: softprops/action-gh-release@v1 + with: + draft: true + files: dist/* From ae19a912f747fcf918ed35c011507342342c3669 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 4 May 2022 01:59:36 +0200 Subject: [PATCH 9/9] Do not pin twine Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index bb48ecef8b0..cc54c189f30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,4 +8,4 @@ pytest==7.0.1 pytest-forked==1.4.0 pytest-xdist==2.5.0 tomli==1.2.3 -twine==4.0.0 \ No newline at end of file +twine \ No newline at end of file