From 2bb2038a9d80081a44a87f59fe3a006ad498fac7 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 23 Sep 2021 18:44:17 +0200 Subject: [PATCH 1/6] Use tag file to restart release tests We now create a tag file after each succesful test of a release archive and this mean we can restart from the last successful tests and not restest everything. Azure and/or the network are rather flaky lately so this way we can run the full release matrix even if it means restarting the test command several times. Signed-off-by: Philippe Ombredanne --- etc/release/scancode-create-release.sh | 119 ++++++++++++++----------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh index 3f2be031bfc..03242ac6dd1 100755 --- a/etc/release/scancode-create-release.sh +++ b/etc/release/scancode-create-release.sh @@ -65,34 +65,42 @@ function run_app_smoke_tests { echo " " echo "### Testing app with Python $python_app_dot_version on OS: $operating_system" - archive_to_test=$(ls -1 -R release/archives/ | grep "$python_app_version-$operating_system") - echo "#### Testing $archive_to_test with Python $python_app_dot_version on OS: $operating_system" + # check if the tag file exist meaning we have already completed the tests + tag_file="scancode-release-tested-for-python-$python_app_dot_version-$operating_system" + if [ ! -f $tag_file ]; then + archive_to_test=$(ls -1 -R release/archives/ | grep "$python_app_version-$operating_system") - # Check checksum of archive and script since it transits through file.io - sha_arch=$(sha256sum release/archives/$archive_to_test | awk '{ print $1 }') - sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }') + echo "#### Testing $archive_to_test with Python $python_app_dot_version on OS: $operating_system" - echo "#### Creating a temp archive that contains the tested archive: $archive_file and the test script" - archive_file=input.tar.gz - tar -czf $archive_file \ - -C release/archives $archive_to_test \ - -C ../../etc/release scancode_release_tests.py + # Check checksum of archive and script since it transits through file.io + sha_arch=$(sha256sum release/archives/$archive_to_test | awk '{ print $1 }') + sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }') - tar -tvf $archive_file + echo "#### Creating a temp archive that contains the tested archive: $archive_file and the test script" + archive_file=input.tar.gz + tar -czf $archive_file \ + -C release/archives $archive_to_test \ + -C ../../etc/release scancode_release_tests.py - echo "#### Remote test command: python scancode_release_tests.py app $archive_to_test sha_arch:$sha_arch sha_py:$sha_py" + tar -tvf $archive_file - romp \ - --interpreter cpython \ - --architecture x86_64 \ - --check-period 5 \ - --version $python_app_dot_version \ - --platform $operating_system \ - --archive-file $archive_file \ - --command "python scancode_release_tests.py app $archive_to_test $sha_arch $sha_py" + echo "#### Remote test command: python scancode_release_tests.py app $archive_to_test sha_arch:$sha_arch sha_py:$sha_py" - echo "#### RELEASE TEST: Completed App tests of $archive_to_test with Python $python_app_dot_version on OS: $operating_system" + romp \ + --interpreter cpython \ + --architecture x86_64 \ + --check-period 5 \ + --version $python_app_dot_version \ + --platform $operating_system \ + --archive-file $archive_file \ + --command "python scancode_release_tests.py app $archive_to_test $sha_arch $sha_py" + + echo "#### RELEASE TEST: Completed App tests of $archive_to_test with Python $python_app_dot_version on OS: $operating_system" + touch $tag_file + else + echo "#### RELEASE TEST: NOT RE-TESTING $archive_to_test with Python $python_app_dot_version on OS: $operating_system" + fi } @@ -109,46 +117,55 @@ function run_pypi_smoke_tests { python_dot_versions=$2 operating_systems=$3 + # check if the tag file exist meaning we have already completed the tests + tag_file="scancode-release-pypi-tested-for-$archive_to_test" + echo " " echo "### Testing $archive_to_test with Pythons: $python_dot_versions on OSses: $operating_systems" - # Check checksum of archive and script since it transits through file.io - sha_arch=$(sha256sum release/pypi/$archive_to_test | awk '{ print $1 }') - sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }') + if [ ! -f $tag_file ]; then - echo "#### Creating a temp archive that contains the tested archive: $archive_file and the test script" - archive_file=input.tar.gz - tar -czf $archive_file \ - -C release/pypi $archive_to_test \ - -C ../../etc/release scancode_release_tests.py + # Check checksum of archive and script since it transits through file.io + sha_arch=$(sha256sum release/pypi/$archive_to_test | awk '{ print $1 }') + sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }') - tar -tvf $archive_file + echo "#### Creating a temp archive that contains the tested archive: $archive_to_test and the test script: release scancode_release_tests.py" + archive_file=input.tar.gz + tar -czf $archive_file \ + -C release/pypi $archive_to_test \ + -C ../../etc/release scancode_release_tests.py - echo "#### Remote test command: python scancode_release_tests.py pypi archive_to_test:$archive_to_test sha_arch:$sha_arch sha_py:$sha_py" + tar -tvf $archive_file - # build options for Python versions and OS - ver_opts=" " - for pdv in $python_dot_versions - do - ver_opts="$ver_opts --version $pdv" - done + echo "#### Remote test command: python scancode_release_tests.py pypi archive_to_test:$archive_to_test sha_arch:$sha_arch sha_py:$sha_py" - os_opts=" " - for os in $operating_systems - do - os_opts="$os_opts --platform $os" - done + # build options for Python versions and OS + ver_opts=" " + for pdv in $python_dot_versions + do + ver_opts="$ver_opts --version $pdv" + done - romp \ - --interpreter cpython \ - --architecture x86_64 \ - --check-period 5 \ - $ver_opts \ - $os_opts \ - --archive-file $archive_file \ - --command "python scancode_release_tests.py pypi $archive_to_test $sha_arch $sha_py" + os_opts=" " + for os in $operating_systems + do + os_opts="$os_opts --platform $os" + done - echo "#### RELEASE TEST: Completed PyPI tests of $archive_to_test with Pythons: $python_dot_versions on OSses: $operating_systems" + romp \ + --interpreter cpython \ + --architecture x86_64 \ + --check-period 5 \ + $ver_opts \ + $os_opts \ + --archive-file $archive_file \ + --command "python scancode_release_tests.py pypi $archive_to_test $sha_arch $sha_py" + + echo "#### RELEASE TEST: Completed PyPI tests of $archive_to_test with Pythons: $python_dot_versions on OSses: $operating_systems" + touch $tag_file + else + echo "#### RELEASE TEST: NOT RE-TESTING Pypi $archive_to_test with Python $python_app_dot_version on OSses: $operating_systems" + fi } From 6606f64e606a206946a075debb21fe15fc24c2fc Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 23 Sep 2021 21:46:31 +0200 Subject: [PATCH 2/6] Bump release for bug emergency bug fix Referenced-by: https://github.com/nexB/scancode-toolkit/issues/2713 Reported-by: Thomas Druez Signed-off-by: Philippe Ombredanne --- setup-mini.cfg | 2 +- setup.cfg | 2 +- src/scancode_config.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-mini.cfg b/setup-mini.cfg index d868ddda0c7..a7a38191cf7 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit-mini -version = 30.0.0 +version = 30.0.1 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts. scancode-toolkit-mini is a special build that does not come with pre-built binary dependencies by default. These are instead installed separately or with the extra_requires scancode-toolkit-mini[full] diff --git a/setup.cfg b/setup.cfg index 75695bb1f38..a79793f1fc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit -version = 30.0.0 +version = 30.0.1 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft description = ScanCode is a tool to scan code for license, copyright, package and their documented dependencies and other interesting facts. diff --git a/src/scancode_config.py b/src/scancode_config.py index 46b0b608a6c..73ff23e5936 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -77,10 +77,10 @@ def _create_dir(location): # in case package is not installed or we do not have setutools/pkg_resources # on hand fall back to this version -__version__ = '30.0.0' +__version__ = '30.0.1' # used to warn user when the version is out of date -__release_date__ = datetime.datetime(2021, 9, 23) +__release_date__ = datetime.datetime(2021, 9, 24) # See https://github.com/nexB/scancode-toolkit/issues/2653 for more information # on the data format version From 2b487b7374d3d43f09e07ab94f4672be017e1ffa Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 24 Sep 2021 10:51:32 +0200 Subject: [PATCH 3/6] Require at least commoncode 30.0.0 This is a fix for https://github.com/nexB/scancode-toolkit/issues/2713 Reference: https://github.com/nexB/commoncode/issues/31 Reference: https://github.com/nexB/scancode-toolkit/issues/2713 Reported-by: Thomas Druez Signed-off-by: Philippe Ombredanne --- requirements.txt | 2 +- setup-mini.cfg | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index c7406220f07..301a0209989 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ cffi==1.14.5 chardet==4.0.0 click==8.0.1 colorama==0.4.4 -commoncode==21.8.31 +commoncode==30.0.0 construct==2.10.67 cryptography==3.4.7 debian-inspector==30.0.0 diff --git a/setup-mini.cfg b/setup-mini.cfg index a7a38191cf7..21346e5e910 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -59,7 +59,7 @@ install_requires = chardet >= 3.0.0 click >= 6.7, !=7.0 colorama >= 0.3.9 - commoncode >= 21.8.31 + commoncode >= 30.0.0 debian-inspector >= 30.0.0 dparse >= 0.5.1 fasteners diff --git a/setup.cfg b/setup.cfg index a79793f1fc6..5b613aefa60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,7 @@ install_requires = chardet >= 3.0.0 click >= 6.7, !=7.0 colorama >= 0.3.9 - commoncode >= 21.8.31 + commoncode >= 30.0.0 debian-inspector >= 30.0.0 dparse >= 0.5.1 fasteners From 73f5c6f864c7a6b073583a99037db4c551b20b32 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 24 Sep 2021 11:00:56 +0200 Subject: [PATCH 4/6] Add tests for many Click versions #2713 Following a commoncode update this adds tests for running scancode on all supported Click versions. Reference: https://github.com/nexB/commoncode/issues/31 Reference: https://github.com/nexB/scancode-toolkit/issues/2713 Reported-by: Thomas Druez Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a32c038327a..f5815726d31 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -90,6 +90,7 @@ jobs: bin/pytest -n 3 -vvs --test-suite=all \ tests/licensedcode/test_zzzz_cache.py + - template: etc/ci/azure-posix.yml parameters: job_name: mac1014 @@ -159,6 +160,22 @@ jobs: all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py +################################################################################ +# Test using many version of Click to work around regressions in API +################################################################################ + + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu20_test_all_supported_click_versions + image_name: ubuntu-20.04 + python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_architecture: x64 + test_suites: + click_versions: for clk_ver in 8.0.1 7.1.2 7.1.1 7.1 6.7; do pip install click==$clk_ver; scancode -i samples/ -n3 --json foo.json; scancode -i --verbose samples/ -n3 --json foo.json; done + + + ################################################################################ # These jobs are using containers and their own Python 3.6 ################################################################################ From 8a1d7eccf23058179803bb7c08a152cc9cf6dcd3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 24 Sep 2021 11:11:40 +0200 Subject: [PATCH 5/6] Add CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c689443dabe..2f2d41dbefb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -64,6 +64,22 @@ Outputs: +30.0.1 - 2021-09-24 +-------------------- + +This is a minor bug fix release for these bugs: + +- https://github.com/nexB/scancode-toolkit/issues/2713 +- https://github.com/nexB/scancode-toolkit/issues/2713 + +We now correctly work with all supported Click versions. + +Thank you to: +- Konstantin Kochin @vznncv +- Thomas Druez @tdruez + + + 30.0.0 - 2021-09-23 -------------------- @@ -204,6 +220,7 @@ particular: - Sarita Singh @itssingh - Siddhant Khare @Siddhant-K-code - Soim Kim @soimkim +- Thomas Druez @tdruez - Thorsten Godau @tgodau - Yunus Rahbar @yns88 From c53e81298eecc1cdf0da23b3d57962a85c323afb Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 24 Sep 2021 12:01:28 +0200 Subject: [PATCH 6/6] Use correct path in Click version tests #2713 Reference: https://github.com/nexB/commoncode/issues/31 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f5815726d31..389b4363f4f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -172,7 +172,13 @@ jobs: python_versions: ['3.6', '3.7', '3.8', '3.9'] python_architecture: x64 test_suites: - click_versions: for clk_ver in 8.0.1 7.1.2 7.1.1 7.1 6.7; do pip install click==$clk_ver; scancode -i samples/ -n3 --json foo.json; scancode -i --verbose samples/ -n3 --json foo.json; done + click_versions: | + for clk_ver in 8.0.1 7.1.2 7.1.1 7.1 6.7; + do + bin/pip install click==$clk_ver; + bin/scancode -i samples/ -n3 --json foo.json; + bin/scancode -i --verbose samples/ -n3 --json foo.json; + done