Skip to content

Commit 8e264cb

Browse files
pombredannepombredanne
committed
Try to run fewer CI jobs, more efficiently
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
1 parent 37bc3f2 commit 8e264cb

3 files changed

Lines changed: 86 additions & 92 deletions

File tree

‎azure-pipelines.yml‎

Lines changed: 57 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,16 @@ jobs:
1717
image_name: ubuntu-20.04
1818
python_versions: ['3.8']
1919
test_suites:
20-
misc: |
21-
venv/bin/pytest -n 3 -vvs --test-suite=all \
22-
--ignore=tests/scancode \
23-
--ignore=tests/cluecode \
24-
--ignore=tests/licensedcode
25-
26-
scancode: |
20+
misc_and_scancode: |
2721
# cli tests are launched below on all OSes
28-
venv/bin/pytest -vvs --test-suite=all \
29-
--ignore=tests/scancode/test_cli.py \
30-
tests/scancode
31-
32-
cluecode: |
3322
venv/bin/pytest -n 3 -vvs --test-suite=all \
34-
tests/cluecode
23+
--ignore=tests/cluecode \
24+
--ignore=tests/licensedcode \
25+
--ignore=tests/scancode/test_cli.py
3526
36-
license_base: |
27+
cluecode_license_base: |
3728
venv/bin/pytest -n 3 -vvs --test-suite=all \
29+
tests/cluecode \
3830
--ignore=tests/licensedcode/test_zzzz_cache.py \
3931
--ignore=tests/licensedcode/test_detection_datadriven_external.py \
4032
--ignore=tests/licensedcode/test_detection_datadriven1.py \
@@ -43,20 +35,14 @@ jobs:
4335
--ignore=tests/licensedcode/test_detection_datadriven4.py \
4436
tests/licensedcode
4537
46-
license_datadriven1: |
47-
venv/bin/pytest -n 3 -vvs --test-suite=all \
48-
tests/licensedcode/test_detection_datadriven1.py
49-
50-
license_datadriven2: |
38+
license_datadriven1_2: |
5139
venv/bin/pytest -n 3 -vvs --test-suite=all \
40+
tests/licensedcode/test_detection_datadriven1.py \
5241
tests/licensedcode/test_detection_datadriven2.py
5342
54-
license_datadriven3: |
55-
venv/bin/pytest -n 3 -vvs --test-suite=all \
56-
tests/licensedcode/test_detection_datadriven3.py
57-
58-
license_datadriven4: |
43+
license_datadriven3_4: |
5944
venv/bin/pytest -n 3 -vvs --test-suite=all \
45+
tests/licensedcode/test_detection_datadriven3.py \
6046
tests/licensedcode/test_detection_datadriven4.py
6147
6248
license_datadriven_ext: |
@@ -290,28 +276,6 @@ jobs:
290276
# test_suite: venv/bin/pytest -n 2 -vvs
291277

292278

293-
########################################################################
294-
# RELEASE on 3.6. Also check that we can pip install
295-
########################################################################
296-
#
297-
# - job: Build_release_archive_py3
298-
# pool:
299-
# vmImage: ubuntu-18.04
300-
# steps:
301-
# - checkout: self
302-
# fetchDepth: 10
303-
#
304-
# - task: UsePythonVersion@0
305-
# inputs:
306-
# versionSpec: '3.6'
307-
# displayName: 'Install Python 3.6'
308-
#
309-
# - script: ./etc/release/scancode-test-pip-install.sh
310-
# displayName: 'Test pip wheel installation'
311-
#
312-
# - script: ./etc/release/scancode-test-pip-install-editable.sh
313-
# displayName: 'Test pip editable installation'
314-
315279
################################################################################
316280
# Tests using a plain pip install to get the latest of all wheels
317281
################################################################################
@@ -323,44 +287,77 @@ jobs:
323287
image_name: ubuntu-18.04
324288
python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
325289
test_suites:
326-
all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
290+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
327291

328292
- template: etc/ci/azure-posix.yml
329293
parameters:
330294
job_name: ubuntu20_cpython_latest_from_pip
331295
image_name: ubuntu-20.04
332296
python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
333297
test_suites:
334-
all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
298+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
335299

336300
- template: etc/ci/azure-posix.yml
337301
parameters:
338-
job_name: macos1015_cpython_latest_from_pip
302+
job_name: macos1015_cpython_latest_from_pip1
339303
image_name: macos-10.15
340-
python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
304+
python_versions: ['3.6', '3.7']
305+
test_suites:
306+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
307+
308+
- template: etc/ci/azure-posix.yml
309+
parameters:
310+
job_name: macos1015_cpython_latest_from_pip2
311+
image_name: macos-10.15
312+
python_versions: ['3.8', '3.9', '3.10']
341313
test_suites:
342-
all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
314+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
343315

344316
- template: etc/ci/azure-posix.yml
345317
parameters:
346-
job_name: macos11_cpython_latest_from_pip
318+
job_name: macos11_cpython_latest_from_pip1
347319
image_name: macos-11
348-
python_versions: ['3.7', '3.8', '3.9', '3.10']
320+
python_versions: ['3.7', '3.8', ]
321+
test_suites:
322+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
323+
324+
- template: etc/ci/azure-posix.yml
325+
parameters:
326+
job_name: macos11_cpython_latest_from_pip2
327+
image_name: macos-11
328+
python_versions: ['3.9', '3.10']
349329
test_suites:
350-
all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
330+
all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
351331

352332
- template: etc/ci/azure-win.yml
353333
parameters:
354-
job_name: win2019_cpython_latest_from_pip
334+
job_name: win2019_cpython_latest_from_pip1
355335
image_name: windows-2019
356-
python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10']
336+
python_versions: ['3.6', '3.7']
357337
test_suites:
358-
all: venv\Scripts\pip install --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
338+
all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
359339

360340
- template: etc/ci/azure-win.yml
361341
parameters:
362-
job_name: win2022_cpython_latest_from_pip
342+
job_name: win2019_cpython_latest_from_pip2
343+
image_name: windows-2019
344+
python_versions: ['3.8', '3.9', '3.10']
345+
test_suites:
346+
all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
347+
348+
- template: etc/ci/azure-win.yml
349+
parameters:
350+
job_name: win2022_cpython_latest_from_pip1
363351
image_name: windows-2022
364-
python_versions: ['3.7', '3.8', '3.9', '3.10']
352+
python_versions: ['3.7', '3.8']
365353
test_suites:
366-
all: venv\Scripts\pip install --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
354+
all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
355+
356+
- template: etc/ci/azure-win.yml
357+
parameters:
358+
job_name: win2022_cpython_latest_from_pip2
359+
image_name: windows-2022
360+
python_versions: ['3.9', '3.10']
361+
test_suites:
362+
all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
363+

‎etc/ci/azure-posix.yml‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,30 +13,27 @@ jobs:
1313

1414
strategy:
1515
matrix:
16-
${{ each pyver in parameters.python_versions }}:
1716
${{ each tsuite in parameters.test_suites }}:
18-
${{ format('py{0} {1}', pyver, tsuite.key) }}:
19-
python_version: ${{ pyver }}
17+
${{ tsuite.key }}:
2018
test_suite_label: ${{ tsuite.key }}
2119
test_suite: ${{ tsuite.value }}
2220

2321
steps:
2422
- checkout: self
2523
fetchDepth: 10
2624

27-
- task: UsePythonVersion@0
28-
inputs:
29-
versionSpec: '$(python_version)'
30-
architecture: '${{ parameters.python_architecture }}'
31-
displayName: 'Install Python $(python_version)'
25+
- ${{ each pyver in parameters.python_versions }}:
26+
- task: UsePythonVersion@0
27+
inputs:
28+
versionSpec: '${{ pyver }}'
29+
architecture: '${{ parameters.python_architecture }}'
30+
displayName: '${{ pyver }} - Install Python'
3231

33-
- script: |
34-
python --version
35-
python3 --version
36-
python$(python_version) --version
37-
echo "python$(python_version)" > PYTHON_EXECUTABLE
38-
./configure --dev
39-
displayName: 'Run Configure'
32+
- script: |
33+
python${{ pyver }} --version
34+
echo "python${{ pyver }}" > PYTHON_EXECUTABLE
35+
./configure --clean && configure --dev
36+
displayName: '${{ pyver }} - Configure'
4037
41-
- script: $(test_suite)
42-
displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}'
38+
- script: $(test_suite)
39+
displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}'

‎etc/ci/azure-win.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,27 @@ jobs:
1313

1414
strategy:
1515
matrix:
16-
${{ each pyver in parameters.python_versions }}:
1716
${{ each tsuite in parameters.test_suites }}:
18-
${{ format('py{0} {1}', pyver, tsuite.key) }}:
19-
python_version: ${{ pyver }}
17+
${{ tsuite.key }}:
2018
test_suite_label: ${{ tsuite.key }}
2119
test_suite: ${{ tsuite.value }}
20+
2221
steps:
2322
- checkout: self
2423
fetchDepth: 10
2524

26-
- task: UsePythonVersion@0
27-
inputs:
28-
versionSpec: '$(python_version)'
29-
architecture: '${{ parameters.python_architecture }}'
30-
displayName: 'Install Python $(python_version)'
25+
- ${{ each pyver in parameters.python_versions }}:
26+
- task: UsePythonVersion@0
27+
inputs:
28+
versionSpec: '${{ pyver }}'
29+
architecture: '${{ parameters.python_architecture }}'
30+
displayName: '${{ pyver }} - Install Python'
3131

32-
- script: |
33-
python --version
34-
echo | set /p=python> PYTHON_EXECUTABLE
35-
configure --dev
36-
displayName: 'Run Configure'
32+
- script: |
33+
python --version
34+
echo | set /p=python> PYTHON_EXECUTABLE
35+
configure --clean && configure --dev
36+
displayName: '${{ pyver }} - Configure'
3737
38-
- script: $(test_suite)
39-
displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}'
38+
- script: $(test_suite)
39+
displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}'

0 commit comments

Comments
 (0)