Skip to content

Commit f91c7f7

Browse files
pombredannepombredanne
committed
Include source for native builds
* in GH actiona nd release script, we now also include the source code of key packages with native code. This should help improve compatibility * also build archives for Python 3.8, not 3.9 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
1 parent d1616b8 commit f91c7f7

3 files changed

Lines changed: 27 additions & 150 deletions

File tree

‎.github/workflows/scancode-release.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,10 @@ jobs:
159159
--python-version=$python_version \
160160
--operating-system=$operating_system \
161161
--wheels
162+
venv/bin/python etc/scripts/fetch_thirdparty.py \
163+
--requirements=requirements-native.txt \
164+
--dest=thirdparty \
165+
--sdists
162166
venv/bin/python setup.py --quiet sdist --formats=$formats
163167
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
164168
@@ -210,6 +214,10 @@ jobs:
210214
--python-version=$python_version \
211215
--operating-system=$operating_system \
212216
--wheels
217+
venv/bin/python etc/scripts/fetch_thirdparty.py \
218+
--requirements=requirements-native.txt \
219+
--dest=thirdparty \
220+
--sdists
213221
venv/bin/python setup.py --quiet sdist --formats=$formats
214222
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
215223
@@ -260,6 +268,10 @@ jobs:
260268
--python-version=$python_version \
261269
--operating-system=$operating_system \
262270
--wheels
271+
venv/bin/python etc/scripts/fetch_thirdparty.py \
272+
--requirements=requirements-native.txt \
273+
--dest=thirdparty \
274+
--sdists
263275
venv/bin/python setup.py --quiet sdist --formats=$formats
264276
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
265277

‎etc/release/scancode-create-release.sh‎

Lines changed: 9 additions & 150 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717

1818
# Supported current app Python version and OS
1919
# one archive or installer is built for each python x OS combo
20-
PYTHON_APP_DOT_VERSIONS="3.9"
20+
PYTHON_APP_DOT_VERSIONS="3.8"
2121

22-
PYTHON_PYPI_TESTS_DOT_VERSIONS="3.9"
22+
PYTHON_PYPI_TESTS_DOT_VERSIONS="3.8"
2323

2424
PYPI_LINKS=https://thirdparty.aboutcode.org/pypi
2525

@@ -46,153 +46,6 @@ CLI_ARGS=$1
4646

4747
BIN_DIR=venv/bin
4848

49-
################################
50-
# Run tests and exist with --test
51-
################################
52-
53-
function run_app_smoke_tests {
54-
# Call romp with an archive to run the smoke tests on the selected
55-
# operating system remotely using Azure
56-
#
57-
# Arguments:
58-
# operating_system: One of windows, linux or macos
59-
# python_app_dot_version: the python version to test (no dot)
60-
61-
operating_system=$1
62-
python_app_dot_version=$2
63-
# remove dots
64-
python_app_version=${python_app_dot_version/./}
65-
66-
echo " "
67-
echo "### Testing app with Python $python_app_dot_version on OS: $operating_system"
68-
69-
# check if the tag file exist meaning we have already completed the tests
70-
tag_file="scancode-release-tested-for-python-$python_app_dot_version-$operating_system"
71-
if [ ! -f $tag_file ]; then
72-
archive_to_test=$(ls -1 -R release/archives/ | grep "$python_app_version-$operating_system")
73-
74-
echo "#### Testing $archive_to_test with Python $python_app_dot_version on OS: $operating_system"
75-
76-
# Check checksum of archive and script since it transits through file.io
77-
sha_arch=$(sha256sum release/archives/$archive_to_test | awk '{ print $1 }')
78-
sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }')
79-
80-
echo "#### Creating a temp archive that contains the tested archive: $archive_file and the test script"
81-
archive_file=input.tar.gz
82-
tar -czf $archive_file \
83-
-C release/archives $archive_to_test \
84-
-C ../../etc/release scancode_release_tests.py
85-
86-
tar -tvf $archive_file
87-
88-
echo "#### Remote test command: python scancode_release_tests.py app $archive_to_test sha_arch:$sha_arch sha_py:$sha_py"
89-
90-
romp \
91-
--interpreter cpython \
92-
--architecture x86_64 \
93-
--check-period 5 \
94-
--version $python_app_dot_version \
95-
--platform $operating_system \
96-
--archive-file $archive_file \
97-
--command "python scancode_release_tests.py app $archive_to_test $sha_arch $sha_py"
98-
99-
echo "#### RELEASE TEST: Completed App tests of $archive_to_test with Python $python_app_dot_version on OS: $operating_system"
100-
touch $tag_file
101-
else
102-
echo "#### RELEASE TEST: NOT RE-TESTING $archive_to_test with Python $python_app_dot_version on OS: $operating_system"
103-
fi
104-
}
105-
106-
107-
function run_pypi_smoke_tests {
108-
# Call romp with a PyPI archive to pip install and run the smoke tests on
109-
# the selected Python and operating system remotely using Azure
110-
#
111-
# Arguments:
112-
# archive_to_test: The naem of a wheel file or tar.gz sdist file
113-
# python_dot_versions: run with these Python version as in "3.6 3.7"
114-
# operating_systems: run on these operating_systems as in "windows linux macos"
115-
116-
archive_to_test=$1
117-
python_dot_versions=$2
118-
operating_systems=$3
119-
120-
# check if the tag file exist meaning we have already completed the tests
121-
tag_file="scancode-release-pypi-tested-for-$archive_to_test"
122-
123-
echo " "
124-
echo "### Testing $archive_to_test with Pythons: $python_dot_versions on OSses: $operating_systems"
125-
126-
if [ ! -f $tag_file ]; then
127-
128-
# Check checksum of archive and script since it transits through file.io
129-
sha_arch=$(sha256sum release/pypi/$archive_to_test | awk '{ print $1 }')
130-
sha_py=$(sha256sum etc/release/scancode_release_tests.py | awk '{ print $1 }')
131-
132-
echo "#### Creating a temp archive that contains the tested archive: $archive_to_test and the test script: release scancode_release_tests.py"
133-
archive_file=input.tar.gz
134-
tar -czf $archive_file \
135-
-C release/pypi $archive_to_test \
136-
-C ../../etc/release scancode_release_tests.py
137-
138-
tar -tvf $archive_file
139-
140-
echo "#### Remote test command: python scancode_release_tests.py pypi archive_to_test:$archive_to_test sha_arch:$sha_arch sha_py:$sha_py"
141-
142-
# build options for Python versions and OS
143-
ver_opts=" "
144-
for pdv in $python_dot_versions
145-
do
146-
ver_opts="$ver_opts --version $pdv"
147-
done
148-
149-
os_opts=" "
150-
for os in $operating_systems
151-
do
152-
os_opts="$os_opts --platform $os"
153-
done
154-
155-
romp \
156-
--interpreter cpython \
157-
--architecture x86_64 \
158-
--check-period 5 \
159-
$ver_opts \
160-
$os_opts \
161-
--archive-file $archive_file \
162-
--command "python scancode_release_tests.py pypi $archive_to_test $sha_arch $sha_py"
163-
164-
echo "#### RELEASE TEST: Completed PyPI tests of $archive_to_test with Pythons: $python_dot_versions on OSses: $operating_systems"
165-
touch $tag_file
166-
else
167-
echo "#### RELEASE TEST: NOT RE-TESTING Pypi $archive_to_test with Python $python_app_dot_version on OSses: $operating_systems"
168-
fi
169-
170-
}
171-
172-
173-
if [ "$CLI_ARGS" == "--test" ]; then
174-
echo "##########################################################################"
175-
echo "### TESTING build for Python: $PYTHON_APP_DOT_VERSIONS on OS: $OPERATING_SYSTEMS"
176-
for operating_system in $OPERATING_SYSTEMS
177-
do
178-
for pyver in $PYTHON_APP_DOT_VERSIONS
179-
do
180-
run_app_smoke_tests $operating_system $pyver
181-
done
182-
done
183-
184-
echo "##########################################################################"
185-
echo "### TESTING PyPI archives for Python: $PYTHON_PYPI_TESTS_DOT_VERSIONS on OS: $OPERATING_SYSTEMS"
186-
for archive in $(find release/pypi/ -type f -printf "%f\n")
187-
do
188-
run_pypi_smoke_tests $archive "$PYTHON_PYPI_TESTS_DOT_VERSIONS" "$OPERATING_SYSTEMS"
189-
done
190-
set +e
191-
set +x
192-
exit
193-
fi
194-
195-
19649
echo "##########################################################################"
19750
echo "### BUILDING App for Python: $PYTHON_APP_VERSIONS on OS: $OPERATING_SYSTEMS"
19851

@@ -304,7 +157,13 @@ function build_app_archive {
304157
--operating-system=$operating_system \
305158
--wheels
306159

307-
# 2. Create tarball or zip.
160+
# 2. Collect thirdparty deps source for native
161+
$BIN_DIR/python etc/scripts/fetch_thirdparty.py \
162+
--requirements=requirements-native.txt \
163+
--dest=thirdparty \
164+
--sdists
165+
166+
# 3. Create tarball or zip.
308167
# For now as a shortcut we use the Python setup.py sdist to create a tarball.
309168
# This is hackish and we should instead use our own archiving code that
310169
# would take a distutils manifest-like input

‎requirements-native.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
cffi==1.15.0
2+
intbitset==3.0.1
3+
MarkupSafe==2.0.1
4+
pyahocorasick==2.0.0b1
5+
PyYAML==6.0
6+

0 commit comments

Comments
 (0)