Skip to content

Commit 746c336

Browse files
JonoYangJonoYang
authored
Merge pull request #2992 from nexB/improve-gradle
Improve packagecode and other release prep
2 parents 9d885d5 + 03193c0 commit 746c336

241 files changed

Lines changed: 9540 additions & 1352 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/scancode-release.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -153,16 +153,16 @@ jobs:
153153
rm -rf thirdparty build dist
154154
rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info
155155
mkdir -p thirdparty
156+
venv/bin/python etc/scripts/fetch_thirdparty.py \
157+
--requirements=requirements-native.txt \
158+
--dest=thirdparty \
159+
--sdists
156160
venv/bin/python etc/scripts/fetch_thirdparty.py \
157161
--requirements=requirements.txt \
158162
--dest=thirdparty \
159163
--python-version=$python_version \
160164
--operating-system=$operating_system \
161165
--wheels
162-
venv/bin/python etc/scripts/fetch_thirdparty.py \
163-
--requirements=requirements-native.txt \
164-
--dest=thirdparty \
165-
--sdists
166166
venv/bin/python setup.py --quiet sdist --formats=$formats
167167
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
168168
@@ -208,16 +208,16 @@ jobs:
208208
rm -rf thirdparty build dist
209209
rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info
210210
mkdir -p thirdparty
211+
venv/bin/python etc/scripts/fetch_thirdparty.py \
212+
--requirements=requirements-native.txt \
213+
--dest=thirdparty \
214+
--sdists
211215
venv/bin/python etc/scripts/fetch_thirdparty.py \
212216
--requirements=requirements.txt \
213217
--dest=thirdparty \
214218
--python-version=$python_version \
215219
--operating-system=$operating_system \
216220
--wheels
217-
venv/bin/python etc/scripts/fetch_thirdparty.py \
218-
--requirements=requirements-native.txt \
219-
--dest=thirdparty \
220-
--sdists
221221
venv/bin/python setup.py --quiet sdist --formats=$formats
222222
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
223223
@@ -262,16 +262,16 @@ jobs:
262262
rm -rf thirdparty build dist
263263
rm -rf .eggs src/scancode_toolkit.egg-info src/scancode_toolkit_mini.egg-info
264264
mkdir -p thirdparty
265+
venv/bin/python etc/scripts/fetch_thirdparty.py \
266+
--requirements=requirements-native.txt \
267+
--dest=thirdparty \
268+
--sdists
265269
venv/bin/python etc/scripts/fetch_thirdparty.py \
266270
--requirements=requirements.txt \
267271
--dest=thirdparty \
268272
--python-version=$python_version \
269273
--operating-system=$operating_system \
270274
--wheels
271-
venv/bin/python etc/scripts/fetch_thirdparty.py \
272-
--requirements=requirements-native.txt \
273-
--dest=thirdparty \
274-
--sdists
275275
venv/bin/python setup.py --quiet sdist --formats=$formats
276276
venv/bin/python etc/release/scancode_rename_archives.py dist/ _py$python_version-$operating_system
277277

‎configure‎

Lines changed: 133 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,129 @@
11
#!/usr/bin/env bash
22
#
33
# Copyright (c) nexB Inc. and others. All rights reserved.
4-
# SPDX-License-Identifier: Apache-2.0
4+
# SPDX-License-Identifier: Apache-2.0 AND MIT
55
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# ScanCode is a trademark of nexB Inc.
67
# See https://github.com/nexB/ for support or download.
78
# See https://aboutcode.org for more information about nexB OSS projects.
89
#
910

1011
set -e
1112
#set -x
1213

14+
###################################################################################
15+
###################################################################################
16+
# from https://raw.githubusercontent.com/mkropat/sh-realpath/58c03982cfd8accbcf0c4426a4adf0f120a8b2bb/realpath.sh
17+
# realpath emulation for portability on *nix
18+
# this allow running scancode from arbitrary locations and from symlinks
19+
#
20+
# Copyright (c) 2014 Michael Kropat
21+
#
22+
# Permission is hereby granted, free of charge, to any person obtaining a copy
23+
# of this software and associated documentation files (the "Software"), to deal
24+
# in the Software without restriction, including without limitation the rights
25+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
26+
# copies of the Software, and to permit persons to whom the Software is
27+
# furnished to do so, subject to the following conditions:
28+
#
29+
# The above copyright notice and this permission notice shall be included in
30+
# all copies or substantial portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
38+
# THE SOFTWARE.
39+
40+
41+
realpath() {
42+
canonicalize_path "$(resolve_symlinks "$1")"
43+
}
44+
45+
resolve_symlinks() {
46+
_resolve_symlinks "$1"
47+
}
48+
49+
_resolve_symlinks() {
50+
_assert_no_path_cycles "$@" || return
51+
52+
local dir_context path
53+
path=$(readlink -- "$1")
54+
if [ $? -eq 0 ]; then
55+
dir_context=$(dirname -- "$1")
56+
_resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@"
57+
else
58+
printf '%s\n' "$1"
59+
fi
60+
}
61+
62+
_prepend_dir_context_if_necessary() {
63+
if [ "$1" = . ]; then
64+
printf '%s\n' "$2"
65+
else
66+
_prepend_path_if_relative "$1" "$2"
67+
fi
68+
}
69+
70+
_prepend_path_if_relative() {
71+
case "$2" in
72+
/* ) printf '%s\n' "$2" ;;
73+
* ) printf '%s\n' "$1/$2" ;;
74+
esac
75+
}
76+
77+
_assert_no_path_cycles() {
78+
local target path
79+
80+
target=$1
81+
shift
82+
83+
for path in "$@"; do
84+
if [ "$path" = "$target" ]; then
85+
return 1
86+
fi
87+
done
88+
}
89+
90+
canonicalize_path() {
91+
if [ -d "$1" ]; then
92+
_canonicalize_dir_path "$1"
93+
else
94+
_canonicalize_file_path "$1"
95+
fi
96+
}
97+
98+
_canonicalize_dir_path() {
99+
(cd "$1" 2>/dev/null && pwd -P)
100+
}
101+
102+
_canonicalize_file_path() {
103+
local dir file
104+
dir=$(dirname -- "$1")
105+
file=$(basename -- "$1")
106+
(cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file")
107+
}
108+
109+
###################################################################################
110+
###################################################################################
111+
112+
# Now run configure proper
113+
114+
################################
115+
# Setup current directory where this script lives
116+
CFG_BIN="$( realpath "${BASH_SOURCE[0]}" )"
117+
CFG_ROOT_DIR="$( cd "$( dirname "${CFG_BIN}" )" && pwd )"
118+
119+
CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
120+
121+
# force relaunching under X86-64 architecture on macOS M1/ARM
122+
if [[ $OSTYPE == 'darwin'* && $(uname -m) == 'arm64' ]]; then
123+
arch -x86_64 /bin/bash -c "$CFG_ROOT_DIR/configure $@"
124+
exit $?
125+
fi
126+
13127
################################
14128
# A configuration script to set things up:
15129
# create a virtualenv and install or update thirdparty packages.
@@ -52,14 +166,21 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
52166
CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
53167

54168

169+
################################
170+
# Install with or without and index. With "--no-index" this is using only local wheels
171+
# This is an offline mode with no index and no network operations
172+
NO_INDEX="--no-index "
173+
174+
55175
################################
56176
# Thirdparty package locations and index handling
57-
# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
58-
# offline mode for scancode installation with no index at all
59-
if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then
60-
PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty"
177+
# Find packages from the local thirdparty directory if present
178+
thirddir=$CFG_ROOT_DIR/thirdparty
179+
if [[ "$(echo $thirddir/*.whl)x" != "$thirddir/*.whlx" ]]; then
180+
PIP_EXTRA_ARGS="$NO_INDEX --find-links $CFG_ROOT_DIR/thirdparty"
61181
fi
62182

183+
63184
################################
64185
# Set the quiet flag to empty if not defined
65186
if [[ "$CFG_QUIET" == "" ]]; then
@@ -102,25 +223,13 @@ create_virtualenv() {
102223
wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
103224
fi
104225

105-
if [[ $OSTYPE == 'darwin'* && $(uname -m) == 'arm64' ]]; then
106-
arch -x86_64 /bin/bash -c "$PYTHON_EXECUTABLE \"$VIRTUALENV_PYZ\" \
107-
--wheel embed --pip embed --setuptools embed \
108-
--seeder pip \
109-
--never-download \
110-
--no-periodic-update \
111-
--no-vcs-ignore \
112-
$CFG_QUIET \
113-
\"$CFG_ROOT_DIR/$VENV_DIR\" "
114-
else
115-
$PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
116-
--wheel embed --pip embed --setuptools embed \
117-
--seeder pip \
118-
--never-download \
119-
--no-periodic-update \
120-
--no-vcs-ignore \
121-
$CFG_QUIET \
122-
"$CFG_ROOT_DIR/$VENV_DIR"
123-
fi
226+
$PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
227+
--wheel embed --pip embed --setuptools embed \
228+
--never-download \
229+
--no-periodic-update \
230+
--no-vcs-ignore \
231+
$CFG_QUIET \
232+
"$CFG_ROOT_DIR/$VENV_DIR"
124233
fi
125234
}
126235

‎etc/scripts/fetch_thirdparty.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,6 @@
110110
is_flag=True,
111111
help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
112112
)
113-
114113
@click.help_option("-h", "--help")
115114
def fetch_thirdparty(
116115
requirements_files,

‎etc/scripts/misc/openhub_scraper.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def fetch_and_save_license(url, force=False, directory="openhub_licenses"):
6565

6666
os.makedirs(directory, exist_ok=True)
6767
print(f" Fetching: {url}")
68-
time.sleep(.1)
68+
time.sleep(0.1)
6969
content = urlopen(url).read()
7070
with open(lic_file, "wb") as of:
7171
of.write(content)

‎etc/scripts/misc/test_openhub_scraper.py‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,26 @@
1414

1515

1616
def test_scraping_unicode_and_ascii():
17-
test_file = os.path.join(
18-
os.path.dirname(__file__), "testdata/openhub_html.html"
19-
)
17+
test_file = os.path.join(os.path.dirname(__file__), "testdata/openhub_html.html")
2018
with open(test_file, "r") as f:
2119
test_content = f.read()
2220

2321
licenses = list(openhub_scraper.list_licenses_on_page(test_content))
2422

2523
result = [i for i in licenses if i["name"] == "Sleepycat License"]
26-
expected = [{
27-
"url": "https://www.openhub.net/licenses/sleepycat",
28-
"name": "Sleepycat License",
29-
}]
24+
expected = [
25+
{
26+
"url": "https://www.openhub.net/licenses/sleepycat",
27+
"name": "Sleepycat License",
28+
}
29+
]
3030
assert result == expected
3131

3232
result = [i for i in licenses if i["name"] == "Sun Public License v1.0"]
33-
expected = [{
34-
"url": "https://www.openhub.net/licenses/sun_public",
35-
"name": "Sun Public License v1.0",
36-
}]
33+
expected = [
34+
{
35+
"url": "https://www.openhub.net/licenses/sun_public",
36+
"name": "Sun Public License v1.0",
37+
}
38+
]
3739
assert result == expected

0 commit comments

Comments
 (0)