Skip to content

Commit 7036274

Browse files
pombredannepombredanne
authored
Merge pull request #2118 from Abhishek-Dev09/#295-requirements.txt
Automate the creation of requirement.txt and fetch deps
2 parents 07685a8 + c5ce300 commit 7036274

2 files changed

Lines changed: 262 additions & 0 deletions

File tree

‎etc/scripts/freeze_and_update_reqs.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright nexB Inc. and others. All rights reserved.
4+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
5+
# The ScanCode software is licensed under the Apache License version 2.0.
6+
# Data generated with ScanCode require an acknowledgment.
7+
# ScanCode is a trademark of nexB Inc.
8+
#
9+
# You may not use this software except in compliance with the License.
10+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
11+
# Unless required by applicable law or agreed to in writing, software distributed
12+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
14+
# specific language governing permissions and limitations under the License.
15+
#
16+
# When you publish or redistribute any data created with ScanCode or any ScanCode
17+
# derivative work, you must accompany this data with the following acknowledgment:
18+
#
19+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
20+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
21+
# ScanCode should be considered or used as legal advice. Consult an Attorney
22+
# for any legal advice.
23+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
24+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
25+
26+
from __future__ import absolute_import
27+
from __future__ import print_function
28+
29+
import argparse
30+
from fnmatch import fnmatchcase
31+
import os
32+
from shutil import copy
33+
from subprocess import run
34+
import sys
35+
import tempfile
36+
37+
from commoncode.fileutils import resource_iter
38+
39+
python_version = str(sys.version_info[0]) + str(sys.version_info[1])
40+
py_abi = '{0}cp{1}{0}'.format('*', python_version)
41+
42+
43+
def generate_req_text(find_links, req_file, package_name=None, upgrade=False):
44+
"""
45+
Generate a requirement file as `req_file` of all dependencies wheels and
46+
sdists present at the find_links.If a `package_name` is provided it will
47+
be updated to its latest version and if upgrade option is called,it will
48+
be updated all the wheels to the latest version.
49+
"""
50+
thirdparty = resource_iter(find_links, with_dirs=False)
51+
dependencies = [
52+
files
53+
for files in thirdparty
54+
if fnmatchcase(files, '*py3*')
55+
or fnmatchcase(files, py_abi)
56+
or (
57+
fnmatchcase(files, '*tar.gz*')
58+
and not fnmatchcase(files, '*py2-ipaddress-3.4.1.tar.gz*')
59+
)
60+
]
61+
with tempfile.TemporaryDirectory() as temp_dir:
62+
for deps in dependencies:
63+
copy(deps, temp_dir)
64+
pip_args = [
65+
'pip-compile',
66+
'--generate-hashes',
67+
'--find-links',
68+
temp_dir,
69+
'--output-file',
70+
req_file,
71+
'--allow-unsafe',
72+
'--pip-args',
73+
'--no-index',
74+
]
75+
if upgrade:
76+
pip_args.append('--upgrade')
77+
if package_name:
78+
pip_args.extend(['--upgrade-package', package_name])
79+
run(pip_args)
80+
81+
82+
def main_with_args(args: str) -> None:
83+
parser = argparse.ArgumentParser(
84+
description="""Generate a requirement file as `requirement` of all dependencies wheels and
85+
sdists present at the find_links.If a `upgrade-package` option is called it
86+
will update provided `package_name` to its latest version and if upgrade
87+
option is called,it will be update all the wheels/sdist to the latest version.
88+
""",
89+
formatter_class=argparse.RawDescriptionHelpFormatter,
90+
)
91+
92+
parser.add_argument(
93+
'--find-links',
94+
help='Required: Look for archives in this directory or on this HTML page',
95+
type=str,
96+
required=True,
97+
)
98+
99+
parser.add_argument(
100+
'--requirement',
101+
help='Required: Requirement file name.',
102+
type=str,
103+
required=True,
104+
)
105+
106+
parser.add_argument(
107+
'--upgrade',
108+
help='Optional: Try to upgrade all dependencies to their latest versions',
109+
action='store_true',
110+
)
111+
112+
parser.add_argument(
113+
'--upgrade-package',
114+
help='Optional: Specify particular packages to upgrade.',
115+
type=str,
116+
default=None,
117+
)
118+
119+
args = parser.parse_args()
120+
121+
find_links = args.find_links
122+
requirement = args.requirement
123+
upgrade_package = args.upgrade_package or None
124+
upgrade = args.upgrade or False
125+
generate_req_text(
126+
find_links=find_links,
127+
requirement=requirement,
128+
upgrade_package=upgrade_package,
129+
upgrade=upgrade,
130+
)
131+
132+
133+
def main() -> None:
134+
main_with_args(sys.argv[1:])
135+
136+
137+
if __name__ == '__main__':
138+
main()

‎etc/scripts/search-package.py‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright nexB Inc. and others. All rights reserved.
4+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
5+
# The ScanCode software is licensed under the Apache License version 2.0.
6+
# Data generated with ScanCode require an acknowledgment.
7+
# ScanCode is a trademark of nexB Inc.
8+
#
9+
# You may not use this software except in compliance with the License.
10+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
11+
# Unless required by applicable law or agreed to in writing, software distributed
12+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
14+
# specific language governing permissions and limitations under the License.
15+
#
16+
# When you publish or redistribute any data created with ScanCode or any ScanCode
17+
# derivative work, you must accompany this data with the following acknowledgment:
18+
#
19+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
20+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
21+
# ScanCode should be considered or used as legal advice. Consult an Attorney
22+
# for any legal advice.
23+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
24+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
25+
26+
from __future__ import absolute_import
27+
from __future__ import print_function
28+
29+
import argparse
30+
import fnmatch
31+
import sys
32+
33+
from commoncode.fileutils import resource_iter
34+
35+
def search_package(package_name, target, version=None):
36+
"""
37+
Search `package_name` (with an optional `version`) in the `target` directory.
38+
Print results on screen.
39+
"""
40+
41+
if version:
42+
package_name = '*{}-{}*'.format(package_name, version)
43+
else:
44+
package_name = '*{}*'.format(package_name)
45+
thirdparty = resource_iter(target, with_dirs=False)
46+
dependency = [
47+
files for files in thirdparty if fnmatch.fnmatchcase(files, package_name)
48+
]
49+
if dependency:
50+
whl = [files for files in dependency if files.endswith('.whl')]
51+
## There are multiple version of same package So list of wheel will be considered.
52+
sdist = [files for files in dependency if files.endswith('.tar.gz')]
53+
about = [files for files in dependency if files.endswith('.ABOUT')]
54+
notice = [files for files in dependency if files.endswith('.NOTICE')]
55+
license = [files for files in dependency if files.endswith('.LICENSE')]
56+
print('\nSearched package wheel are:')
57+
print(*whl, sep='\n')
58+
if sdist:
59+
print('\nCorresponding sdist are:')
60+
print(*sdist, sep='\n')
61+
else:
62+
print('\nCorresponding sdist does not exits in target')
63+
if about:
64+
print('\nCorresponding .ABOUT are:')
65+
print(*about, sep='\n')
66+
else:
67+
print('\nCorresponding .ABOUT does not exits in target\n')
68+
if license:
69+
print('\nCorresponding .LICENSE are:')
70+
print(*licence, sep='\n')
71+
else:
72+
print('\nCorresponding .LICENSE does not exits in target')
73+
if notice:
74+
print('\nCorresponding .NOTICE are:')
75+
print(*notice, sep='\n')
76+
else:
77+
print('\nCorresponding .NOTICE does not exits in target\n')
78+
else:
79+
print('\nSpecified package does not exist\n')
80+
81+
82+
def main_with_args(args: str) -> None:
83+
parser = argparse.ArgumentParser(
84+
description="""Search PACKAGE_NAME (with an optional VERSION_OF_PACKAGE) in the
85+
TARGET_DIR directory.Print results on screen.
86+
""",
87+
formatter_class=argparse.RawDescriptionHelpFormatter,
88+
)
89+
90+
parser.add_argument(
91+
'--package_name',
92+
help='Name of the package to search in the TARGET_DIR directory',
93+
type=str,
94+
required=True,
95+
)
96+
97+
parser.add_argument(
98+
'--directory',
99+
help='Directory to search for package wheels and tarballs. [default: thirdparty]',
100+
type=str,
101+
default='thirdparty',
102+
)
103+
104+
parser.add_argument(
105+
'--version',
106+
help='Optional package version to search.',
107+
type=str,
108+
default=None,
109+
)
110+
111+
args = parser.parse_args()
112+
113+
package_name = args.package_name
114+
directory = args.directory
115+
version = args.version
116+
search_package(package_name, directory, version)
117+
118+
119+
def main() -> None:
120+
main_with_args(sys.argv[1:])
121+
122+
123+
if __name__ == '__main__':
124+
main()

0 commit comments

Comments
 (0)