Skip to content

Commit 3068680

Browse files
pombredannepombredanne
authored
Merge pull request #2078 from rpotter12/requirements-parser
Detect Python requirements.txt as Package dependencies Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
2 parents f488b6c + 1ee7309 commit 3068680

17 files changed

Lines changed: 570 additions & 26 deletions

‎src/packagedcode/pypi.py‎

Lines changed: 69 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ def logger_debug(*args):
9090

9191
@attr.s()
9292
class PythonPackage(models.Package):
93-
metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg')
93+
metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg', '*requirements*.txt', '*requirements*.in')
9494
extensions = ('.egg', '.whl', '.pyz', '.pex',)
9595
default_type = 'pypi'
9696
default_primary_language = 'Python'
@@ -119,6 +119,8 @@ def parse(location):
119119
file_name = fileutils.file_name(location)
120120
parsers = {
121121
'setup.py': parse_setup_py,
122+
'requirements.txt': parse_requirements_txt,
123+
'requirements.in': parse_requirements_txt,
122124
'metadata.json': parse_metadata,
123125
'PKG-INFO': parse_pkg_info,
124126
'.whl': parse_wheel,
@@ -194,12 +196,29 @@ def parse_dependencies(location, package):
194196
package.dependencies = dependencies
195197

196198

199+
dependency_type_by_extensions = {
200+
('.txt', '.in'): 'requirements.txt',
201+
}
202+
203+
204+
def get_dependency_type(file_name, dependency_type_by_extensions=dependency_type_by_extensions):
205+
"""
206+
Return the type of a dependency as a string or None given a `file_name` string.
207+
"""
208+
for extensions, dependency_type in dependency_type_by_extensions.items():
209+
if file_name.endswith(extensions):
210+
return dependency_type
211+
212+
197213
def parse_with_dparse(location):
198214
is_dir = filetype.is_dir(location)
199215
if is_dir:
200216
return
201217
file_name = fileutils.file_name(location)
202-
if file_name not in (filetypes.requirements_txt,
218+
219+
dependency_type = get_dependency_type(file_name)
220+
221+
if dependency_type not in (filetypes.requirements_txt,
203222
filetypes.conda_yml,
204223
filetypes.tox_ini,
205224
filetypes.pipfile,
@@ -211,27 +230,54 @@ def parse_with_dparse(location):
211230
mode = 'r'
212231
with open(location, mode) as f:
213232
content = f.read()
214-
df = dparse.parse(content, file_type=file_name)
215-
df_dependencies = df.dependencies
216-
if not df_dependencies:
217-
return
218-
package_dependencies = []
219-
for df_dependency in df_dependencies:
220-
specs = df_dependency.specs
221-
requirement = None
222-
if specs:
223-
requirement = str(specs)
224-
package_dependencies.append(
225-
models.DependentPackage(
226-
purl=PackageURL(
227-
type='pypi', name=df_dependency.name).to_string(),
228-
scope='dependencies',
229-
is_runtime=True,
230-
is_optional=False,
231-
requirement=requirement,
232-
)
233+
234+
df = dparse.parse(content, file_type=dependency_type)
235+
df_dependencies = df.dependencies
236+
237+
if not df_dependencies:
238+
return
239+
240+
package_dependencies = []
241+
for df_dependency in df_dependencies:
242+
specs = list(df_dependency.specs._specs)
243+
is_resolved = False
244+
requirement = None
245+
purl = PackageURL(
246+
type='pypi',
247+
name=df_dependency.name
248+
).to_string()
249+
if specs:
250+
requirement = str(df_dependency.specs)
251+
for spec in specs:
252+
operator = spec.operator
253+
version = spec.version
254+
if any(operator == element for element in ('==', '===')):
255+
is_resolved = True
256+
purl = PackageURL(
257+
type='pypi',
258+
name=df_dependency.name,
259+
version=version
260+
).to_string()
261+
package_dependencies.append(
262+
models.DependentPackage(
263+
purl=purl,
264+
scope='dependencies',
265+
is_runtime=True,
266+
is_optional=False,
267+
is_resolved=is_resolved,
268+
requirement=requirement
233269
)
234-
return package_dependencies
270+
)
271+
272+
return package_dependencies
273+
274+
275+
def parse_requirements_txt(location):
276+
"""
277+
Return a package built from requirements.txt.
278+
"""
279+
package_dependencies = parse_with_dparse(location)
280+
return PythonPackage(dependencies=package_dependencies)
235281

236282

237283
def parse_setup_py(location):
@@ -349,7 +395,7 @@ def parse_metadata(location):
349395
parties.append(models.Party(type=models.party_person, name=name, role='contact'))
350396

351397
description = build_description(
352-
infos.get('summary') ,
398+
infos.get('summary'),
353399
infos.get('description')
354400
)
355401

‎tests/packagedcode/data/plugin/help.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ Package: nuget
203203
--------------------------------------------
204204
Package: pypi
205205
class: packagedcode.pypi:PythonPackage
206-
metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg
206+
metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg, *requirements*.txt, *requirements*.in
207207
extensions: .egg, .whl, .pyz, .pex
208208

209209
--------------------------------------------

‎tests/packagedcode/data/pypi/requirements_txt/sample1/output.expected.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": null,
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": null,
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": null,
25+
"declared_license": null,
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": [
29+
{
30+
"purl": "pkg:pypi/setuptools",
31+
"requirement": ">=32.0.0",
32+
"scope": "dependencies",
33+
"is_runtime": true,
34+
"is_optional": false,
35+
"is_resolved": false
36+
},
37+
{
38+
"purl": "pkg:pypi/nose",
39+
"requirement": ">=1.3.7",
40+
"scope": "dependencies",
41+
"is_runtime": true,
42+
"is_optional": false,
43+
"is_resolved": false
44+
},
45+
{
46+
"purl": "pkg:pypi/chardet",
47+
"requirement": ">=3.0.4",
48+
"scope": "dependencies",
49+
"is_runtime": true,
50+
"is_optional": false,
51+
"is_resolved": false
52+
}
53+
],
54+
"contains_source_code": null,
55+
"source_packages": [],
56+
"purl": null,
57+
"repository_homepage_url": null,
58+
"repository_download_url": null,
59+
"api_data_url": null
60+
}

‎tests/packagedcode/data/pypi/requirements_txt/sample1/requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setuptools>=32.0.0
2+
nose>=1.3.7
3+
chardet>=3.0.4

‎tests/packagedcode/data/pypi/requirements_txt/sample2/output.expected.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": null,
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": null,
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": null,
25+
"declared_license": null,
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": [
29+
{
30+
"purl": "pkg:pypi/aiohttp@3.6.2",
31+
"requirement": "==3.6.2",
32+
"scope": "dependencies",
33+
"is_runtime": true,
34+
"is_optional": false,
35+
"is_resolved": true
36+
},
37+
{
38+
"purl": "pkg:pypi/async-timeout@3.0.1",
39+
"requirement": "==3.0.1",
40+
"scope": "dependencies",
41+
"is_runtime": true,
42+
"is_optional": false,
43+
"is_resolved": true
44+
},
45+
{
46+
"purl": "pkg:pypi/attrs@19.3.0",
47+
"requirement": "==19.3.0",
48+
"scope": "dependencies",
49+
"is_runtime": true,
50+
"is_optional": false,
51+
"is_resolved": true
52+
}
53+
],
54+
"contains_source_code": null,
55+
"source_packages": [],
56+
"purl": null,
57+
"repository_homepage_url": null,
58+
"repository_download_url": null,
59+
"api_data_url": null
60+
}

‎tests/packagedcode/data/pypi/requirements_txt/sample2/sample-requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
aiohttp==3.6.2
2+
async-timeout==3.0.1
3+
attrs==19.3.0

‎tests/packagedcode/data/pypi/requirements_txt/sample3/output.expected.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": null,
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": null,
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": null,
25+
"declared_license": null,
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": [
29+
{
30+
"purl": "pkg:pypi/prompt-toolkit",
31+
"requirement": ">=1.0.14",
32+
"scope": "dependencies",
33+
"is_runtime": true,
34+
"is_optional": false,
35+
"is_resolved": false
36+
},
37+
{
38+
"purl": "pkg:pypi/pygments@2.2.0",
39+
"requirement": "==2.2.0",
40+
"scope": "dependencies",
41+
"is_runtime": true,
42+
"is_optional": false,
43+
"is_resolved": true
44+
},
45+
{
46+
"purl": "pkg:pypi/regex",
47+
"requirement": null,
48+
"scope": "dependencies",
49+
"is_runtime": true,
50+
"is_optional": false,
51+
"is_resolved": false
52+
}
53+
],
54+
"contains_source_code": null,
55+
"source_packages": [],
56+
"purl": null,
57+
"repository_homepage_url": null,
58+
"repository_download_url": null,
59+
"api_data_url": null
60+
}

‎tests/packagedcode/data/pypi/requirements_txt/sample3/requirements-dev.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
prompt_toolkit>=1.0.14
2+
Pygments==2.2.0
3+
regex

‎tests/packagedcode/data/pypi/requirements_txt/sample4/output.expected.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": null,
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": null,
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": null,
25+
"declared_license": null,
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": [
29+
{
30+
"purl": "pkg:pypi/setuptools",
31+
"requirement": null,
32+
"scope": "dependencies",
33+
"is_runtime": true,
34+
"is_optional": false,
35+
"is_resolved": false
36+
},
37+
{
38+
"purl": "pkg:pypi/nose",
39+
"requirement": null,
40+
"scope": "dependencies",
41+
"is_runtime": true,
42+
"is_optional": false,
43+
"is_resolved": false
44+
}
45+
],
46+
"contains_source_code": null,
47+
"source_packages": [],
48+
"purl": null,
49+
"repository_homepage_url": null,
50+
"repository_download_url": null,
51+
"api_data_url": null
52+
}

‎tests/packagedcode/data/pypi/requirements_txt/sample4/requirements.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
setuptools
2+
nose

0 commit comments

Comments
 (0)