Skip to content

Commit f4aa8cd

Browse files
authored
Merge pull request #1454 from ritiek/parse-more-fields-in-cargo-manifest
Detect description and author fields from Cargo.toml manifest Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
2 parents 69f1b17 + 7416220 commit f4aa8cd

7 files changed

Lines changed: 237 additions & 56 deletions

File tree

src/packagedcode/cargo.py

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@
2727
from __future__ import unicode_literals
2828

2929
from collections import OrderedDict
30-
3130
import io
3231
import toml
3332
import logging
33+
import re
3434

3535
import attr
3636

@@ -58,9 +58,9 @@ class RustCargoCrate(models.Package):
5858
metafiles = ('Cargo.toml',)
5959
default_type = 'cargo'
6060
default_primary_language = 'Rust'
61-
default_web_baseurl = "https://crates.io/"
62-
default_download_baseurl = "https://crates.io/api/v1/"
63-
default_api_baseurl = "https://crates.io/api/v1/"
61+
default_web_baseurl = 'https://crates.io'
62+
default_download_baseurl = 'https://crates.io/api/v1'
63+
default_api_baseurl = 'https://crates.io/api/v1'
6464

6565
@classmethod
6666
def recognize(cls, location):
@@ -94,9 +94,7 @@ def parse(location):
9494
if not is_cargo_toml(location):
9595
return
9696

97-
with io.open(location, encoding='utf-8') as loc:
98-
package_data = toml.load(location, _dict=OrderedDict)
99-
97+
package_data = toml.load(location, _dict=OrderedDict)
10098
return build_package(package_data)
10199

102100

@@ -105,20 +103,80 @@ def build_package(package_data):
105103
Return a Pacakge object from a package data mapping or None.
106104
"""
107105

108-
name = package_data.get('package').get('name')
109-
version = package_data.get('package').get('version')
106+
core_package_data = package_data.get('package', {})
107+
name = core_package_data.get('name')
108+
version = core_package_data.get('version')
109+
description = core_package_data.get('description')
110+
if description:
111+
description = description.strip()
110112

111-
# TODO: Remove this ordered_dict_map once cargo.py is able to handle
112-
# the appropriate data (source_packages, dependencies, etc..)
113-
# At the moment, this is only useful for making tests pass
114-
ordered_dict_map = {}
115-
for key in ("source_packages", "dependencies", "keywords", "parties"):
116-
ordered_dict_map[key] = OrderedDict()
113+
authors = core_package_data.get('authors')
114+
parties = list(party_mapper(authors, party_role='author'))
117115

118116
package = RustCargoCrate(
119117
name=name,
120118
version=version,
121-
**ordered_dict_map
119+
description=description,
120+
parties=parties,
122121
)
123122

124123
return package
124+
125+
126+
def party_mapper(party, party_role):
127+
"""
128+
Yields a Party object with party of `party_role`.
129+
https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional
130+
"""
131+
for person in party:
132+
name, email = parse_person(person)
133+
yield models.Party(
134+
type=models.party_person,
135+
name=name,
136+
role=party_role,
137+
email=email)
138+
139+
140+
def parse_person(person):
141+
"""
142+
https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional
143+
A "person" is an object with an optional "name" or "email" field.
144+
145+
A person can be in the form:
146+
"author": "Isaac Z. Schlueter <i@izs.me>"
147+
148+
For example:
149+
>>> parse_person('Barney Rubble <b@rubble.com>')
150+
(u'Barney Rubble', u'b@rubble.com')
151+
>>> parse_person('Barney Rubble')
152+
(u'Barney Rubble', None)
153+
>>> parse_person('<b@rubble.com>')
154+
(None, u'b@rubble.com')
155+
"""
156+
157+
parsed = person_parser(person)
158+
if not parsed:
159+
name = None
160+
parsed = person_parser_no_name(person)
161+
else:
162+
name = parsed.group('name')
163+
164+
email = parsed.group('email')
165+
166+
if name:
167+
name = name.strip()
168+
if email:
169+
email = email.strip('<> ')
170+
171+
return name, email
172+
173+
174+
person_parser = re.compile(
175+
r'^(?P<name>[^\(<]+)'
176+
r'\s?'
177+
r'(?P<email><([^>]+)>)?'
178+
).match
179+
180+
person_parser_no_name = re.compile(
181+
r'(?P<email><([^>]+)>)?'
182+
).match

tests/packagedcode/data/cargo/clap/Cargo.toml.expected

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,18 @@
66
"qualifiers": null,
77
"subpath": null,
88
"primary_language": "Rust",
9-
"description": null,
9+
"description": "A simple to use, efficient, and full featured Command Line Argument Parser",
1010
"release_date": null,
11-
"parties": {},
12-
"keywords": {},
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Kevin K.",
16+
"email": "kbknapp@gmail.com",
17+
"url": null
18+
}
19+
],
20+
"keywords": [],
1321
"homepage_url": null,
1422
"download_url": null,
1523
"size": null,
@@ -25,11 +33,11 @@
2533
"declared_license": null,
2634
"notice_text": null,
2735
"manifest_path": null,
28-
"dependencies": {},
36+
"dependencies": [],
2937
"contains_source_code": null,
30-
"source_packages": {},
38+
"source_packages": [],
3139
"purl": "pkg:cargo/clap@2.32.0",
32-
"repository_homepage_url": "https://crates.io//crates/clap",
33-
"repository_download_url": "https://crates.io/api/v1//crates/clap/2.32.0/download",
34-
"api_data_url": "https://crates.io/api/v1//crates/clap"
40+
"repository_homepage_url": "https://crates.io/crates/clap",
41+
"repository_download_url": "https://crates.io/api/v1/crates/clap/2.32.0/download",
42+
"api_data_url": "https://crates.io/api/v1/crates/clap"
3543
}

tests/packagedcode/data/cargo/clippy/Cargo.toml.expected

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,46 @@
66
"qualifiers": null,
77
"subpath": null,
88
"primary_language": "Rust",
9-
"description": null,
9+
"description": "A bunch of helpful lints to avoid common pitfalls in Rust",
1010
"release_date": null,
11-
"parties": {},
12-
"keywords": {},
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Manish Goregaokar",
16+
"email": "manishsmail@gmail.com",
17+
"url": null
18+
},
19+
{
20+
"type": "person",
21+
"role": "author",
22+
"name": "Andre Bogus",
23+
"email": "bogusandre@gmail.com",
24+
"url": null
25+
},
26+
{
27+
"type": "person",
28+
"role": "author",
29+
"name": "Georg Brandl",
30+
"email": "georg@python.org",
31+
"url": null
32+
},
33+
{
34+
"type": "person",
35+
"role": "author",
36+
"name": "Martin Carton",
37+
"email": "cartonmartin@gmail.com",
38+
"url": null
39+
},
40+
{
41+
"type": "person",
42+
"role": "author",
43+
"name": "Oliver Schneider",
44+
"email": "clippy-iethah7aipeen8neex1a@oli-obk.de",
45+
"url": null
46+
}
47+
],
48+
"keywords": [],
1349
"homepage_url": null,
1450
"download_url": null,
1551
"size": null,
@@ -25,11 +61,11 @@
2561
"declared_license": null,
2662
"notice_text": null,
2763
"manifest_path": null,
28-
"dependencies": {},
64+
"dependencies": [],
2965
"contains_source_code": null,
30-
"source_packages": {},
66+
"source_packages": [],
3167
"purl": "pkg:cargo/clippy@0.0.212",
32-
"repository_homepage_url": "https://crates.io//crates/clippy",
33-
"repository_download_url": "https://crates.io/api/v1//crates/clippy/0.0.212/download",
34-
"api_data_url": "https://crates.io/api/v1//crates/clippy"
68+
"repository_homepage_url": "https://crates.io/crates/clippy",
69+
"repository_download_url": "https://crates.io/api/v1/crates/clippy/0.0.212/download",
70+
"api_data_url": "https://crates.io/api/v1/crates/clippy"
3571
}

tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,32 @@
66
"qualifiers": null,
77
"subpath": null,
88
"primary_language": "Rust",
9-
"description": null,
9+
"description": "Create books from markdown files",
1010
"release_date": null,
11-
"parties": {},
12-
"keywords": {},
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Mathieu David",
16+
"email": "mathieudavid@mathieudavid.org",
17+
"url": null
18+
},
19+
{
20+
"type": "person",
21+
"role": "author",
22+
"name": "Michael-F-Bryan",
23+
"email": "michaelfbryan@gmail.com",
24+
"url": null
25+
},
26+
{
27+
"type": "person",
28+
"role": "author",
29+
"name": "Matt Ickstadt",
30+
"email": "mattico8@gmail.com",
31+
"url": null
32+
}
33+
],
34+
"keywords": [],
1335
"homepage_url": null,
1436
"download_url": null,
1537
"size": null,
@@ -25,11 +47,11 @@
2547
"declared_license": null,
2648
"notice_text": null,
2749
"manifest_path": null,
28-
"dependencies": {},
50+
"dependencies": [],
2951
"contains_source_code": null,
30-
"source_packages": {},
52+
"source_packages": [],
3153
"purl": "pkg:cargo/mdbook@0.2.4-alpha.0",
32-
"repository_homepage_url": "https://crates.io//crates/mdbook",
33-
"repository_download_url": "https://crates.io/api/v1//crates/mdbook/0.2.4-alpha.0/download",
34-
"api_data_url": "https://crates.io/api/v1//crates/mdbook"
54+
"repository_homepage_url": "https://crates.io/crates/mdbook",
55+
"repository_download_url": "https://crates.io/api/v1/crates/mdbook/0.2.4-alpha.0/download",
56+
"api_data_url": "https://crates.io/api/v1/crates/mdbook"
3557
}

tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,25 @@
66
"qualifiers": null,
77
"subpath": null,
88
"primary_language": "Rust",
9-
"description": null,
9+
"description": "Tool to find and fix Rust formatting issues",
1010
"release_date": null,
11-
"parties": {},
12-
"keywords": {},
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Nicholas Cameron",
16+
"email": "ncameron@mozilla.com",
17+
"url": null
18+
},
19+
{
20+
"type": "person",
21+
"role": "author",
22+
"name": "The Rustfmt developers",
23+
"email": null,
24+
"url": null
25+
}
26+
],
27+
"keywords": [],
1328
"homepage_url": null,
1429
"download_url": null,
1530
"size": null,
@@ -25,11 +40,11 @@
2540
"declared_license": null,
2641
"notice_text": null,
2742
"manifest_path": null,
28-
"dependencies": {},
43+
"dependencies": [],
2944
"contains_source_code": null,
30-
"source_packages": {},
45+
"source_packages": [],
3146
"purl": "pkg:cargo/rustfmt-nightly@1.0.3",
32-
"repository_homepage_url": "https://crates.io//crates/rustfmt-nightly",
33-
"repository_download_url": "https://crates.io/api/v1//crates/rustfmt-nightly/1.0.3/download",
34-
"api_data_url": "https://crates.io/api/v1//crates/rustfmt-nightly"
47+
"repository_homepage_url": "https://crates.io/crates/rustfmt-nightly",
48+
"repository_download_url": "https://crates.io/api/v1/crates/rustfmt-nightly/1.0.3/download",
49+
"api_data_url": "https://crates.io/api/v1/crates/rustfmt-nightly"
3550
}

tests/packagedcode/data/cargo/rustup/Cargo.toml.expected

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,18 @@
66
"qualifiers": null,
77
"subpath": null,
88
"primary_language": "Rust",
9-
"description": null,
9+
"description": "Manage multiple rust installations with ease",
1010
"release_date": null,
11-
"parties": {},
12-
"keywords": {},
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Diggory Blake",
16+
"email": "diggsey@googlemail.com",
17+
"url": null
18+
}
19+
],
20+
"keywords": [],
1321
"homepage_url": null,
1422
"download_url": null,
1523
"size": null,
@@ -25,11 +33,11 @@
2533
"declared_license": null,
2634
"notice_text": null,
2735
"manifest_path": null,
28-
"dependencies": {},
36+
"dependencies": [],
2937
"contains_source_code": null,
30-
"source_packages": {},
38+
"source_packages": [],
3139
"purl": "pkg:cargo/rustup@1.17.0",
32-
"repository_homepage_url": "https://crates.io//crates/rustup",
33-
"repository_download_url": "https://crates.io/api/v1//crates/rustup/1.17.0/download",
34-
"api_data_url": "https://crates.io/api/v1//crates/rustup"
40+
"repository_homepage_url": "https://crates.io/crates/rustup",
41+
"repository_download_url": "https://crates.io/api/v1/crates/rustup/1.17.0/download",
42+
"api_data_url": "https://crates.io/api/v1/crates/rustup"
3543
}

0 commit comments

Comments
 (0)