Skip to content

Detect description and author fields from Cargo.toml manifest #1454

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pombredanne merged 11 commits into from
Mar 23, 2019
Merged

Detect description and author fields from Cargo.toml manifest #1454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2b1651b
Parse authors from Cargo.toml manifest
ritiek Mar 16, 2019
40049cf
Parse crate description from Cargo.toml manifest
ritiek Mar 16, 2019
7679320
Fix crates.io base URL formatting
ritiek Mar 16, 2019
b473e5d
Update tests
ritiek Mar 16, 2019
92c4604
Handle when description or authors are not present
ritiek Mar 16, 2019
10c9458
Add docstrings!
ritiek Mar 16, 2019
fb3d509
Code style consistency
ritiek Mar 19, 2019
1f3aac4
Remove dead code opening Cargo.toml manifest
ritiek Mar 19, 2019
bc20482
Pass "parties" to constructor in RustCargoCrate class
ritiek Mar 19, 2019
e8b2836
Do not unnecessarily map unimplemented manifest fields to an OrderedD…
ritiek Mar 19, 2019
7416220
Add tests for RegEx parsing
ritiek Mar 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 74 additions & 16 deletions src/packagedcode/cargo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@
from __future__ import unicode_literals

from collections import OrderedDict

import io
import toml
import logging
import re

import attr

Expand Down Expand Up @@ -58,9 +58,9 @@ class RustCargoCrate(models.Package):
metafiles = ('Cargo.toml',)
default_type = 'cargo'
default_primary_language = 'Rust'
default_web_baseurl = "https://crates.io/"
default_download_baseurl = "https://crates.io/api/v1/"
default_api_baseurl = "https://crates.io/api/v1/"
default_web_baseurl = 'https://crates.io'
default_download_baseurl = 'https://crates.io/api/v1'
default_api_baseurl = 'https://crates.io/api/v1'

@classmethod
def recognize(cls, location):
Expand Down Expand Up @@ -94,9 +94,7 @@ def parse(location):
if not is_cargo_toml(location):
return

with io.open(location, encoding='utf-8') as loc:
package_data = toml.load(location, _dict=OrderedDict)

package_data = toml.load(location, _dict=OrderedDict)
return build_package(package_data)


Expand All @@ -105,20 +103,80 @@ def build_package(package_data):
Return a Pacakge object from a package data mapping or None.
"""

name = package_data.get('package').get('name')
version = package_data.get('package').get('version')
core_package_data = package_data.get('package', {})
name = core_package_data.get('name')
version = core_package_data.get('version')
description = core_package_data.get('description')
if description:
description = description.strip()

# TODO: Remove this ordered_dict_map once cargo.py is able to handle
# the appropriate data (source_packages, dependencies, etc..)
# At the moment, this is only useful for making tests pass
ordered_dict_map = {}
for key in ("source_packages", "dependencies", "keywords", "parties"):
ordered_dict_map[key] = OrderedDict()
authors = core_package_data.get('authors')
parties = list(party_mapper(authors, party_role='author'))

package = RustCargoCrate(
name=name,
version=version,
**ordered_dict_map
description=description,
parties=parties,
)

return package


def party_mapper(party, party_role):
"""
Yields a Party object with party of `party_role`.
https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional
"""
for person in party:
name, email = parse_person(person)
yield models.Party(
type=models.party_person,
name=name,
role=party_role,
email=email)


def parse_person(person):
"""
https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional
A "person" is an object with an optional "name" or "email" field.

A person can be in the form:
"author": "Isaac Z. Schlueter <i@izs.me>"

For example:
>>> parse_person('Barney Rubble <b@rubble.com>')
(u'Barney Rubble', u'b@rubble.com')
>>> parse_person('Barney Rubble')
(u'Barney Rubble', None)
>>> parse_person('<b@rubble.com>')
(None, u'b@rubble.com')
"""

parsed = person_parser(person)
if not parsed:
name = None
parsed = person_parser_no_name(person)
else:
name = parsed.group('name')

email = parsed.group('email')

if name:
name = name.strip()
if email:
email = email.strip('<> ')

return name, email


person_parser = re.compile(

@pombredanne pombredanne Mar 19, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mind adding a few unit tests that showcase how this regex and the regex below behave with real data? This helps understand things better.

@ritiek ritiek Mar 19, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I'll add in some tests shortly.

r'^(?P<name>[^\(<]+)'
r'\s?'
r'(?P<email><([^>]+)>)?'
).match

person_parser_no_name = re.compile(
r'(?P<email><([^>]+)>)?'
).match
24 changes: 16 additions & 8 deletions tests/packagedcode/data/cargo/clap/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@
"qualifiers": null,
"subpath": null,
"primary_language": "Rust",
"description": null,
"description": "A simple to use, efficient, and full featured Command Line Argument Parser",
"release_date": null,
"parties": {},
"keywords": {},
"parties": [
{
"type": "person",
"role": "author",
"name": "Kevin K.",
"email": "kbknapp@gmail.com",
"url": null
}
],
"keywords": [],
"homepage_url": null,
"download_url": null,
"size": null,
Expand All @@ -25,11 +33,11 @@
"declared_license": null,
"notice_text": null,
"manifest_path": null,
"dependencies": {},
"dependencies": [],

@pombredanne pombredanne Mar 20, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was important as your use of empty dicts was changing the default type for this: it is a list.

"contains_source_code": null,
"source_packages": {},
"source_packages": [],
"purl": "pkg:cargo/clap@2.32.0",
"repository_homepage_url": "https://crates.io//crates/clap",
"repository_download_url": "https://crates.io/api/v1//crates/clap/2.32.0/download",
"api_data_url": "https://crates.io/api/v1//crates/clap"
"repository_homepage_url": "https://crates.io/crates/clap",
"repository_download_url": "https://crates.io/api/v1/crates/clap/2.32.0/download",
"api_data_url": "https://crates.io/api/v1/crates/clap"
}
52 changes: 44 additions & 8 deletions tests/packagedcode/data/cargo/clippy/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,46 @@
"qualifiers": null,
"subpath": null,
"primary_language": "Rust",
"description": null,
"description": "A bunch of helpful lints to avoid common pitfalls in Rust",
"release_date": null,
"parties": {},
"keywords": {},
"parties": [
{
"type": "person",
"role": "author",
"name": "Manish Goregaokar",
"email": "manishsmail@gmail.com",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Andre Bogus",
"email": "bogusandre@gmail.com",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Georg Brandl",
"email": "georg@python.org",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Martin Carton",
"email": "cartonmartin@gmail.com",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Oliver Schneider",
"email": "clippy-iethah7aipeen8neex1a@oli-obk.de",
"url": null
}
],
"keywords": [],
"homepage_url": null,
"download_url": null,
"size": null,
Expand All @@ -25,11 +61,11 @@
"declared_license": null,
"notice_text": null,
"manifest_path": null,
"dependencies": {},
"dependencies": [],
"contains_source_code": null,
"source_packages": {},
"source_packages": [],
"purl": "pkg:cargo/clippy@0.0.212",
"repository_homepage_url": "https://crates.io//crates/clippy",
"repository_download_url": "https://crates.io/api/v1//crates/clippy/0.0.212/download",
"api_data_url": "https://crates.io/api/v1//crates/clippy"
"repository_homepage_url": "https://crates.io/crates/clippy",
"repository_download_url": "https://crates.io/api/v1/crates/clippy/0.0.212/download",
"api_data_url": "https://crates.io/api/v1/crates/clippy"
}
38 changes: 30 additions & 8 deletions tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,32 @@
"qualifiers": null,
"subpath": null,
"primary_language": "Rust",
"description": null,
"description": "Create books from markdown files",
"release_date": null,
"parties": {},
"keywords": {},
"parties": [
{
"type": "person",
"role": "author",
"name": "Mathieu David",
"email": "mathieudavid@mathieudavid.org",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Michael-F-Bryan",
"email": "michaelfbryan@gmail.com",
"url": null
},
{
"type": "person",
"role": "author",
"name": "Matt Ickstadt",
"email": "mattico8@gmail.com",
"url": null
}
],
"keywords": [],
"homepage_url": null,
"download_url": null,
"size": null,
Expand All @@ -25,11 +47,11 @@
"declared_license": null,
"notice_text": null,
"manifest_path": null,
"dependencies": {},
"dependencies": [],
"contains_source_code": null,
"source_packages": {},
"source_packages": [],
"purl": "pkg:cargo/mdbook@0.2.4-alpha.0",
"repository_homepage_url": "https://crates.io//crates/mdbook",
"repository_download_url": "https://crates.io/api/v1//crates/mdbook/0.2.4-alpha.0/download",
"api_data_url": "https://crates.io/api/v1//crates/mdbook"
"repository_homepage_url": "https://crates.io/crates/mdbook",
"repository_download_url": "https://crates.io/api/v1/crates/mdbook/0.2.4-alpha.0/download",
"api_data_url": "https://crates.io/api/v1/crates/mdbook"
}
31 changes: 23 additions & 8 deletions tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,25 @@
"qualifiers": null,
"subpath": null,
"primary_language": "Rust",
"description": null,
"description": "Tool to find and fix Rust formatting issues",
"release_date": null,
"parties": {},
"keywords": {},
"parties": [
{
"type": "person",
"role": "author",
"name": "Nicholas Cameron",
"email": "ncameron@mozilla.com",
"url": null
},
{
"type": "person",
"role": "author",
"name": "The Rustfmt developers",
"email": null,
"url": null
}
],
"keywords": [],
"homepage_url": null,
"download_url": null,
"size": null,
Expand All @@ -25,11 +40,11 @@
"declared_license": null,
"notice_text": null,
"manifest_path": null,
"dependencies": {},
"dependencies": [],
"contains_source_code": null,
"source_packages": {},
"source_packages": [],
"purl": "pkg:cargo/rustfmt-nightly@1.0.3",
"repository_homepage_url": "https://crates.io//crates/rustfmt-nightly",
"repository_download_url": "https://crates.io/api/v1//crates/rustfmt-nightly/1.0.3/download",
"api_data_url": "https://crates.io/api/v1//crates/rustfmt-nightly"
"repository_homepage_url": "https://crates.io/crates/rustfmt-nightly",
"repository_download_url": "https://crates.io/api/v1/crates/rustfmt-nightly/1.0.3/download",
"api_data_url": "https://crates.io/api/v1/crates/rustfmt-nightly"
}
24 changes: 16 additions & 8 deletions tests/packagedcode/data/cargo/rustup/Cargo.toml.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,18 @@
"qualifiers": null,
"subpath": null,
"primary_language": "Rust",
"description": null,
"description": "Manage multiple rust installations with ease",
"release_date": null,
"parties": {},
"keywords": {},
"parties": [
{
"type": "person",
"role": "author",
"name": "Diggory Blake",
"email": "diggsey@googlemail.com",
"url": null
}
],
"keywords": [],
"homepage_url": null,
"download_url": null,
"size": null,
Expand All @@ -25,11 +33,11 @@
"declared_license": null,
"notice_text": null,
"manifest_path": null,
"dependencies": {},
"dependencies": [],
"contains_source_code": null,
"source_packages": {},
"source_packages": [],
"purl": "pkg:cargo/rustup@1.17.0",
"repository_homepage_url": "https://crates.io//crates/rustup",
"repository_download_url": "https://crates.io/api/v1//crates/rustup/1.17.0/download",
"api_data_url": "https://crates.io/api/v1//crates/rustup"
"repository_homepage_url": "https://crates.io/crates/rustup",
"repository_download_url": "https://crates.io/api/v1/crates/rustup/1.17.0/download",
"api_data_url": "https://crates.io/api/v1/crates/rustup"
}
Loading