diff --git a/src/packagedcode/cargo.py b/src/packagedcode/cargo.py index bde7a1ac8dc..1b3748f723e 100644 --- a/src/packagedcode/cargo.py +++ b/src/packagedcode/cargo.py @@ -27,10 +27,10 @@ from __future__ import unicode_literals from collections import OrderedDict - import io import toml import logging +import re import attr @@ -58,9 +58,9 @@ class RustCargoCrate(models.Package): metafiles = ('Cargo.toml',) default_type = 'cargo' default_primary_language = 'Rust' - default_web_baseurl = "https://crates.io/" - default_download_baseurl = "https://crates.io/api/v1/" - default_api_baseurl = "https://crates.io/api/v1/" + default_web_baseurl = 'https://crates.io' + default_download_baseurl = 'https://crates.io/api/v1' + default_api_baseurl = 'https://crates.io/api/v1' @classmethod def recognize(cls, location): @@ -94,9 +94,7 @@ def parse(location): if not is_cargo_toml(location): return - with io.open(location, encoding='utf-8') as loc: - package_data = toml.load(location, _dict=OrderedDict) - + package_data = toml.load(location, _dict=OrderedDict) return build_package(package_data) @@ -105,20 +103,80 @@ def build_package(package_data): Return a Pacakge object from a package data mapping or None. """ - name = package_data.get('package').get('name') - version = package_data.get('package').get('version') + core_package_data = package_data.get('package', {}) + name = core_package_data.get('name') + version = core_package_data.get('version') + description = core_package_data.get('description') + if description: + description = description.strip() - # TODO: Remove this ordered_dict_map once cargo.py is able to handle - # the appropriate data (source_packages, dependencies, etc..) - # At the moment, this is only useful for making tests pass - ordered_dict_map = {} - for key in ("source_packages", "dependencies", "keywords", "parties"): - ordered_dict_map[key] = OrderedDict() + authors = core_package_data.get('authors') + parties = list(party_mapper(authors, party_role='author')) package = RustCargoCrate( name=name, version=version, - **ordered_dict_map + description=description, + parties=parties, ) return package + + +def party_mapper(party, party_role): + """ + Yields a Party object with party of `party_role`. + https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional + """ + for person in party: + name, email = parse_person(person) + yield models.Party( + type=models.party_person, + name=name, + role=party_role, + email=email) + + +def parse_person(person): + """ + https://doc.rust-lang.org/cargo/reference/manifest.html#the-authors-field-optional + A "person" is an object with an optional "name" or "email" field. + + A person can be in the form: + "author": "Isaac Z. Schlueter " + + For example: + >>> parse_person('Barney Rubble ') + (u'Barney Rubble', u'b@rubble.com') + >>> parse_person('Barney Rubble') + (u'Barney Rubble', None) + >>> parse_person('') + (None, u'b@rubble.com') + """ + + parsed = person_parser(person) + if not parsed: + name = None + parsed = person_parser_no_name(person) + else: + name = parsed.group('name') + + email = parsed.group('email') + + if name: + name = name.strip() + if email: + email = email.strip('<> ') + + return name, email + + +person_parser = re.compile( + r'^(?P[^\(<]+)' + r'\s?' + r'(?P<([^>]+)>)?' +).match + +person_parser_no_name = re.compile( + r'(?P<([^>]+)>)?' +).match diff --git a/tests/packagedcode/data/cargo/clap/Cargo.toml.expected b/tests/packagedcode/data/cargo/clap/Cargo.toml.expected index ac6afeec877..bfc6faf9aff 100644 --- a/tests/packagedcode/data/cargo/clap/Cargo.toml.expected +++ b/tests/packagedcode/data/cargo/clap/Cargo.toml.expected @@ -6,10 +6,18 @@ "qualifiers": null, "subpath": null, "primary_language": "Rust", - "description": null, + "description": "A simple to use, efficient, and full featured Command Line Argument Parser", "release_date": null, - "parties": {}, - "keywords": {}, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Kevin K.", + "email": "kbknapp@gmail.com", + "url": null + } + ], + "keywords": [], "homepage_url": null, "download_url": null, "size": null, @@ -25,11 +33,11 @@ "declared_license": null, "notice_text": null, "manifest_path": null, - "dependencies": {}, + "dependencies": [], "contains_source_code": null, - "source_packages": {}, + "source_packages": [], "purl": "pkg:cargo/clap@2.32.0", - "repository_homepage_url": "https://crates.io//crates/clap", - "repository_download_url": "https://crates.io/api/v1//crates/clap/2.32.0/download", - "api_data_url": "https://crates.io/api/v1//crates/clap" + "repository_homepage_url": "https://crates.io/crates/clap", + "repository_download_url": "https://crates.io/api/v1/crates/clap/2.32.0/download", + "api_data_url": "https://crates.io/api/v1/crates/clap" } diff --git a/tests/packagedcode/data/cargo/clippy/Cargo.toml.expected b/tests/packagedcode/data/cargo/clippy/Cargo.toml.expected index 7cd77746e0e..af95dc1f9f5 100644 --- a/tests/packagedcode/data/cargo/clippy/Cargo.toml.expected +++ b/tests/packagedcode/data/cargo/clippy/Cargo.toml.expected @@ -6,10 +6,46 @@ "qualifiers": null, "subpath": null, "primary_language": "Rust", - "description": null, + "description": "A bunch of helpful lints to avoid common pitfalls in Rust", "release_date": null, - "parties": {}, - "keywords": {}, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Manish Goregaokar", + "email": "manishsmail@gmail.com", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Andre Bogus", + "email": "bogusandre@gmail.com", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Georg Brandl", + "email": "georg@python.org", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Martin Carton", + "email": "cartonmartin@gmail.com", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Oliver Schneider", + "email": "clippy-iethah7aipeen8neex1a@oli-obk.de", + "url": null + } + ], + "keywords": [], "homepage_url": null, "download_url": null, "size": null, @@ -25,11 +61,11 @@ "declared_license": null, "notice_text": null, "manifest_path": null, - "dependencies": {}, + "dependencies": [], "contains_source_code": null, - "source_packages": {}, + "source_packages": [], "purl": "pkg:cargo/clippy@0.0.212", - "repository_homepage_url": "https://crates.io//crates/clippy", - "repository_download_url": "https://crates.io/api/v1//crates/clippy/0.0.212/download", - "api_data_url": "https://crates.io/api/v1//crates/clippy" + "repository_homepage_url": "https://crates.io/crates/clippy", + "repository_download_url": "https://crates.io/api/v1/crates/clippy/0.0.212/download", + "api_data_url": "https://crates.io/api/v1/crates/clippy" } diff --git a/tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected b/tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected index febed74eaf9..4dd0403b2d5 100644 --- a/tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected +++ b/tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected @@ -6,10 +6,32 @@ "qualifiers": null, "subpath": null, "primary_language": "Rust", - "description": null, + "description": "Create books from markdown files", "release_date": null, - "parties": {}, - "keywords": {}, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Mathieu David", + "email": "mathieudavid@mathieudavid.org", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Michael-F-Bryan", + "email": "michaelfbryan@gmail.com", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Matt Ickstadt", + "email": "mattico8@gmail.com", + "url": null + } + ], + "keywords": [], "homepage_url": null, "download_url": null, "size": null, @@ -25,11 +47,11 @@ "declared_license": null, "notice_text": null, "manifest_path": null, - "dependencies": {}, + "dependencies": [], "contains_source_code": null, - "source_packages": {}, + "source_packages": [], "purl": "pkg:cargo/mdbook@0.2.4-alpha.0", - "repository_homepage_url": "https://crates.io//crates/mdbook", - "repository_download_url": "https://crates.io/api/v1//crates/mdbook/0.2.4-alpha.0/download", - "api_data_url": "https://crates.io/api/v1//crates/mdbook" + "repository_homepage_url": "https://crates.io/crates/mdbook", + "repository_download_url": "https://crates.io/api/v1/crates/mdbook/0.2.4-alpha.0/download", + "api_data_url": "https://crates.io/api/v1/crates/mdbook" } diff --git a/tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected b/tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected index 91273dbe431..34676e86fe0 100644 --- a/tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected +++ b/tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected @@ -6,10 +6,25 @@ "qualifiers": null, "subpath": null, "primary_language": "Rust", - "description": null, + "description": "Tool to find and fix Rust formatting issues", "release_date": null, - "parties": {}, - "keywords": {}, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Nicholas Cameron", + "email": "ncameron@mozilla.com", + "url": null + }, + { + "type": "person", + "role": "author", + "name": "The Rustfmt developers", + "email": null, + "url": null + } + ], + "keywords": [], "homepage_url": null, "download_url": null, "size": null, @@ -25,11 +40,11 @@ "declared_license": null, "notice_text": null, "manifest_path": null, - "dependencies": {}, + "dependencies": [], "contains_source_code": null, - "source_packages": {}, + "source_packages": [], "purl": "pkg:cargo/rustfmt-nightly@1.0.3", - "repository_homepage_url": "https://crates.io//crates/rustfmt-nightly", - "repository_download_url": "https://crates.io/api/v1//crates/rustfmt-nightly/1.0.3/download", - "api_data_url": "https://crates.io/api/v1//crates/rustfmt-nightly" + "repository_homepage_url": "https://crates.io/crates/rustfmt-nightly", + "repository_download_url": "https://crates.io/api/v1/crates/rustfmt-nightly/1.0.3/download", + "api_data_url": "https://crates.io/api/v1/crates/rustfmt-nightly" } diff --git a/tests/packagedcode/data/cargo/rustup/Cargo.toml.expected b/tests/packagedcode/data/cargo/rustup/Cargo.toml.expected index 60edc410175..80e95ae24f0 100644 --- a/tests/packagedcode/data/cargo/rustup/Cargo.toml.expected +++ b/tests/packagedcode/data/cargo/rustup/Cargo.toml.expected @@ -6,10 +6,18 @@ "qualifiers": null, "subpath": null, "primary_language": "Rust", - "description": null, + "description": "Manage multiple rust installations with ease", "release_date": null, - "parties": {}, - "keywords": {}, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Diggory Blake", + "email": "diggsey@googlemail.com", + "url": null + } + ], + "keywords": [], "homepage_url": null, "download_url": null, "size": null, @@ -25,11 +33,11 @@ "declared_license": null, "notice_text": null, "manifest_path": null, - "dependencies": {}, + "dependencies": [], "contains_source_code": null, - "source_packages": {}, + "source_packages": [], "purl": "pkg:cargo/rustup@1.17.0", - "repository_homepage_url": "https://crates.io//crates/rustup", - "repository_download_url": "https://crates.io/api/v1//crates/rustup/1.17.0/download", - "api_data_url": "https://crates.io/api/v1//crates/rustup" + "repository_homepage_url": "https://crates.io/crates/rustup", + "repository_download_url": "https://crates.io/api/v1/crates/rustup/1.17.0/download", + "api_data_url": "https://crates.io/api/v1/crates/rustup" } diff --git a/tests/packagedcode/test_cargo.py b/tests/packagedcode/test_cargo.py index 741487b26c0..b9b0e3ab5ef 100644 --- a/tests/packagedcode/test_cargo.py +++ b/tests/packagedcode/test_cargo.py @@ -27,6 +27,7 @@ from __future__ import unicode_literals import os +import pytest from packagedcode import cargo @@ -65,3 +66,36 @@ def test_parse_rustup(self): expected_loc = self.get_test_loc('cargo/rustup/Cargo.toml.expected') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) + + +PERSON_PARSER_TEST_TABLE = [ + ('Barney Rubble ', ('Barney Rubble ', '')), + ('Barney Rubble', ('Barney Rubble', None)), + ('Some Good Guy ', ('Some Good Guy ', '')), + ('Some Good Guy', ('Some Good Guy', None)), +] + +PERSON_NO_NAME_PARSER_TEST_TABLE = [ + ('', (None, '')), + ('', (None, '')), +] + +class TestRegex(object): + @pytest.mark.parametrize('person, expected_person', PERSON_PARSER_TEST_TABLE) + def test_person_parser(self, person, expected_person): + parsed_person = cargo.person_parser(person) + person_information = parsed_person.groupdict() + name, email = person_information.get('name'), person_information.get('email') + assert (name, email) == expected_person + + def test_person_parser_no_name_failure(self): + person = '' + parsed_person = cargo.person_parser(person) + assert parsed_person is None + + @pytest.mark.parametrize('person, expected_person', PERSON_NO_NAME_PARSER_TEST_TABLE) + def test_person_parser_no_name(self, person, expected_person): + parsed_person = cargo.person_parser_no_name(person) + person_information = parsed_person.groupdict() + name, email = person_information.get('name'), person_information.get('email') + assert (name, email) == expected_person