Fix alpine cocoapods go maven rpm win tests

Only top-level packages tests remaining to fix, other than
a few quirks.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/alpine.py

@@ -899,7 +899,7 @@ def m_maintainer_handler(value, **kwargs):
             name=name,
             email=email,
         )
-        parties.append(maintainer)
+        parties.append(maintainer.to_dict())
     return {'parties': parties}
 
 
@@ -995,7 +995,7 @@ def D_dependencies_handler(value, dependencies=None, **kwargs):
             is_resolved=is_resolved,
         )
         if dependency not in dependencies:
-            dependencies.append(dependency)
+            dependencies.append(dependency.to_dict())
 
     return {'dependencies': dependencies}
 
@@ -1047,7 +1047,7 @@ def R_filename_handler(value, current_dir, file_references=None, **kwargs):
     file_references = file_references[:] if file_references else []
 
     current_file = models.FileReference(path=posixpath.join(current_dir, value))
-    file_references.append(current_file)
+    file_references.append(current_file.to_dict())
     return {'current_file': current_file, 'file_references': file_references}
 
 

src/packagedcode/cocoapods.py

@@ -111,7 +111,7 @@ def parse(cls, location):
 
         name = podspec.get('name')
         version = podspec.get('version')
-        homepage_url = podspec.get('homepage_url')
+        homepage_url = podspec.get('homepage')
         declared_license = podspec.get('license')
         description = utils.build_description(
             summary=podspec.get('summary'),
@@ -380,7 +380,7 @@ def party_mapper(author, email):
 
 person_parser = re.compile(
     r'^(?P<name>[\w\s(),-_.,]+)'
-    r'=>'
+    r'[\s]=[>\s]?'
     r'(?P<email>[\S+]+$)'
 ).match
 

src/packagedcode/go_mod.py

@@ -236,7 +236,7 @@ def parse_gosum(location):
     gosums = []
 
     for line in lines:
-        line = line.replace('*/go.mod', '')
+        line = line.replace('/go.mod', '')
         parsed_dep = get_dependency(line)
 
         ns_name = parsed_dep.group('ns_name')

src/packagedcode/godeps.py

@@ -25,7 +25,6 @@
 modules.
 """
 # FIXME: update to use the latest vendor conventions.
-# TODO: Use attrs
 # consider other legacy format?
 # https://github.com/golang/dep/blob/master/Gopkg.lock
 # https://github.com/golang/dep/blob/master/Gopkg.toml
@@ -81,7 +80,7 @@ def assign_package_to_resources(cls, package, resource, codebase):
 
 
 @attr.s
-class Dep(namedtuple('Dep', 'import_path revision comment')):
+class Dep:
     import_path = datautils.String()
     revision = datautils.String()
     comment = datautils.String()

src/packagedcode/maven.py

@@ -72,7 +72,7 @@ def is_datafile(cls, location, filetypes=tuple()):
         Return True if the file at location is highly likely to be a POM.
         """
         if super().is_datafile(location, filetypes=filetypes):
-            return
+            return True
 
         T = contenttype.get_type(location)
         if not T.is_text:
@@ -95,7 +95,7 @@ def is_datafile(cls, location, filetypes=tuple()):
                     return True
 
     @classmethod
-    def parse(cls, location, base_url='http://repo1.maven.org/maven2'):
+    def parse(cls, location, base_url='https://repo1.maven.org/maven2'):
         return parse(
             location=location,
             datasource_id=cls.datasource_id,
@@ -242,7 +242,7 @@ def build_url(
     artifact_id,
     version,
     filename=None,
-    base_url='http://repo1.maven.org/maven2',
+    base_url='https://repo1.maven.org/maven2',
 ):
     """
     Return a download URL for a Maven artifact built from its POM "coordinates".
@@ -302,11 +302,11 @@ def to_dict(self):
         Return a mapping representing this POM
         """
         return {
-            'group_id', self.group_id,
-            'artifact_id', self.artifact_id,
-            'version', str(self.version) if self.version else None,
-            'classifier', self.classifier,
-            'type', self.type,
+            'group_id': self.group_id,
+            'artifact_id': self.artifact_id,
+            'version': str(self.version) if self.version else None,
+            'classifier': self.classifier,
+            'type': self.type,
         }
 
 
@@ -908,7 +908,7 @@ def get_maven_pom(location=None):
     if not hbpa:
         if TRACE:
             logger.debug(f'get_maven_pom: has_basic_pom_attributes: {hbpa}')
-        return {}
+        return
     return pom
 
 
@@ -1033,7 +1033,7 @@ def get_parties(pom):
     return parties
 
 
-def get_urls(namespace, name, version, qualifiers, base_url='http://repo1.maven.org/maven2'):
+def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven.org/maven2'):
     """
     Return a mapping of URLs.
     """
@@ -1089,7 +1089,7 @@ def parse(
     datasource_id,
     package_type,
     primary_language,
-    base_url='http://repo1.maven.org/maven2',
+    base_url='https://repo1.maven.org/maven2',
 ):
     """
     Yield Packagedata objects from parsing a Maven pom file at `location` or
@@ -1153,7 +1153,6 @@ def parse(
 
     scm = pom.scm or {}
     urls = build_vcs_and_code_view_urls(scm)
-    print(urls)
     urls.update(get_urls(
         namespace=group_id,
         name=artifact_id, version=version,
@@ -1206,7 +1205,7 @@ def build_vcs_and_code_view_urls(scm):
         if code_view_url:
             # we can craft a vcs_url in some cases
             vcs_url = code_view_url
-        return vcs_url, code_view_url
+        return dict(vcs_url=vcs_url, code_view_url=code_view_url,)
 
     vcs_url = parse_scm_connection(vcs_url)
 

src/packagedcode/rpm.py

@@ -126,9 +126,11 @@ class BaseRpmInstalledDatabaseHandler(models.DatafileHandler):
     def parse(cls, location):
         # dump and parse the rpmdb to XMLish
         xmlish_loc = rpm_installed.collect_installed_rpmdb_xmlish_from_rpmdb_loc(location)
-        package_data = rpm_installed.parse_rpm_xmlish(xmlish_loc)
-        package_data.datasource_id = cls.datasource_id
-        package_data.type = cls.default_package_type
+        package_data = rpm_installed.parse_rpm_xmlish(
+            location=xmlish_loc,
+            datasource_id=cls.datasource_id,
+            package_type=cls.default_package_type,
+        )
         # TODO: package_data.namespace = cls.default_package_namespace
         return package_data
 

src/packagedcode/rpm_installed.py

@@ -35,7 +35,7 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
-def parse_rpm_xmlish(location):
+def parse_rpm_xmlish(location, datasource_id, package_type):
     """
     Yield RpmPackage(s) from a RPM XML'ish file at `location`. This is a file
     created with the rpm CLI with the xml query option.
@@ -53,7 +53,7 @@ def parse_rpm_xmlish(location):
 
     for rpm_raw_tags in collect_rpms(rpms):
         tags = collect_tags(rpm_raw_tags)
-        yield build_package(tags)
+        yield build_package(tags, datasource_id, package_type)
 
 
 def collect_rpms(text):
@@ -128,14 +128,19 @@ def collect_tags(raw_tags):
         yield name, value_type, value
 
 
-def build_package(rpm_tags, datasource_id, package_type, package_namespace):
+def build_package(rpm_tags, datasource_id, package_type, package_namespace=None):
     """
     Return a PackageData object from an ``rpm_tags`` iterable of (name,
     value_type, value) tuples.
     """
 
     # mapping of real Package field name -> value converted to expected format
-    converted = {}
+    converted = {
+        'datasource_id': datasource_id,
+        'type': package_type,
+        'namespace': package_namespace
+    }
+
     for name, value_type, value in rpm_tags:
         handler = RPM_TAG_HANDLER_BY_NAME.get(name)
         # FIXME: we need to handle EVRA correctly

src/packagedcode/rubygems.py

@@ -143,7 +143,8 @@ def parse(cls, location):
 
         name = gemspec.get('name')
         version = gemspec.get('version')
-        homepage_url = gemspec.get('homepage_url')
+        homepage_url = gemspec.get('homepage')
+
         description = build_description(
             summary=gemspec.get('summary'),
             description=gemspec.get('description'),
@@ -158,6 +159,8 @@ def parse(cls, location):
         parties = get_parties(gemspec)
         dependencies = gemspec.get('dependencies') or []
 
+        urls = get_urls(name=name, version=version)
+
         package_data = models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -168,7 +171,8 @@ def parse(cls, location):
             description=description,
             declared_license=declared_license,
             primary_language=cls.default_primary_language,
-            dependencies=dependencies
+            dependencies=dependencies,
+            **urls
         )
 
         yield package_data
@@ -285,14 +289,16 @@ def parse(cls, location):
                         is_resolved=True,
                     )
                 )
+            urls = get_urls(gem.name, gem.version)
 
             yield models.PackageData(
                 datasource_id=cls.datasource_id,
                 primary_language=cls.default_primary_language,
                 type=cls.default_package_type,
                 name=gem.name,
                 version=gem.version,
-                dependencies=deps
+                dependencies=deps,
+                **urls
             )
 
 
@@ -487,7 +493,7 @@ def build_rubygem_package_data(gem_data, datasource_id):
     package_data = models.PackageData(
         datasource_id=datasource_id,
         type=GemArchiveHandler.default_package_type,
-        primary_languages=GemArchiveHandler.default_primary_language,
+        primary_language=GemArchiveHandler.default_primary_language,
         name=name,
         version=version,
         qualifiers=qualifiers,
@@ -529,7 +535,7 @@ def build_rubygem_package_data(gem_data, datasource_id):
     #   "documentation_uri" => "https://www.example.info/gems/bestgemever/0.0.1",
 
     if not package_data.homepage_url:
-        package_data.homepage_url = package_data.repository_homepage_url()
+        package_data.homepage_url = rubygems_homepage_url(name, version)
 
     return package_data
 
@@ -709,14 +715,20 @@ def get_dependencies(dependencies):
 }
 
 
-def party_mapper(names, role):
+def party_mapper(role, names=[], emails=[]):
     """
     Yields Party with ``role`` objects from a ``names`` list of string.
     """
-    return (
-        models.Party(type=models.party_person, name=name, role=role)
-        for name in names
-    )
+    if names:
+        return (
+            models.Party(type=models.party_person, name=name, role=role)
+            for name in names
+        )
+    elif emails:
+        return (
+            models.Party(type=models.party_person, email=email, role=role)
+            for email in emails
+        )
 
 
 def get_parties(gem_data):
@@ -725,10 +737,10 @@ def get_parties(gem_data):
     """
     parties = []
     authors = gem_data.get('author') or []
-    parties.extend(party_mapper(authors, role='author'))
+    parties.extend(party_mapper(names=authors, role='author'))
     # FIXME: emails is NOT a party
     emails = gem_data.get('email') or []
-    parties.extend(party_mapper(emails, role='email'))
+    parties.extend(party_mapper(emails=emails, role='author'))
     return parties
 
 

src/packagedcode/spec.py

@@ -69,12 +69,12 @@ def get_source(line):
 
 # mapping of parser callable by its field name
 PARSER_BY_NAME = {
-    'name': partial(get_value, name='name', match=parse_name),
-    'version': partial(get_value, name='version', match=parse_version),
-    'license': partial(get_value, name='license', match=parse_license),
-    'summary': partial(get_value, name='summary', match=parse_summary),
-    'description': partial(get_value, name='description', match=parse_description),
-    'homepage': partial(get_value, name='homepage', match=parse_homepage),
+    'name': partial(get_value, name='name', matcher=parse_name),
+    'version': partial(get_value, name='version', matcher=parse_version),
+    'license': partial(get_value, name='license', matcher=parse_license),
+    'summary': partial(get_value, name='summary', matcher=parse_summary),
+    'description': partial(get_value, name='description', matcher=parse_description),
+    'homepage': partial(get_value, name='homepage', matcher=parse_homepage),
     'source': get_source,
 }
 
@@ -92,11 +92,11 @@ def parse_spec(location, package_type):
             line = pre_process(line)
 
             for attribute_name, parser in PARSER_BY_NAME.items():
-                parsed = parser(line)
+                parsed = parser(line=line)
                 if parsed:
                     spec_data[attribute_name] = parsed
 
-            match = parse_description(line)
+            match = parse_description.match(line)
             if match:
                 if location.endswith('.gemspec'):
                     # FIXME: description can be in single or multi-lines
@@ -146,14 +146,15 @@ def get_dependent_packages(location, package_type):
         }
 
     dependencies = GemfileParser(location).parse()
-    for deps in dependencies.values():
-        for dep in deps:
-            scope = dep['scope']
-            flags = flags_by_scope.get(scope, 'runtime')
+
+    for key in dependencies:
+        depends = dependencies.get(key, []) or []
+        for dep in depends:
+            flags = flags_by_scope.get(key, 'runtime')
             yield models.DependentPackage(
-                purl=PackageURL(type=package_type, name=dep['name']),
-                extracted_requirement=', '.join(dep['requirement']),
-                scope=scope,
+                purl=PackageURL(type=package_type, name=dep.name).to_string(),
+                extracted_requirement=', '.join(dep.requirement),
+                scope=key,
                 is_resolved=False,
                 **flags,
             )