Merge pull request #2748 from nexB/2098-top-level-packages

Add PackageManifest Class

Author: pombredanne

src/packagedcode/__init__.py

@@ -20,6 +20,7 @@
 from packagedcode import freebsd
 from packagedcode import golang
 from packagedcode import haxe
+from packagedcode import jar_manifest
 from packagedcode import maven
 from packagedcode import models
 from packagedcode import msi
@@ -38,73 +39,99 @@
 
 # Note: the order matters: from the most to the least specific
 # Package classes MUST be added to this list to be active
-PACKAGE_TYPES = [
-    rpm.RpmPackage,
+PACKAGE_MANIFEST_TYPES = [
+    rpm.RpmManifest,
     debian.DebianPackage,
 
     models.JavaJar,
+    jar_manifest.JavaManifest,
     models.JavaEar,
     models.JavaWar,
     maven.MavenPomPackage,
-    models.IvyJar,
+    jar_manifest.IvyJar,
     models.JBossSar,
     models.Axis2Mar,
 
-    about.AboutPackage,
-    npm.NpmPackage,
-    phpcomposer.PHPComposerPackage,
-    haxe.HaxePackage,
-    cargo.RustCargoCrate,
-    cocoapods.CocoapodsPackage,
-    opam.OpamPackage,
+    about.Aboutfile,
+    npm.PackageJson,
+    npm.PackageLockJson,
+    npm.YarnLockJson,
+    phpcomposer.ComposerJson,
+    phpcomposer.ComposerLock,
+    haxe.HaxelibJson,
+    cargo.CargoToml,
+    cargo.CargoLock,
+    cocoapods.Podspec,
+    cocoapods.PodfileLock,
+    cocoapods.PodspecJson,
+    opam.OpamFile,
     models.MeteorPackage,
-    bower.BowerPackage,
-    freebsd.FreeBSDPackage,
+    bower.BowerJson,
+    freebsd.CompactManifest,
     models.CpanModule,
-    rubygems.RubyGem,
+    rubygems.GemArchive,
+    rubygems.GemArchiveExtracted,
+    rubygems.GemSpec,
+    rubygems.GemfileLock,
     models.AndroidApp,
     models.AndroidLibrary,
     models.MozillaExtension,
     models.ChromeExtension,
     models.IOSApp,
-    pypi.PythonPackage,
-    golang.GolangPackage,
+    pypi.MetadataFile,
+    pypi.BinaryDistArchive,
+    pypi.SourceDistArchive,
+    pypi.SetupPy,
+    pypi.DependencyFile,
+    pypi.PipfileLock,
+    pypi.RequirementsFile,
+    golang.GoMod,
+    golang.GoSum,
     models.CabPackage,
     models.InstallShieldPackage,
     models.NSISInstallerPackage,
-    nuget.NugetPackage,
+    nuget.Nuspec,
     models.SharPackage,
     models.AppleDmgPackage,
     models.IsoImagePackage,
     models.SquashfsPackage,
-    chef.ChefPackage,
+    chef.MetadataJson,
+    chef.Metadatarb,
     build.BazelPackage,
     build.BuckPackage,
     build.AutotoolsPackage,
-    conda.CondaPackage,
-    win_pe.WindowsExecutable,
-    readme.ReadmePackage,
+    conda.Condayml,
+    win_pe.WindowsExecutableManifest,
+    readme.ReadmeManifest,
     build.MetadataBzl,
     msi.MsiInstallerPackage,
-    windows.MicrosoftUpdateManifestPackage,
-    pubspec.PubspecPackage,
+    windows.MicrosoftUpdateManifest,
+    pubspec.PubspecYaml,
+    pubspec.PubspecLock
 ]
 
-PACKAGES_BY_TYPE = {cls.default_type: cls for cls in PACKAGE_TYPES}
-
+PACKAGE_MANIFESTS_BY_TYPE = {
+    (
+        cls.package_manifest_type
+        if isinstance(cls, models.PackageManifest)
+        else cls.default_type
+    ): cls
+    for cls in PACKAGE_MANIFEST_TYPES
+}
 # We cannot have two package classes with the same type
-if len(PACKAGES_BY_TYPE) != len(PACKAGE_TYPES):
+if len(PACKAGE_MANIFESTS_BY_TYPE) != len(PACKAGE_MANIFEST_TYPES):
     seen_types = {}
-    for pt in PACKAGE_TYPES:
-        assert pt.default_type
-        seen = seen_types.get(pt.default_type)
+    for pmt in PACKAGE_MANIFEST_TYPES:
+        manifest = pmt()
+        assert manifest.package_manifest_type
+        seen = seen_types.get(manifest.package_manifest_type)
         if seen:
             msg = ('Invalid duplicated packagedcode.Package types: '
                    '"{}:{}" and "{}:{}" have the same type.'
-                  .format(pt.default_type, pt.__name__, seen.default_type, seen.__name__,))
+                  .format(manifest.package_manifest_type, manifest.__name__, seen.package_manifest_type, seen.__name__,))
             raise Exception(msg)
         else:
-            seen_types[pt.default_type] = pt
+            seen_types[manifest.package_manifest_type] = manifest
 
 
 def get_package_class(scan_data, default=models.Package):
@@ -130,7 +157,7 @@ def get_package_class(scan_data, default=models.Package):
     if not ptype:
         # basic type for default package types
         return default
-    ptype_class = PACKAGES_BY_TYPE.get(ptype)
+    ptype_class = PACKAGE_MANIFESTS_BY_TYPE.get(ptype)
     return ptype_class or default
 
 

src/packagedcode/about.py

@@ -30,12 +30,8 @@
 
 @attr.s()
 class AboutPackage(models.Package):
-    metafiles = ('*.ABOUT',)
-    default_type = 'about'
 
-    @classmethod
-    def recognize(cls, location):
-        yield parse(location)
+    default_type = 'about'
 
     def get_package_root(self, manifest_resource, codebase):
         about_resource = self.extra_data.get('about_resource')
@@ -47,53 +43,54 @@ def get_package_root(self, manifest_resource, codebase):
         return manifest_resource
 
 
-def is_about_file(location):
-    return (filetype.is_file(location)
-            and location.lower().endswith(('.about',)))
-
-
-def parse(location):
-    """
-    Return a Package object from an ABOUT file or None.
-    """
-    if not is_about_file(location):
-        return
-
-    with io.open(location, encoding='utf-8') as loc:
-        package_data = saneyaml.load(loc.read())
-
-    return build_package(package_data)
-
-
-def build_package(package_data):
-    """
-    Return a Package built from `package_data` obtained by an ABOUT file.
-    """
-    name = package_data.get('name')
-    # FIXME: having no name may not be a problem See #1514
-    if not name:
-        return
-
-    version = package_data.get('version')
-    homepage_url = package_data.get('home_url') or package_data.get('homepage_url')
-    download_url = package_data.get('download_url')
-    declared_license = package_data.get('license_expression')
-    copyright_statement = package_data.get('copyright')
-
-    owner = package_data.get('owner')
-    if not isinstance(owner, str):
-        owner = repr(owner)
-    parties = [models.Party(type=models.party_person, name=owner, role='owner')]
-
-    about_package = AboutPackage(
-        type='about',
-        name=name,
-        version=version,
-        declared_license=declared_license,
-        copyright=copyright_statement,
-        parties=parties,
-        homepage_url=homepage_url,
-        download_url=download_url,
-    )
-    about_package.extra_data['about_resource'] = package_data.get('about_resource')
-    return about_package
+@attr.s()
+class Aboutfile(AboutPackage, models.PackageManifest):
+
+    file_patterns = ('*.ABOUT',)
+    extensions = ('.ABOUT',)
+
+    @classmethod
+    def is_manifest(cls, location):
+        """
+        Return True if the file at ``location`` is likely a manifest of this type.
+        """
+        return filetype.is_file(location) and location.lower().endswith(('.about',))
+
+    @classmethod
+    def recognize(cls, location):
+        """
+        Yield one or more Package manifest objects given a file ``location`` pointing to a
+        package archive, manifest or similar.
+        """
+        with io.open(location, encoding='utf-8') as loc:
+            package_data = saneyaml.load(loc.read())
+
+        name = package_data.get('name')
+        # FIXME: having no name may not be a problem See #1514
+        if not name:
+            return
+
+        version = package_data.get('version')
+        homepage_url = package_data.get('home_url') or package_data.get('homepage_url')
+        download_url = package_data.get('download_url')
+        declared_license = package_data.get('license_expression')
+        copyright_statement = package_data.get('copyright')
+
+        owner = package_data.get('owner')
+        if not isinstance(owner, str):
+            owner = repr(owner)
+        parties = [models.Party(type=models.party_person, name=owner, role='owner')]
+
+        about_package = cls(
+            type='about',
+            name=name,
+            version=version,
+            declared_license=declared_license,
+            copyright=copyright_statement,
+            parties=parties,
+            homepage_url=homepage_url,
+            download_url=download_url,
+        )
+
+        about_package.extra_data['about_resource'] = package_data.get('about_resource')
+        yield about_package

src/packagedcode/alpine.py

@@ -27,7 +27,7 @@
 
 
 @attr.s()
-class AlpinePackage(models.Package):
+class AlpinePackage(models.Package, models.PackageManifest):
     extensions = ('.apk', 'APKBUILD')
     default_type = 'alpine'