Yield Packages before other yieldables #3028

    * Update assembly method of cocoapods and pypi package handlers to yield Package objects before Resources and Dependencies

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/packagedcode/alpine.py

@@ -92,6 +92,7 @@ def assemble(cls, package_data, resource, codebase):
             for ref in package.file_references
         }
 
+        resources = []
         for res in root_resource.walk(codebase):
             ref = file_references_by_path.get(res.path)
             if not ref:
@@ -102,15 +103,15 @@ def assemble(cls, package_data, resource, codebase):
             del file_references_by_path[res.path]
             res.for_packages.append(package_uid)
             res.save(codebase)
-
-            yield res
+            resources.append(res)
 
         # if we have left over file references, add these to extra data
         if file_references_by_path:
             missing = sorted(file_references_by_path.values(), key=lambda r:r.path)
             package.extra_data['missing_file_references'] = missing
 
         yield package
+        yield from resources
 
 
 class AlpineApkbuildHandler(models.DatafileHandler):

src/packagedcode/cocoapods.py

@@ -164,7 +164,6 @@ def assemble(cls, package_data, resource, codebase):
 
                 for resource in sibling_podspecs:
                     datafile_path = resource.path
-                    yield resource
                     for package_data in resource.package_data:
                         package_data = models.PackageData.from_dict(package_data)
                         package = models.Package.from_package_data(
@@ -173,6 +172,7 @@ def assemble(cls, package_data, resource, codebase):
                         )
                         cls.assign_package_to_resources(package, resource, codebase)
                         yield package
+                    yield resource
 
             else:
                 # has_no_podspec:

src/packagedcode/debian.py

@@ -279,6 +279,7 @@ def assemble(cls, package_data, resource, codebase):
 
             f'usr/share/doc/{package_name}/copyright',
         ]))
+        resources = []
         # TODO: keep track of missing files
         for res in root_resource.walk(codebase):
             if not res.path.endswith(assemblable_paths):
@@ -305,8 +306,7 @@ def assemble(cls, package_data, resource, codebase):
                     package_uid=package_uid,
                 )
 
-            # we yield this as we do not want this further processed
-            yield res
+            resources.append(res)
 
         root_path = Path(root_resource.path)
 
@@ -332,14 +332,15 @@ def assemble(cls, package_data, resource, codebase):
             res.for_packages.append(package_uid)
             res.save(codebase)
 
-            yield res
+            resources.append(res)
 
         # if we have left over file references, add these to extra data
         if file_references_by_path:
             missing = sorted(file_references_by_path.values(), key=lambda r:r.path)
             package.extra_data['missing_file_references'] = missing
 
         yield package
+        yield from resources
 
 
 class DebianDistrolessInstalledDatabaseHandler(models.DatafileHandler):
@@ -383,38 +384,41 @@ def assemble(cls, package_data, resource, codebase):
         )
         package_uid = package.package_uid
 
-        dependent_packages = package_data.dependencies
-        if dependent_packages:
-            yield from models.Dependency.from_dependent_packages(
-                dependent_packages=dependent_packages,
-                datafile_path=resource.path,
-                datasource_id=package_data.datasource_id,
-                package_uid=package_uid,
-            )
-
         # collect copyright file for this package
         # and merge in package data
         assemblable_paths = (
             f'usr/share/doc/{package_name}/copyright',
         )
+        resources = []
         if package_uid:
             for res in root_resource.walk(codebase):
                 if not res.path.endswith(assemblable_paths):
                     continue
-    
+
                 for pkgdt in res.package_data:
                     package.update(
                         package_data=pkgdt,
                         datafile_path=res.path,
                     )
-    
+
                 res.for_packages.append(package_uid)
                 res.save(codebase)
-    
-                yield res
+
+                resources.append(res)
 
         yield package
 
+        dependent_packages = package_data.dependencies
+        if dependent_packages:
+            yield from models.Dependency.from_dependent_packages(
+                dependent_packages=dependent_packages,
+                datafile_path=resource.path,
+                datasource_id=package_data.datasource_id,
+                package_uid=package_uid,
+            )
+
+        yield from resources
+
 
 class DebianInstalledFilelistHandler(models.DatafileHandler):
     # seen in installed rootfs in:

src/packagedcode/pypi.py

@@ -120,6 +120,16 @@ def assign_package_to_resources(cls, package, resource, codebase):
         return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase)
 
 
+def create_package_from_package_data(package_data, datafile_path):
+    package = models.Package.from_package_data(
+        package_data=package_data,
+        datafile_path=datafile_path,
+    )
+    if not package.license_expression:
+        package.license_expression = compute_normalized_license(package.declared_license)
+    return package
+
+
 class BaseExtractedPythonLayout(BasePypiHandler):
     """
     Base class for development repos, sdist tarballs and other related extracted
@@ -152,10 +162,12 @@ def assemble(cls, package_data, resource, codebase):
             pkg_data = package_resource.package_data[0]
             pkg_data = models.PackageData.from_dict(pkg_data)
             if pkg_data.purl:
-                package = models.Package.from_package_data(
+                package = create_package_from_package_data(
                     package_data=pkg_data,
-                    datafile_path=package_resource.path,
+                    datafile_path=package_resource.path
                 )
+                yield package
+
                 package_resource.for_packages.append(package.package_uid)
                 package_resource.save(codebase)
                 yield package_resource
@@ -183,10 +195,11 @@ def assemble(cls, package_data, resource, codebase):
                 for setup_resource, setup_pkg_data in setup_package_data:
                     if setup_pkg_data.purl:
                         if not package:
-                            package = models.Package.from_package_data(
+                            package = create_package_from_package_data(
                                 package_data=setup_pkg_data,
-                                datafile_path=setup_resource.path,
+                                datafile_path=setup_resource.path
                             )
+                            yield package
                             package_resource = setup_resource
                         else:
                             package.update(setup_pkg_data, setup_resource.path)
@@ -203,8 +216,6 @@ def assemble(cls, package_data, resource, codebase):
                         )
 
         if package:
-            if not package.license_expression:
-                package.license_expression = compute_normalized_license(package.declared_license)
             package_uid = package.package_uid
 
             root = package_resource.parent(codebase)
@@ -221,8 +232,6 @@ def assemble(cls, package_data, resource, codebase):
                     package_resource.for_packages.append(package_uid)
                     package_resource.save(codebase)
 
-            yield package
-
         else:
             package_uid = None
 

src/packagedcode/rpm.py

@@ -188,20 +188,8 @@ def assemble(cls, package_data, resource, codebase):
         _declared, detected = detect_declared_license(package.declared_license)
         package.license_expression = detected
 
-        # yield deps
-        dependent_packages = package_data.dependencies
-        if dependent_packages:
-            for dep in models.Dependency.from_dependent_packages(
-                dependent_packages=dependent_packages,
-                datafile_path=resource.path,
-                datasource_id=package_data.datasource_id,
-                package_uid=package_uid,
-            ):
-                if not dep.namespace:
-                    dep.namespace = namespace
-                yield dep
-
         # tag files from refs
+        resources = []
         missing_file_references = []
         # a file ref extends from the root of the filesystem
         for ref in package.file_references:
@@ -215,6 +203,7 @@ def assemble(cls, package_data, resource, codebase):
                     # found all of them
                     res.for_packages.append(package_uid)
                     res.save(codebase)
+                    resources.append(res)
 
         # if we have left over file references, add these to extra data
         if missing_file_references:
@@ -223,6 +212,21 @@ def assemble(cls, package_data, resource, codebase):
 
         yield package
 
+        # yield deps
+        dependent_packages = package_data.dependencies
+        if dependent_packages:
+            for dep in models.Dependency.from_dependent_packages(
+                dependent_packages=dependent_packages,
+                datafile_path=resource.path,
+                datasource_id=package_data.datasource_id,
+                package_uid=package_uid,
+            ):
+                if not dep.namespace:
+                    dep.namespace = namespace
+                yield dep
+
+        yield from resources
+
 
 # TODO: add dependencies!!!
 class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):