diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py index fa6287598d4..b1ff3fed735 100644 --- a/src/packagedcode/alpine.py +++ b/src/packagedcode/alpine.py @@ -92,6 +92,7 @@ def assemble(cls, package_data, resource, codebase): for ref in package.file_references } + resources = [] for res in root_resource.walk(codebase): ref = file_references_by_path.get(res.path) if not ref: @@ -102,8 +103,7 @@ def assemble(cls, package_data, resource, codebase): del file_references_by_path[res.path] res.for_packages.append(package_uid) res.save(codebase) - - yield res + resources.append(res) # if we have left over file references, add these to extra data if file_references_by_path: @@ -111,6 +111,7 @@ def assemble(cls, package_data, resource, codebase): package.extra_data['missing_file_references'] = missing yield package + yield from resources class AlpineApkbuildHandler(models.DatafileHandler): diff --git a/src/packagedcode/cocoapods.py b/src/packagedcode/cocoapods.py index 3552e2bae5c..22a9c5a959b 100644 --- a/src/packagedcode/cocoapods.py +++ b/src/packagedcode/cocoapods.py @@ -164,7 +164,6 @@ def assemble(cls, package_data, resource, codebase): for resource in sibling_podspecs: datafile_path = resource.path - yield resource for package_data in resource.package_data: package_data = models.PackageData.from_dict(package_data) package = models.Package.from_package_data( @@ -173,6 +172,7 @@ def assemble(cls, package_data, resource, codebase): ) cls.assign_package_to_resources(package, resource, codebase) yield package + yield resource else: # has_no_podspec: diff --git a/src/packagedcode/debian.py b/src/packagedcode/debian.py index 77e7cc006be..b51a69ac2ae 100644 --- a/src/packagedcode/debian.py +++ b/src/packagedcode/debian.py @@ -279,6 +279,7 @@ def assemble(cls, package_data, resource, codebase): f'usr/share/doc/{package_name}/copyright', ])) + resources = [] # TODO: keep track of missing files for res in root_resource.walk(codebase): if not res.path.endswith(assemblable_paths): @@ -305,8 +306,7 @@ def assemble(cls, package_data, resource, codebase): package_uid=package_uid, ) - # we yield this as we do not want this further processed - yield res + resources.append(res) root_path = Path(root_resource.path) @@ -332,7 +332,7 @@ def assemble(cls, package_data, resource, codebase): res.for_packages.append(package_uid) res.save(codebase) - yield res + resources.append(res) # if we have left over file references, add these to extra data if file_references_by_path: @@ -340,6 +340,7 @@ def assemble(cls, package_data, resource, codebase): package.extra_data['missing_file_references'] = missing yield package + yield from resources class DebianDistrolessInstalledDatabaseHandler(models.DatafileHandler): @@ -383,38 +384,41 @@ def assemble(cls, package_data, resource, codebase): ) package_uid = package.package_uid - dependent_packages = package_data.dependencies - if dependent_packages: - yield from models.Dependency.from_dependent_packages( - dependent_packages=dependent_packages, - datafile_path=resource.path, - datasource_id=package_data.datasource_id, - package_uid=package_uid, - ) - # collect copyright file for this package # and merge in package data assemblable_paths = ( f'usr/share/doc/{package_name}/copyright', ) + resources = [] if package_uid: for res in root_resource.walk(codebase): if not res.path.endswith(assemblable_paths): continue - + for pkgdt in res.package_data: package.update( package_data=pkgdt, datafile_path=res.path, ) - + res.for_packages.append(package_uid) res.save(codebase) - - yield res + + resources.append(res) yield package + dependent_packages = package_data.dependencies + if dependent_packages: + yield from models.Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=resource.path, + datasource_id=package_data.datasource_id, + package_uid=package_uid, + ) + + yield from resources + class DebianInstalledFilelistHandler(models.DatafileHandler): # seen in installed rootfs in: diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index e1c6a04492b..8a359b657c3 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -120,6 +120,16 @@ def assign_package_to_resources(cls, package, resource, codebase): return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase) +def create_package_from_package_data(package_data, datafile_path): + package = models.Package.from_package_data( + package_data=package_data, + datafile_path=datafile_path, + ) + if not package.license_expression: + package.license_expression = compute_normalized_license(package.declared_license) + return package + + class BaseExtractedPythonLayout(BasePypiHandler): """ Base class for development repos, sdist tarballs and other related extracted @@ -152,10 +162,12 @@ def assemble(cls, package_data, resource, codebase): pkg_data = package_resource.package_data[0] pkg_data = models.PackageData.from_dict(pkg_data) if pkg_data.purl: - package = models.Package.from_package_data( + package = create_package_from_package_data( package_data=pkg_data, - datafile_path=package_resource.path, + datafile_path=package_resource.path ) + yield package + package_resource.for_packages.append(package.package_uid) package_resource.save(codebase) yield package_resource @@ -183,10 +195,11 @@ def assemble(cls, package_data, resource, codebase): for setup_resource, setup_pkg_data in setup_package_data: if setup_pkg_data.purl: if not package: - package = models.Package.from_package_data( + package = create_package_from_package_data( package_data=setup_pkg_data, - datafile_path=setup_resource.path, + datafile_path=setup_resource.path ) + yield package package_resource = setup_resource else: package.update(setup_pkg_data, setup_resource.path) @@ -203,8 +216,6 @@ def assemble(cls, package_data, resource, codebase): ) if package: - if not package.license_expression: - package.license_expression = compute_normalized_license(package.declared_license) package_uid = package.package_uid root = package_resource.parent(codebase) @@ -221,8 +232,6 @@ def assemble(cls, package_data, resource, codebase): package_resource.for_packages.append(package_uid) package_resource.save(codebase) - yield package - else: package_uid = None diff --git a/src/packagedcode/rpm.py b/src/packagedcode/rpm.py index 4fbb345bff9..645c2b0d720 100644 --- a/src/packagedcode/rpm.py +++ b/src/packagedcode/rpm.py @@ -188,20 +188,8 @@ def assemble(cls, package_data, resource, codebase): _declared, detected = detect_declared_license(package.declared_license) package.license_expression = detected - # yield deps - dependent_packages = package_data.dependencies - if dependent_packages: - for dep in models.Dependency.from_dependent_packages( - dependent_packages=dependent_packages, - datafile_path=resource.path, - datasource_id=package_data.datasource_id, - package_uid=package_uid, - ): - if not dep.namespace: - dep.namespace = namespace - yield dep - # tag files from refs + resources = [] missing_file_references = [] # a file ref extends from the root of the filesystem for ref in package.file_references: @@ -215,6 +203,7 @@ def assemble(cls, package_data, resource, codebase): # found all of them res.for_packages.append(package_uid) res.save(codebase) + resources.append(res) # if we have left over file references, add these to extra data if missing_file_references: @@ -223,6 +212,21 @@ def assemble(cls, package_data, resource, codebase): yield package + # yield deps + dependent_packages = package_data.dependencies + if dependent_packages: + for dep in models.Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=resource.path, + datasource_id=package_data.datasource_id, + package_uid=package_uid, + ): + if not dep.namespace: + dep.namespace = namespace + yield dep + + yield from resources + # TODO: add dependencies!!! class RpmInstalledNdbDatabaseHandler(BaseRpmInstalledDatabaseHandler):