Prefer using PKG-INFO from .egg-info in assemble #3083

    * Add test for checking that the .egg-info PKG-INFO is the only Package source reported
    * Update test expectations

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

.gitignore

@@ -11,6 +11,7 @@
 /src/*.egg-info
 *.egg-info
 !tests/packagedcode/data/pypi/source-package/pip-22.0.4/src/pip.egg-info
+!tests/packagedcode/data/pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery/celery.egg-info
 /dist
 /build
 /bin

src/packagedcode/pypi.py

@@ -152,7 +152,36 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
         package_resource = None
         if resource.name == 'PKG-INFO':
+            # Initially use current Resource as `package_resource`.
+            # We'll want update `package_resource` with the Resource of a
+            # PKG-INFO file that's in an .egg-info Directory.
             package_resource = resource
+            # We want to use the PKG-INFO file from an .egg-info directory, as
+            # the package info collected from a *.egg_info/PKG-INFO file has
+            # dependency information that a PKG-INFO from the root of a Python
+            # project lacks.
+            parent_resource = resource.parent(codebase)
+            if (
+                parent_resource
+                and not parent_resource.path.endswith('.egg-info')
+            ):
+                # If we are not in an .egg-info directory, we assume we are at
+                # the root of a Python codebase and we want to find the
+                # .egg_info dir
+                egg_info_dir = None
+                for sibling in resource.siblings(codebase):
+                    if sibling.path.endswith('.egg-info'):
+                        egg_info_dir = sibling
+                        break
+
+                # If we find the .egg_info dir, then we look for the PKG-INFO
+                # file in it and use that as our package_resource
+                if egg_info_dir:
+                    for child in egg_info_dir.children(codebase):
+                        if not child.name == 'PKG-INFO':
+                            continue
+                        package_resource = child
+                        break
         elif resource.name in datafile_name_patterns:
             if resource.has_parent():
                 siblings = resource.siblings(codebase)
@@ -221,7 +250,17 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 package.license_expression = compute_normalized_license(package.declared_license)
             package_uid = package.package_uid
 
-            root = package_resource.parent(codebase)
+            package_resource_parent = package_resource.parent(codebase)
+            if (
+                package_resource_parent
+                and package_resource_parent.path.endswith('.egg-info')
+            ):
+                root = package_resource_parent.parent(codebase)
+            else:
+                # We're assuming that our package resource is already at the
+                # root
+                root = package_resource_parent
+
             if root:
                 for py_res in cls.walk_pypi(resource=root, codebase=codebase):
                     if py_res.is_dir:

tests/packagedcode/data/about/aboutfiles.expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 3
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

tests/packagedcode/data/build/bazel/end2end-expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json-pp": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 6
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

tests/packagedcode/data/build/buck/end2end-expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json-pp": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 7
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

tests/packagedcode/data/cocoapods/assemble/solo/Podfile-expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [],
   "files": [

tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources.podspec-expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

tests/packagedcode/data/npm/get_package_resources.scan.expected.json

@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 3
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {