aboutcode-org · JonoYang · Sep 6, 2022 · Sep 3, 2022 · Sep 6, 2022 · Sep 6, 2022
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,7 @@
 /src/*.egg-info
 *.egg-info
 !tests/packagedcode/data/pypi/source-package/pip-22.0.4/src/pip.egg-info
+!tests/packagedcode/data/pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery/celery.egg-info
 /dist
 /build
 /bin

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -14,6 +14,23 @@ Package detection:
   - OpenWRT packages.
   - Yocto/BitBake .bb recipes.
 
+- Update ``GemfileLockParser`` to track the gem which the Gemfile.lock is for,
+  which we assign to the new ``GemfileLockParser.primary_gem`` field. Update
+  ``GemfileLockHandler.parse()`` to handle the case where there is a primary gem
+  detected from a gemfile.lock. If there is a primary gem, a single ``Package``
+  is created and the detected gem data within the gemfile.lock are assigned as
+  dependencies. If there is no primary gem, then all of the dependencies are
+  collected into Package with no name and yielded.
+
+  https://github.com/nexB/scancode-toolkit/issues/3072
+
+- Fix issue where dependencies were not reported when scanning an extracted
+  Python project by modifying ``BaseExtractedPythonLayout.assemble()`` to favor
+  using package data from a PKG-INFO file from an egg-info directory. Package
+  data from a PKG-INFO file from an egg-info directory contains the dependency
+  information collected from the requirements.txt file along side PKG-INFO.
+
+  https://github.com/nexB/scancode-toolkit/issues/3083
 
 License detection:
 ~~~~~~~~~~~~~~~~~~~

diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py
@@ -29,6 +29,7 @@
 import pkginfo2
 from commoncode import fileutils
 from commoncode.fileutils import as_posixpath
+from commoncode.resource import Resource
 from packaging.specifiers import SpecifierSet
 from packageurl import PackageURL
 from packaging import markers
@@ -133,6 +134,16 @@ def create_package_from_package_data(package_data, datafile_path):
     return package
 
 
+def is_egg_info_directory(resource):
+    """
+    Return True if `resource` is a Python .egg-info directory
+    """
+    return (
+        isinstance(resource, Resource)
+        and resource.path.endswith('.egg-info')
+    )
+
+
 class BaseExtractedPythonLayout(BasePypiHandler):
     """
     Base class for development repos, sdist tarballs and other related extracted
@@ -152,7 +163,33 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
         package_resource = None
         if resource.name == 'PKG-INFO':
+            # Initially use current Resource as `package_resource`.
+            # We'll want update `package_resource` with the Resource of a
+            # PKG-INFO file that's in an .egg-info Directory.
             package_resource = resource
+            # We want to use the PKG-INFO file from an .egg-info directory, as
+            # the package info collected from a *.egg_info/PKG-INFO file has
+            # dependency information that a PKG-INFO from the root of a Python
+            # project lacks.
+            parent_resource = resource.parent(codebase)
+            if not is_egg_info_directory(parent_resource):
+                # If we are not in an .egg-info directory, we assume we are at
+                # the root of a Python codebase and we want to find the
+                # .egg_info dir
+                egg_info_dir = None
+                for sibling in resource.siblings(codebase):
+                    if sibling.path.endswith('.egg-info'):
+                        egg_info_dir = sibling
+                        break
+
+                # If we find the .egg_info dir, then we look for the PKG-INFO
+                # file in it and use that as our package_resource
+                if egg_info_dir:
+                    for child in egg_info_dir.children(codebase):
+                        if not child.name == 'PKG-INFO':
+                            continue
+                        package_resource = child
+                        break
         elif resource.name in datafile_name_patterns:
             if resource.has_parent():
                 siblings = resource.siblings(codebase)
@@ -221,7 +258,14 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 package.license_expression = compute_normalized_license(package.declared_license)
             package_uid = package.package_uid
 
-            root = package_resource.parent(codebase)
+            package_resource_parent = package_resource.parent(codebase)
+            if is_egg_info_directory(package_resource_parent):
+                root = package_resource_parent.parent(codebase)
+            else:
+                # We're assuming that our package resource is already at the
+                # root
+                root = package_resource_parent
+
             if root:
                 for py_res in cls.walk_pypi(resource=root, codebase=codebase):
                     if py_res.is_dir:
@@ -724,14 +768,16 @@ def parse(cls, location):
                         name="python",
                     )
                     resolved_purl = get_resolved_purl(purl=purl, specifiers=SpecifierSet(python_requires_specifier))
-                    dependent_packages.append(models.DependentPackage(
-                    purl=str(resolved_purl.purl),
-                    scope=scope,
-                    is_runtime=True,
-                    is_optional=False,
-                    is_resolved=resolved_purl.is_resolved,
-                    extracted_requirement=f"python_requires{python_requires_specifier}",
-                    ))
+                    dependent_packages.append(
+                        models.DependentPackage(
+                            purl=str(resolved_purl.purl),
+                            scope=scope,
+                            is_runtime=True,
+                            is_optional=False,
+                            is_resolved=resolved_purl.is_resolved,
+                            extracted_requirement=f"python_requires{python_requires_specifier}",
+                        )
+                    )
 
             if section.name == "options.extras_require":
                 for sub_section in section:
@@ -931,11 +977,7 @@ def get_requirements_txt_dependencies(location, include_nested=False):
             purl = None
 
         purl = purl and purl.to_string() or None
-
-        if req.is_editable:
-            requirement = req.dumps()
-        else:
-            requirement = req.dumps(with_name=False)
+        requirement = req.dumps()
 
         if location.endswith(
             (
@@ -1311,15 +1353,16 @@ def get_requires_dependencies(requires, default_scope='install'):
                     is_resolved = True
                     purl = purl._replace(version=specifier.version)
 
-        # we use the extra as scope if avialble
-        scope = get_extra(req.marker) or default_scope
+        # we use the extra as scope if available
+        extra = get_extra(req.marker)
+        scope = extra or default_scope
 
         dependent_packages.append(
             models.DependentPackage(
                 purl=purl.to_string(),
                 scope=scope,
                 is_runtime=True,
-                is_optional=False,
+                is_optional=True if bool(extra) else False,
                 is_resolved=is_resolved,
                 extracted_requirement=str(req),
         ))

diff --git a/tests/cluecode/data/plugin_email_url/emails-threshold.expected.json b/tests/cluecode/data/plugin_email_url/emails-threshold.expected.json
@@ -1,32 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--email": true,
-        "--json": "<file>",
-        "--max-email": "2",
-        "--strip-root": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 2
-      }
-    }
-  ],
   "files": [
     {
       "path": "3w-xxxx.c",

diff --git a/tests/cluecode/data/plugin_email_url/emails.expected.json b/tests/cluecode/data/plugin_email_url/emails.expected.json
@@ -1,31 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--email": true,
-        "--json": "<file>",
-        "--strip-root": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 2
-      }
-    }
-  ],
   "files": [
     {
       "path": "3w-xxxx.c",

diff --git a/tests/cluecode/data/plugin_email_url/urls-threshold.expected.json b/tests/cluecode/data/plugin_email_url/urls-threshold.expected.json
@@ -1,32 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--max-url": "2",
-        "--strip-root": true,
-        "--url": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 2
-      }
-    }
-  ],
   "files": [
     {
       "path": "3w-xxxx.c",

diff --git a/tests/cluecode/data/plugin_email_url/urls.expected.json b/tests/cluecode/data/plugin_email_url/urls.expected.json
@@ -1,31 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--strip-root": true,
-        "--url": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 2
-      }
-    }
-  ],
   "files": [
     {
       "path": "3w-xxxx.c",

diff --git a/tests/cluecode/data/plugin_filter_clues/filtered-expected.json b/tests/cluecode/data/plugin_filter_clues/filtered-expected.json
@@ -1,35 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--copyright": true,
-        "--email": true,
-        "--filter-clues": true,
-        "--info": true,
-        "--json": "<file>",
-        "--license": true,
-        "--url": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "files": [
     {
       "path": "LICENSE",

diff --git a/tests/cluecode/data/plugin_filter_clues/filtered-expected2.json b/tests/cluecode/data/plugin_filter_clues/filtered-expected2.json
@@ -1,35 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--copyright": true,
-        "--email": true,
-        "--filter-clues": true,
-        "--info": true,
-        "--json": "<file>",
-        "--license": true,
-        "--url": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "files": [
     {
       "path": "LICENSE2",