Improve dependency resolver for lockfiles

Handle various lockfile cases where:
* Same package/dependencies are present in different lockfiles
* Independent lockfiles without a manifest and root package
* Ecosystems which have only a single version of package in
  their environment
* Dependency graphs where a resolved package can have many
  parent packages.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

scanpipe/models.py

@@ -3477,10 +3477,28 @@ class DiscoveredDependency(
             "The identifier for the datafile handler used to obtain this dependency."
         ),
     )
-    is_runtime = models.BooleanField(default=False)
-    is_optional = models.BooleanField(default=False)
-    is_resolved = models.BooleanField(default=False)
-    is_direct = models.BooleanField(default=False)
+    is_runtime = models.BooleanField(
+        default=False,
+        help_text=_("True if this dependency is a runtime dependency."),
+    )
+    is_optional = models.BooleanField(
+        default=False,
+        help_text=_("True if this dependency is an optional dependency"),
+    )
+    is_resolved = models.BooleanField(
+        default=False,
+        help_text=_(
+            "True if this dependency version requirement has been pinned "
+            "and this dependency points to an exact version."
+        ),
+    )
+    is_direct = models.BooleanField(
+        default=False,
+        help_text=_(
+            "True if this is a direct, first-level dependency relationship "
+            "for a package."
+        ),
+    )
 
     objects = DiscoveredDependencyQuerySet.as_manager()
 

scanpipe/pipes/__init__.py

@@ -270,9 +270,10 @@ def update_or_create_dependency(
     if ignore_dependency_scope(project, dependency_data):
         return  # Do not create the DiscoveredDependency record.
 
+    dependencies = []
     if not dependency_uid:
         purl_data = DiscoveredDependency.extract_purl_data(dependency_data)
-        dependency = DiscoveredDependency.objects.get_or_none(
+        dependencies = DiscoveredDependency.objects.filter(
             project=project,
             extracted_requirement=extracted_requirement,
             **purl_data,
@@ -282,12 +283,36 @@ def update_or_create_dependency(
             project=project,
             dependency_uid=dependency_uid,
         )
+        if dependency:
+            dependencies.append(dependency)
+
+    for dependency in dependencies:
+        # This dependency relationship is for a new package
+        if (
+            for_package
+            and dependency.for_package
+            and dependency.for_package != for_package
+        ):
+            DiscoveredDependency.populate_dependency_uuid(dependency_data)
+            dependency = DiscoveredDependency.create_from_data(
+                project,
+                dependency_data,
+                for_package=for_package,
+                resolved_to_package=resolved_to_package,
+                datafile_resource=datafile_resource,
+                datasource_id=datasource_id,
+                strip_datafile_path_root=strip_datafile_path_root,
+            )
+            break
 
-    if dependency:
-        dependency.update_from_data(dependency_data)
-        if resolved_to_package and not dependency.resolved_to_package:
-            dependency.update(resolved_to_package=resolved_to_package)
-    else:
+        elif dependency:
+            dependency.update_from_data(dependency_data)
+            if resolved_to_package and not dependency.resolved_to_package:
+                dependency.update(resolved_to_package=resolved_to_package)
+            if for_package and not dependency.for_package:
+                dependency.update(for_package=for_package)
+
+    if not dependencies:
         DiscoveredDependency.populate_dependency_uuid(dependency_data)
         dependency = DiscoveredDependency.create_from_data(
             project,

scanpipe/pipes/scancode.py

@@ -528,11 +528,15 @@ def resolve_dependencies(project):
             package = None
             if pd.purl:
                 purl_data = DiscoveredPackage.extract_purl_data(package_mapping)
-                package = DiscoveredPackage.objects.get_or_none(
+                packages = DiscoveredPackage.objects.filter(
                     project=project,
                     **purl_data,
                 )
 
+                for package in packages:
+                    if resource.location in package.datafile_paths:
+                        break
+
             dependencies = package_mapping.get("dependencies") or []
             update_packages_and_dependencies(
                 project=project,
@@ -583,6 +587,8 @@ def update_packages_and_dependencies(
             dependency_data=dep,
             for_package=package,
             resolved_to_package=resolved_to_package,
+            datafile_resource=resource,
+            datasource_id=datasource_id,
         )
 
 
@@ -628,6 +634,15 @@ def match_and_resolve_dependencies(project):
                 )
             ]
 
+            # This should be done only in the case of lockfiles where only one version
+            # of a package is present for an environment
+            if not other_dependencies:
+                other_dependencies = [
+                    dep
+                    for dep in matched_dependencies
+                    if (dep.purl == dependency.purl and dep.resolved_to_package)
+                ]
+
         if other_dependencies:
             resolved_dependency = other_dependencies.pop()
             dependency.update(

scanpipe/templates/scanpipe/dependency_list.html

@@ -48,6 +48,9 @@
             <td>
               <a href="?is_resolved={{ dependency.is_resolved }}" class="is-black-link">{{ dependency.is_resolved }}</a>
             </td>
+            <td>
+              <a href="?is_resolved={{ dependency.is_direct }}" class="is-black-link">{{ dependency.is_direct }}</a>
+            </td>
             <td>
               {% if dependency.for_package %}
                 {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #}

scanpipe/views.py

@@ -1546,6 +1546,10 @@ class DiscoveredDependencyListView(
             "field_name": "is_resolved",
             "filter_fieldname": "is_resolved",
         },
+        {
+            "field_name": "is_direct",
+            "filter_fieldname": "is_direct",
+        },
         "for_package",
         "resolved_to_package",
         "datafile_resource",