Refactor the relation filtering #688

Signed-off-by: Thomas Druez <tdruez@nexb.com>

Author: tdruez

scanpipe/filters.py

@@ -121,6 +121,7 @@ def filter_queryset(self, queryset):
         for name, value in self.form.cleaned_data.items():
             field_name = self.filters[name].field_name
             if value == self.empty_value:
+                print(name, value)
                 queryset = queryset.filter(**{f"{field_name}__in": EMPTY_VALUES})
             elif value == self.other_value and hasattr(queryset, "less_common"):
                 return queryset.less_common(name)
@@ -278,6 +279,26 @@ def filter(self, qs, value):
         return qs
 
 
+class RelationMatchTypeFilter(django_filters.ChoiceFilter):
+    def __init__(self, *args, **kwargs):
+        kwargs["choices"] = (
+            ("", "All"),
+            ("none", "No match"),
+            ("any", "Any match"),
+            ("java_to_class", "java to class"),
+            ("path", "path"),
+            ("sha1", "sha1"),
+        )
+        super().__init__(*args, **kwargs)
+
+    def filter(self, qs, value):
+        if value == "none":
+            return qs.has_no_relation()
+        elif value == "any":
+            return qs.has_relation()
+        return super().filter(qs, value)
+
+
 class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = django_filters.CharFilter(
         label="Search",
@@ -297,6 +318,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "mime_type",
             "tag",
             "compliance_alert",
+            "related_from__match_type",
         ],
     )
     license_key = JSONContainsFilter(
@@ -311,6 +333,10 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
         choices=CodebaseResource.Compliance.choices + [("EMPTY", "EMPTY")]
     )
     in_package = InPackageFilter(label="In a Package")
+    relation_match_type = RelationMatchTypeFilter(
+        label="Relation match type",
+        field_name="related_from__match_type",
+    )
 
     class Meta:
         model = CodebaseResource
@@ -341,8 +367,7 @@ class Meta:
             "emails",
             "urls",
             "in_package",
-            # CodebaseRelation
-            "related_from__match_type",
+            "relation_match_type",
         ]
 
     @classmethod

scanpipe/models.py

@@ -1419,18 +1419,14 @@ def to_codebase(self):
         """Resources in to/ directory"""
         return self.filter(path__startswith="to/")
 
+    def has_relation(self):
+        """Resources assigned to at least one CodebaseRelation"""
+        return self.filter(Q(related_from__isnull=False) | Q(related_to__isnull=False))
+
     def has_no_relation(self):
         """Resources not part of any CodebaseRelation"""
         return self.filter(related_from__isnull=True, related_to__isnull=True)
 
-    def missing_in_to(self):
-        """Resources in from/ not found in to/"""
-        return self.from_codebase().filter(related_to__isnull=True)
-
-    def missing_in_from(self):
-        """Resources in to/ not found in from/"""
-        return self.to_codebase().filter(related_from__isnull=True)
-
     def has_value(self, field_name):
         return self.filter(~Q((field_name, "")))
 

scanpipe/templates/scanpipe/includes/match_type_filter_dropdown.html

@@ -9,17 +9,20 @@
       <a href="?" class="dropdown-item">
         All
       </a>
-      <a href="?missing_only=1" class="dropdown-item">
+      <a href="?relation_match_type=none" class="dropdown-item">
         No match
       </a>
-      <a href="?related_from__match_type=java_to_class" class="dropdown-item">
-        java_to_class
+      <a href="?relation_match_type=any" class="dropdown-item">
+        Any match
       </a>
-      <a href="?related_from__match_type=path" class="dropdown-item">
-        path
+      <a href="?relation_match_type=java_to_class" class="dropdown-item">
+        Java to class
       </a>
-      <a href="?related_from__match_type=sha1" class="dropdown-item">
-        sha1
+      <a href="?relation_match_type=path" class="dropdown-item">
+        Path
+      </a>
+      <a href="?relation_match_type=sha1" class="dropdown-item">
+        SHA1
       </a>
     </div>
   </div>

scanpipe/templates/scanpipe/relation_list.html

@@ -22,7 +22,10 @@
           <tr>
             <th style="width: 45%">To resource</th>
             <th>
-              Match type
+              <a href="?sort={% if "-" not in filter.data.sort %}-{% endif %}related_from__match_type" class="is-black-link">Match type</a>
+              {% if "related_from__match_type" in filter.data.sort %}
+                <i class="fa fa-sort-{% if "-" in filter.data.sort %}down{% else %}up{% endif %}"></i>
+              {% endif %}
               <div style="float: right">
                 {% include 'scanpipe/includes/match_type_filter_dropdown.html' %}
               </div>
@@ -66,6 +69,12 @@
                 <td></td>
               </tr>
             {% endfor %}
+          {% empty %}
+            <tr>
+              <td colspan="10" class="has-text-centered p-3">
+                No Relations found. <a href="?">Clear search and filters</a>
+              </td>
+            </tr>
           {% endfor %}
         </tbody>
       </table>

scanpipe/tests/test_models.py

@@ -48,6 +48,7 @@
 from rq.job import JobStatus
 
 from scancodeio import __version__ as scancodeio_version
+from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
@@ -1081,6 +1082,17 @@ def test_scanpipe_codebase_resource_queryset_methods(self):
         self.assertEqual(1, CodebaseResource.objects.in_package().count())
         self.assertEqual(2, CodebaseResource.objects.not_in_package().count())
 
+        self.assertEqual(0, CodebaseResource.objects.has_relation().count())
+        self.assertEqual(3, CodebaseResource.objects.has_no_relation().count())
+        CodebaseRelation.objects.create(
+            project=self.project1,
+            from_resource=file,
+            to_resource=directory,
+            relationship=CodebaseRelation.Relationship.IDENTICAL,
+        )
+        self.assertEqual(2, CodebaseResource.objects.has_relation().count())
+        self.assertEqual(1, CodebaseResource.objects.has_no_relation().count())
+
     def test_scanpipe_codebase_resource_queryset_licenses_categories(self):
         CodebaseResource.objects.all().delete()
 

scanpipe/views.py

@@ -953,17 +953,14 @@ class CodebaseRelationListView(
     paginate_by = settings.SCANCODEIO_PAGINATE_BY.get("relation", 100)
 
     def get_queryset(self):
-        queryset = (
+        return (
             super()
             .get_queryset()
             .files()
             .to_codebase()
             .prefetch_related("related_from__from_resource")
             .distinct()
         )
-        if self.request.GET.get("missing_only"):
-            queryset = queryset.has_no_relation()
-        return queryset
 
     @staticmethod
     def get_rows(qs):