aboutcode-org
diff --git a/‎CHANGELOG.rst‎
Lines changed: 28 additions & 2 deletions b/‎CHANGELOG.rst‎
Lines changed: 28 additions & 2 deletions
diff --git a/‎docs/rest-api.rst‎
Lines changed: 1 addition & 1 deletion b/‎docs/rest-api.rst‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎scanpipe/api/serializers.py‎
Lines changed: 43 additions & 4 deletions b/‎scanpipe/api/serializers.py‎
Lines changed: 43 additions & 4 deletions
diff --git a/‎scanpipe/api/views.py‎
Lines changed: 11 additions & 0 deletions b/‎scanpipe/api/views.py‎
Lines changed: 11 additions & 0 deletions
diff --git a/‎scanpipe/filters.py‎
Lines changed: 30 additions & 0 deletions b/‎scanpipe/filters.py‎
Lines changed: 30 additions & 0 deletions
diff --git a/‎scanpipe/migrations/0022_rename_dependencies_discoveredpackage_dependencies_data_and_more.py‎
Lines changed: 93 additions & 0 deletions b/‎scanpipe/migrations/0022_rename_dependencies_discoveredpackage_dependencies_data_and_more.py‎
Lines changed: 93 additions & 0 deletions
diff --git a/‎scanpipe/migrations/0023_migrate_dependencies.py‎
Lines changed: 49 additions & 0 deletions b/‎scanpipe/migrations/0023_migrate_dependencies.py‎
Lines changed: 49 additions & 0 deletions
@@ -99,16 +99,42 @@ v31.0.0 (2022-08-25)
   - Package data detected from a file are now stored on the
     CodebaseResource.package_data field.
   - A second processing step is now done after scanning for Package data, where
-    Package Resources are determined and DiscoveredPackages are created.
+    Package Resources are determined and DiscoveredPackages and
+    DiscoveredDependencies are created.
 
   https://github.com/nexB/scancode.io/issues/444
 
 - ``CodebaseResource.for_packages`` now returns a list of
   ``DiscoveredPackage.package_uid`` or ``DiscoveredPackage.package_url`` if
   ``DiscoveredPackage.package_uid`` is not present. This is done to reflect the
-  how scancode-toolkit's JSON output returns ``package_uid`` in the
+  how scancode-toolkit's JSON output returns ``package_uid``s in the
   ``for_packages`` field for Resources.
 
+- Add the model DiscoveredDependency. This represents Package dependencies
+  discovered in a Project. The ``scan_codebase`` and ``scan_packages`` pipelines
+  have been updated to create DiscoveredDepdendency objects. The Project API has
+  been updated with new fields:
+
+  - ``dependency_count``
+    - The number of DiscoveredDependencies associated with the project.
+
+  - ``discovered_dependencies_summary``
+    - A mapping that contains following fields:
+
+      - ``total``
+        - The number of DiscoveredDependencies associated with the project.
+      - ``is_runtime``
+        - The number of runtime dependencies.
+      - ``is_optional``
+        - The number of optional dependencies.
+      - ``is_resolved``
+        - The number of resolved dependencies.
+
+  These values are also available on the Project view.
+  https://github.com/nexB/scancode.io/issues/447
+
+- The ``dependencies`` field has been removed from the DiscoveredPackage model.
+
 v30.2.0 (2021-12-17)
 --------------------
 
 
@@ -188,7 +188,7 @@ The project details view returns all information available about a project.
         "codebase_resources_summary": {
             "application-package": 1
         },
-        "discovered_package_summary": {
+        "discovered_packages_summary": {
             "total": 1,
             "with_missing_resources": 0,
             "with_modified_resources": 0
 
@@ -26,6 +26,7 @@
 
 from scanpipe.api import ExcludeFromListViewMixin
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -112,7 +113,8 @@ class ProjectSerializer(
     runs = RunSerializer(many=True, read_only=True)
     input_sources = serializers.JSONField(source="input_sources_list", read_only=True)
     codebase_resources_summary = serializers.SerializerMethodField()
-    discovered_package_summary = serializers.SerializerMethodField()
+    discovered_packages_summary = serializers.SerializerMethodField()
+    discovered_dependencies_summary = serializers.SerializerMethodField()
 
     class Meta:
         model = Project
@@ -136,8 +138,10 @@ class Meta:
             "error_count",
             "resource_count",
             "package_count",
+            "dependency_count",
             "codebase_resources_summary",
-            "discovered_package_summary",
+            "discovered_packages_summary",
+            "discovered_dependencies_summary",
         )
 
         exclude_from_list_view = [
@@ -147,22 +151,33 @@ class Meta:
             "error_count",
             "resource_count",
             "package_count",
+            "dependency_count",
             "codebase_resources_summary",
-            "discovered_package_summary",
+            "discovered_packages_summary",
+            "discovered_dependencies_summary",
         ]
 
     def get_codebase_resources_summary(self, project):
         queryset = project.codebaseresources.all()
         return count_group_by(queryset, "status")
 
-    def get_discovered_package_summary(self, project):
+    def get_discovered_packages_summary(self, project):
         base_qs = project.discoveredpackages
         return {
             "total": base_qs.count(),
             "with_missing_resources": base_qs.exclude(missing_resources=[]).count(),
             "with_modified_resources": base_qs.exclude(modified_resources=[]).count(),
         }
 
+    def get_discovered_dependencies_summary(self, project):
+        base_qs = project.discovereddependencies
+        return {
+            "total": base_qs.count(),
+            "is_runtime": base_qs.filter(is_runtime=True).count(),
+            "is_optional": base_qs.filter(is_optional=True).count(),
+            "is_resolved": base_qs.filter(is_resolved=True).count(),
+        }
+
     def create(self, validated_data):
         """
         Creates a new `project` with `upload_file` and `pipeline` as optional.
@@ -219,6 +234,29 @@ class Meta:
         ]
 
 
+class DiscoveredDependencySerializer(serializers.ModelSerializer):
+    purl = serializers.ReadOnlyField()
+    for_package_uid = serializers.ReadOnlyField()
+    datafile_path = serializers.ReadOnlyField()
+    package_type = serializers.ReadOnlyField(source="type")
+
+    class Meta:
+        model = DiscoveredDependency
+        fields = [
+            "purl",
+            "package_type",
+            "extracted_requirement",
+            "scope",
+            "is_runtime",
+            "is_optional",
+            "is_resolved",
+            "dependency_uid",
+            "for_package_uid",
+            "datafile_path",
+            "datasource_id",
+        ]
+
+
 class ProjectErrorSerializer(serializers.ModelSerializer):
     traceback = serializers.SerializerMethodField()
 
@@ -257,6 +295,7 @@ def get_model_serializer(model_class):
     serializer = {
         CodebaseResource: CodebaseResourceSerializer,
         DiscoveredPackage: DiscoveredPackageSerializer,
+        DiscoveredDependency: DiscoveredDependencySerializer,
         ProjectError: ProjectErrorSerializer,
     }.get(model_class, None)
 
 
@@ -36,6 +36,7 @@
 from rest_framework.response import Response
 
 from scanpipe.api.serializers import CodebaseResourceSerializer
+from scanpipe.api.serializers import DiscoveredDependencySerializer
 from scanpipe.api.serializers import DiscoveredPackageSerializer
 from scanpipe.api.serializers import PipelineSerializer
 from scanpipe.api.serializers import ProjectErrorSerializer
@@ -180,6 +181,16 @@ def packages(self, request, *args, **kwargs):
 
         return Response(serializer.data)
 
+    @action(detail=True)
+    def dependencies(self, request, *args, **kwargs):
+        project = self.get_object()
+        queryset = project.discovereddependencies.all()
+
+        paginated_qs = self.paginate_queryset(queryset)
+        serializer = DiscoveredDependencySerializer(paginated_qs, many=True)
+
+        return Response(serializer.data)
+
     @action(detail=True)
     def errors(self, request, *args, **kwargs):
         project = self.get_object()
 
@@ -32,6 +32,7 @@
 from packageurl.contrib.django.filters import PackageURLFilter
 
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
@@ -159,6 +160,7 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             "created_date",
             "name",
             "discoveredpackages_count",
+            "discovereddependencies_count",
             "codebaseresources_count",
             "projecterrors_count",
         ],
@@ -169,6 +171,8 @@ class ProjectFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
             ("-name", "Name (z-A)"),
             ("-discoveredpackages_count", "Packages (+)"),
             ("discoveredpackages_count", "Packages (-)"),
+            ("-discovereddependencies_count", "Dependencies (+)"),
+            ("discovereddependencies_count", "Dependencies (-)"),
             ("-codebaseresources_count", "Resources (+)"),
             ("codebaseresources_count", "Resources (-)"),
             ("-projecterrors_count", "Errors (+)"),
@@ -364,6 +368,32 @@ class Meta:
         ]
 
 
+class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
+    search = django_filters.CharFilter(
+        label="Search", field_name="name", lookup_expr="icontains"
+    )
+    purl = PackageURLFilter(label="Package URL")
+
+    class Meta:
+        model = DiscoveredDependency
+        fields = [
+            "search",
+            "purl",
+            "dependency_uid",
+            "type",
+            "namespace",
+            "name",
+            "version",
+            "qualifiers",
+            "subpath",
+            "scope",
+            "is_runtime",
+            "is_optional",
+            "is_resolved",
+            "datasource_id",
+        ]
+
+
 class ErrorFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
     search = django_filters.CharFilter(
         label="Search", field_name="message", lookup_expr="icontains"
 
@@ -0,0 +1,93 @@
+# Generated by Django 4.0.6 on 2022-08-13 00:27
+
+from django.db import migrations, models
+import django.db.models.deletion
+import scanpipe.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0021_codebaseresource_package_data'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='discoveredpackage',
+            old_name='dependencies',
+            new_name='dependencies_data',
+        ),
+        migrations.CreateModel(
+            name='DiscoveredDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('dependency_uid', models.CharField(help_text='The unique identifier of this dependency.', max_length=1024)),
+                ('purl', models.CharField(help_text='The Package URL of this dependency.', max_length=1024)),
+                ('extracted_requirement', models.CharField(blank=True, help_text='The version requirements of this dependency.', max_length=64)),
+                ('scope', models.CharField(blank=True, help_text='The scope of this dependency, how it is used in a project.', max_length=64)),
+                ('is_runtime', models.BooleanField(default=False)),
+                ('is_optional', models.BooleanField(default=False)),
+                ('is_resolved', models.BooleanField(default=False)),
+                ('datasource_id', models.CharField(blank=True, help_text='The identifier for the datafile handler used to obtain this dependency.', max_length=64)),
+                ('datafile_resource', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='dependencies', to='scanpipe.codebaseresource')),
+                ('for_package', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='dependencies', to='scanpipe.discoveredpackage')),
+                ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='discovereddependencies', to='scanpipe.project')),
+            ],
+            options={
+                'verbose_name': 'discovered dependency',
+                'verbose_name_plural': 'discovered dependencies',
+                'ordering': ['-is_runtime', '-is_resolved', 'is_optional', 'dependency_uid'],
+            },
+            bases=(models.Model, scanpipe.models.SaveProjectErrorMixin),
+        ),
+        migrations.AddConstraint(
+            model_name='discovereddependency',
+            constraint=models.UniqueConstraint(condition=models.Q(('dependency_uid', ''), _negated=True), fields=('project', 'dependency_uid'), name='scanpipe_discovereddependency_unique_dependency_uid_within_project'),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='name',
+            field=models.CharField(blank=True, help_text='Name of the package.', max_length=100),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='namespace',
+            field=models.CharField(blank=True, help_text='Package name prefix, such as Maven groupid, Docker image owner, GitHub user or organization, etc.', max_length=255),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='qualifiers',
+            field=models.CharField(blank=True, help_text='Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.', max_length=1024),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='subpath',
+            field=models.CharField(blank=True, help_text='Extra subpath within a package, relative to the package root.', max_length=200),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='type',
+            field=models.CharField(blank=True, help_text='A short code to identify the type of this package. For example: gem for a Rubygem, docker for a container, pypi for a Python Wheel or Egg, maven for a Maven Jar, deb for a Debian package, etc.', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='discovereddependency',
+            name='version',
+            field=models.CharField(blank=True, help_text='Version of the package.', max_length=100),
+        ),
+        migrations.AlterModelOptions(
+            name="discovereddependency",
+            options={
+                "ordering": [
+                    "-is_runtime",
+                    "-is_resolved",
+                    "is_optional",
+                    "dependency_uid",
+                    "for_package",
+                    "datafile_resource",
+                    "datasource_id",
+                ],
+                "verbose_name": "discovered dependency",
+                "verbose_name_plural": "discovered dependencies",
+            },
+        ),
+    ]
@@ -0,0 +1,49 @@
+# Generated by Django 4.0.6 on 2022-08-12 23:13
+
+from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.db import migrations
+
+
+def migrate_dependencies_to_discovereddependencies(apps, schema_editor):
+    DiscoveredPackage = apps.get_model('scanpipe', 'DiscoveredPackage')
+    DiscoveredDependency = apps.get_model('scanpipe', 'DiscoveredDependency')
+
+    package_with_dependencies = DiscoveredPackage.objects.exclude(dependencies_data=[])
+
+    for package in package_with_dependencies:
+        for dependency_data in package.dependencies_data:
+            project = package.project
+
+            # Remove non-supported fields from the data dict
+            dependency_data.pop("extra_data", None)
+            dependency_data.pop("resolved_package", None)
+
+            for_package_uid = dependency_data.get("for_package_uid")
+            try:
+                for_package = project.discoveredpackages.get(package_uid=for_package_uid)
+            except (ObjectDoesNotExist, MultipleObjectsReturned):
+                for_package = None
+
+            datafile_path = dependency_data.get("datafile_path")
+            try:
+                datafile_resource = project.codebaseresources.get(path=datafile_path)
+            except (ObjectDoesNotExist, MultipleObjectsReturned):
+                datafile_resource = None
+
+            DiscoveredDependency.objects.create(
+                project=project,
+                for_package=for_package,
+                datafile_resource=datafile_resource,
+                **dependency_data,
+            )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0022_rename_dependencies_discoveredpackage_dependencies_data_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_dependencies_to_discovereddependencies),
+    ]