Add tag field to the CodebaseResource #443 (#456)

* Add tag field to the CodebaseResource #443

Craft and store layer_tag during the docker pipeline

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Fix failing test #443

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add CAHNGELOG entry #443

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Regen the expected results test files #443

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Skip RPM test if not on linux #443

Signed-off-by: Thomas Druez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -27,6 +27,10 @@ v31.0.0 (next)
   Reference: https://tracker.debian.org/pkg/wait-for-it
   https://github.com/nexB/scancode.io/issues/387
 
+- Add a "tag" field on the CodebaseResource model.
+  The layer details are stored in this field in the "docker" pipeline.
+  https://github.com/nexB/scancode.io/issues/443
+
 - Add support for multiple inputs in the LoadInventory pipeline.
   https://github.com/nexB/scancode.io/issues/451
 

scancodeio/settings.py

@@ -224,6 +224,10 @@
             "level": env.str("SCANCODEIO_LOG_LEVEL", "INFO"),
             "propagate": False,
         },
+        "django": {
+            "handlers": ["null"] if IS_TESTS else ["console"],
+            "propagate": False,
+        },
     },
 }
 

scanpipe/filters.py

@@ -149,6 +149,7 @@ class Meta:
             "sha512",
             "size",
             "status",
+            "tag",
             "type",
             "name",
             "extension",

scanpipe/migrations/0018_codebaseresource_tag.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.5 on 2022-06-20 08:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0017_alter_discoveredpackage_package_uid_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='codebaseresource',
+            name='tag',
+            field=models.CharField(blank=True, max_length=50),
+        ),
+    ]

scanpipe/models.py

@@ -1388,6 +1388,10 @@ class CodebaseResource(
         max_length=30,
         help_text=_("Analysis status for this resource."),
     )
+    tag = models.CharField(
+        blank=True,
+        max_length=50,
+    )
 
     class Type(models.TextChoices):
         """

scanpipe/pipes/__init__.py

@@ -36,16 +36,25 @@
 logger = logging.getLogger("scanpipe.pipes")
 
 
-def make_codebase_resource(project, location, rootfs_path=None):
+def make_codebase_resource(project, location, **extra_fields):
     """
     Creates a CodebaseResource instance in the database for the given `project`.
 
     The provided `location` is the absolute path of this resource.
     It must be rooted in `project.codebase_path` as only the relative path within the
     project codebase/ directory is stored in the database.
 
-    `rootfs_path` is an optional path relative to a rootfs root within an
-    Image/VM filesystem context. e.g.: "/var/log/file.log"
+    Extra fields can be provided as keywords arguments to this function call:
+
+        make_codebase_resource(
+            project=project,
+            location=resource.location,
+            rootfs_path=resource.path,
+            tag=layer_tag,
+        )
+
+    In this example, `rootfs_path` is an optional path relative to a rootfs root
+    within an Image/VM filesystem context. e.g.: "/var/log/file.log"
 
     All paths use the POSIX separators.
 
@@ -56,8 +65,8 @@ def make_codebase_resource(project, location, rootfs_path=None):
     relative_path = Path(location).relative_to(project.codebase_path)
     resource_data = scancode.get_resource_info(location=location)
 
-    if rootfs_path:
-        resource_data["rootfs_path"] = rootfs_path
+    if extra_fields:
+        resource_data.update(**extra_fields)
 
     codebase_resource = CodebaseResource(
         project=project,

scanpipe/pipes/docker.py

@@ -60,7 +60,7 @@ def extract_image_from_tarball(input_tarball, extract_target, verify=True):
     Path object and collects the extracted images.
 
     Returns the `images` and an `errors` list of error messages that may have
-    happen during the extraction.
+    happened during the extraction.
     """
     errors = extract_tar(location=input_tarball, target_dir=extract_target)
     images = Image.get_images_from_dir(
@@ -126,16 +126,41 @@ def get_image_data(image, layer_path_segments=2):
     return image_data
 
 
+def get_layer_tag(image_id, layer_id, layer_index, id_length=6):
+    """
+    Returns a "tag" crafted from the provided `image_id`, `layer_id`, and `layer_index`.
+    The purpose of this tag is to be short, clear and sortable.
+
+    For instance, given an image with an id:
+        785df58b6b3e120f59bce6cd10169a0c58b8837b24f382e27593e2eea011a0d8
+
+    and two layers from bottom to top as:
+        0690c89adf3e8c306d4ced085fc16d1d104dcfddd6dc637e141fa78be242a707
+        7a1d89d2653e8e4aa9011fd95034a4857109d6636f2ad32df470a196e5dd1585
+
+    we would get these two tags:
+        img-785df5-layer-01-0690c8
+        img-785df5-layer-02-7a1d89
+    """
+    short_image_id = image_id[:id_length]
+    short_layer_id = layer_id[:id_length]
+    return f"img-{short_image_id}-layer-{layer_index:02}-{short_layer_id}"
+
+
 def create_codebase_resources(project, image):
     """
     Creates the CodebaseResource for an `image` in a `project`.
     """
-    for layer_resource in image.get_layers_resources():
-        pipes.make_codebase_resource(
-            project=project,
-            location=layer_resource.location,
-            rootfs_path=layer_resource.path,
-        )
+    for layer_index, layer in enumerate(image.layers, start=1):
+        layer_tag = get_layer_tag(image.image_id, layer.layer_id, layer_index)
+
+        for resource in layer.get_resources():
+            pipes.make_codebase_resource(
+                project=project,
+                location=resource.location,
+                rootfs_path=resource.path,
+                tag=layer_tag,
+            )
 
 
 def scan_image_for_system_packages(project, image, detect_licenses=True):

scanpipe/templates/scanpipe/resource_list.html

@@ -26,6 +26,7 @@
             <th>Extension</th>
             <th>Programming language</th>
             <th>Mime type</th>
+            <th>Tag</th>
             <th>License expressions</th>
             {% if include_compliance_alert %}
               <th>Compliance alert</th>
@@ -58,6 +59,9 @@
               <td>
                 <a href="?mime_type={{ resource.mime_type }}" class="is-black-link">{{ resource.mime_type }}</a>
               </td>
+              <td>
+                <a href="?tag={{ resource.tag }}" class="is-black-link">{{ resource.tag }}</a>
+              </td>
               <td>
                  <ul>
                   {% for license_expression in resource.unique_license_expressions %}