diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 697447812d..27f853d3c8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,10 @@ v31.0.0 (next) Reference: https://tracker.debian.org/pkg/wait-for-it https://github.com/nexB/scancode.io/issues/387 +- Add a "tag" field on the CodebaseResource model. + The layer details are stored in this field in the "docker" pipeline. + https://github.com/nexB/scancode.io/issues/443 + - Add support for multiple inputs in the LoadInventory pipeline. https://github.com/nexB/scancode.io/issues/451 diff --git a/scancodeio/settings.py b/scancodeio/settings.py index de87340d5c..c30a925037 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -224,6 +224,10 @@ "level": env.str("SCANCODEIO_LOG_LEVEL", "INFO"), "propagate": False, }, + "django": { + "handlers": ["null"] if IS_TESTS else ["console"], + "propagate": False, + }, }, } diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 1f322fb81c..2a2bec5e0c 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -149,6 +149,7 @@ class Meta: "sha512", "size", "status", + "tag", "type", "name", "extension", diff --git a/scanpipe/migrations/0018_codebaseresource_tag.py b/scanpipe/migrations/0018_codebaseresource_tag.py new file mode 100644 index 0000000000..b6f7587033 --- /dev/null +++ b/scanpipe/migrations/0018_codebaseresource_tag.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.5 on 2022-06-20 08:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0017_alter_discoveredpackage_package_uid_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='codebaseresource', + name='tag', + field=models.CharField(blank=True, max_length=50), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 629a952977..06100f3a16 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1388,6 +1388,10 @@ class CodebaseResource( max_length=30, help_text=_("Analysis status for this resource."), ) + tag = models.CharField( + blank=True, + max_length=50, + ) class Type(models.TextChoices): """ diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 5f4c75528f..2b81392d6c 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -36,7 +36,7 @@ logger = logging.getLogger("scanpipe.pipes") -def make_codebase_resource(project, location, rootfs_path=None): +def make_codebase_resource(project, location, **extra_fields): """ Creates a CodebaseResource instance in the database for the given `project`. @@ -44,8 +44,17 @@ def make_codebase_resource(project, location, rootfs_path=None): It must be rooted in `project.codebase_path` as only the relative path within the project codebase/ directory is stored in the database. - `rootfs_path` is an optional path relative to a rootfs root within an - Image/VM filesystem context. e.g.: "/var/log/file.log" + Extra fields can be provided as keywords arguments to this function call: + + make_codebase_resource( + project=project, + location=resource.location, + rootfs_path=resource.path, + tag=layer_tag, + ) + + In this example, `rootfs_path` is an optional path relative to a rootfs root + within an Image/VM filesystem context. e.g.: "/var/log/file.log" All paths use the POSIX separators. @@ -56,8 +65,8 @@ def make_codebase_resource(project, location, rootfs_path=None): relative_path = Path(location).relative_to(project.codebase_path) resource_data = scancode.get_resource_info(location=location) - if rootfs_path: - resource_data["rootfs_path"] = rootfs_path + if extra_fields: + resource_data.update(**extra_fields) codebase_resource = CodebaseResource( project=project, diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index 494ad1d422..b252da9d19 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -60,7 +60,7 @@ def extract_image_from_tarball(input_tarball, extract_target, verify=True): Path object and collects the extracted images. Returns the `images` and an `errors` list of error messages that may have - happen during the extraction. + happened during the extraction. """ errors = extract_tar(location=input_tarball, target_dir=extract_target) images = Image.get_images_from_dir( @@ -126,16 +126,41 @@ def get_image_data(image, layer_path_segments=2): return image_data +def get_layer_tag(image_id, layer_id, layer_index, id_length=6): + """ + Returns a "tag" crafted from the provided `image_id`, `layer_id`, and `layer_index`. + The purpose of this tag is to be short, clear and sortable. + + For instance, given an image with an id: + 785df58b6b3e120f59bce6cd10169a0c58b8837b24f382e27593e2eea011a0d8 + + and two layers from bottom to top as: + 0690c89adf3e8c306d4ced085fc16d1d104dcfddd6dc637e141fa78be242a707 + 7a1d89d2653e8e4aa9011fd95034a4857109d6636f2ad32df470a196e5dd1585 + + we would get these two tags: + img-785df5-layer-01-0690c8 + img-785df5-layer-02-7a1d89 + """ + short_image_id = image_id[:id_length] + short_layer_id = layer_id[:id_length] + return f"img-{short_image_id}-layer-{layer_index:02}-{short_layer_id}" + + def create_codebase_resources(project, image): """ Creates the CodebaseResource for an `image` in a `project`. """ - for layer_resource in image.get_layers_resources(): - pipes.make_codebase_resource( - project=project, - location=layer_resource.location, - rootfs_path=layer_resource.path, - ) + for layer_index, layer in enumerate(image.layers, start=1): + layer_tag = get_layer_tag(image.image_id, layer.layer_id, layer_index) + + for resource in layer.get_resources(): + pipes.make_codebase_resource( + project=project, + location=resource.location, + rootfs_path=resource.path, + tag=layer_tag, + ) def scan_image_for_system_packages(project, image, detect_licenses=True): diff --git a/scanpipe/templates/scanpipe/resource_list.html b/scanpipe/templates/scanpipe/resource_list.html index 2e1bb7f05d..d97a3dfb46 100644 --- a/scanpipe/templates/scanpipe/resource_list.html +++ b/scanpipe/templates/scanpipe/resource_list.html @@ -26,6 +26,7 @@ Extension Programming language Mime type + Tag License expressions {% if include_compliance_alert %} Compliance alert @@ -58,6 +59,9 @@ {{ resource.mime_type }} + + {{ resource.tag }} +