Use the common extraction API with extractcode for the Docker pipeline #70 (#108)

* Use the same extraction API from scancode for rootfs and docker pipelines #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Move the extract_images and extract_layers logic into the docker pipes module #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add the tag_empty_files pipe in the docker pipeline #70

For consistency with the root_filesystems pipeline

Signed-off-by: Thomas Druez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -3,6 +3,10 @@
 
 ### v1.1.1 (unreleased)
 
+- Use the extractcode API for the Docker pipeline.
+  This change helps with performance and results consistency between pipelines.
+  https://github.com/nexB/scancode.io/issues/70
+
 - Implement cache to prevent scanning multiple times a duplicated codebase resource.
   https://github.com/nexB/scancode.io/issues/70
 

scanpipe/pipelines/docker.py

@@ -20,8 +20,6 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-import os
-
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import docker
@@ -38,16 +36,17 @@ def extract_images(self):
         """
         Extract the images from tarballs.
         """
-        self.images = docker.get_and_extract_images_from_image_tarballs(self.project)
+        self.images, errors = docker.extract_images_from_inputs(self.project)
+        if errors:
+            self.add_error("\n".join(errors))
 
     def extract_layers(self):
         """
         Extract layers from images.
         """
-        for image in self.images:
-            image_dirname = os.path.basename(image.base_location)
-            target_dir = str(self.project.codebase_path / image_dirname)
-            image.extract_layers(target_dir=target_dir)
+        errors = docker.extract_layers_from_images(self.project, self.images)
+        if errors:
+            self.add_error("\n".join(errors))
 
     def find_images_linux_distro(self):
         """
@@ -86,6 +85,12 @@ def tag_uninteresting_codebase_resources(self):
         docker.tag_whiteout_codebase_resources(self.project)
         rootfs.tag_uninteresting_codebase_resources(self.project)
 
+    def tag_empty_files(self):
+        """
+        Flag empty files.
+        """
+        rootfs.tag_empty_codebase_resources(self.project)
+
     def scan_for_application_packages(self):
         """
         Scan unknown resources for packages infos.
@@ -118,6 +123,7 @@ def tag_not_analyzed_codebase_resources(self):
         collect_and_create_codebase_resources,
         collect_and_create_system_packages,
         tag_uninteresting_codebase_resources,
+        tag_empty_files,
         scan_for_application_packages,
         scan_for_files,
         analyze_scanned_files,

scanpipe/pipelines/root_filesystems.py

@@ -22,8 +22,6 @@
 
 import os
 
-from extractcode.extract import extract_file
-
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import rootfs
@@ -40,13 +38,11 @@ def extract_input_files_to_codebase_directory(self):
         Extract root filesystem input archives with extractcode.
         """
         input_files = self.project.inputs("*")
-        target = str(self.project.codebase_path)
+        target_path = self.project.codebase_path
         extract_errors = []
 
         for input_file in input_files:
-            for event in extract_file(input_file, target):
-                if event.done:
-                    extract_errors.extend(event.errors)
+            extract_errors = scancode.extract(input_file, target_path)
 
         if extract_errors:
             self.add_error("\n".join(extract_errors))
@@ -91,17 +87,17 @@ def tag_uninteresting_codebase_resources(self):
         """
         rootfs.tag_uninteresting_codebase_resources(self.project)
 
-    def scan_for_application_packages(self):
+    def tag_empty_files(self):
         """
-        Scan unknown resources for packages infos.
+        Flag empty files.
         """
-        scancode.scan_for_application_packages(self.project)
+        rootfs.tag_empty_codebase_resources(self.project)
 
-    def ignore_empty_files(self):
+    def scan_for_application_packages(self):
         """
-        Skip and mark as ignored any empty file.
+        Scan unknown resources for packages infos.
         """
-        rootfs.tag_empty_codebase_resources(self.project)
+        scancode.scan_for_application_packages(self.project)
 
     def match_not_analyzed_to_system_packages(self):
         """
@@ -149,8 +145,8 @@ def tag_not_analyzed_codebase_resources(self):
         collect_and_create_codebase_resources,
         collect_and_create_system_packages,
         tag_uninteresting_codebase_resources,
+        tag_empty_files,
         scan_for_application_packages,
-        ignore_empty_files,
         match_not_analyzed_to_system_packages,
         scan_for_files,
         analyze_scanned_files,

scanpipe/pipes/docker.py

@@ -31,48 +31,47 @@
 from scanpipe import pipes
 from scanpipe.models import CodebaseResource
 from scanpipe.pipes import rootfs
+from scanpipe.pipes import scancode
 
 logger = logging.getLogger(__name__)
 
 
-def get_images_from_extracted_codebase(project):
+def extract_images_from_inputs(project):
     """
-    Yield images collected from the project "codebase" directory. The
-    image tarball should be extracted there first, with its manifest.json at the
-    root.
+    Collect all the tarballs from the `project` input/ work directory, extract each
+    tarball to the tmp/ work directory and collect the images.
+    Return the `images` and `errors` that may have happen during the extraction.
     """
-    codebase = project.codebase_path.absolute()
-    for image in Image.get_images_from_dir(location=codebase):
-        yield image
+    target_path = project.tmp_path
+    images = []
+    errors = []
 
+    for input_tarball in project.inputs(pattern="*.tar*"):
+        extract_target = target_path / f"{input_tarball.name}-extract"
+        extract_errors = scancode.extract(input_tarball, extract_target)
+        images.extend(Image.get_images_from_dir(extract_target))
+        errors.extend(extract_errors)
 
-def get_and_extract_images_from_image_tarballs(project, force_extract=False):
-    """
-    Yield images collected from the project "codebase" directory.
-    The process is to:
-    1. collect all the tarballs from this directory.
-    2. for each, extract that under a -extract directory and collect its images.
+    return images, errors
 
-    If `force_extract` is False, do not extract if already extracted.
-    """
-    all_images = []
 
-    for tarball in project.inputs(pattern="*.tar*"):
-        tarball_location = str(tarball.absolute())
-        if tarball_location.endswith("-extract"):
-            continue
-        extracted_location = tarball_location + "-extract"
-        tarball_images = Image.get_images_from_tarball(
-            location=tarball_location,
-            target_dir=extracted_location,
-            force_extract=force_extract,
-        )
+def extract_layers_from_images(project, images):
+    """
+    Extract all the layers from provided `images` into the `project` codebase/ work
+    directory.
+    Return the `errors` that may have happen during the extraction.
+    """
+    errors = []
 
-        for image in tarball_images:
-            logger.info("Collected Docker image: {}".format(image.image_id))
-            all_images.append(image)
+    for image in images:
+        image_dirname = Path(image.base_location).name
+        target_path = project.codebase_path / image_dirname
 
-    return all_images
+        for layer in image.layers:
+            extract_target = target_path / layer.layer_id
+            extract_errors = scancode.extract(layer.layer_location, extract_target)
+            errors.extend(extract_errors)
+            layer.extracted_to_location = str(extract_target)
 
 
 def get_image_data(image):

scanpipe/pipes/rootfs.py

@@ -298,7 +298,7 @@ def match_not_analyzed(
 
 def tag_empty_codebase_resources(project):
     """
-    Tag remaining empty files as ignored
+    Tag empty files as ignored.
     """
     project.codebaseresources.select_for_update().filter(
         type__exact="file",
@@ -324,9 +324,7 @@ def tag_uninteresting_codebase_resources(project):
         "/proc/",
         "/dev/",
         "/run/",
-    ) + (
-        # alpine specific
-        "/lib/apk/db/",
+        "/lib/apk/db/",  # alpine specific
     )
 
     qs = project.codebaseresources.no_status()

scanpipe/pipes/scancode.py

@@ -30,6 +30,7 @@
 import packagedcode
 from commoncode import fileutils
 from commoncode.resource import VirtualCodebase
+from extractcode.extract import extract_file
 from packageurl import PackageURL
 from scancode import ScancodeError
 from scancode import api as scancode_api
@@ -45,6 +46,19 @@
 """
 
 
+def extract(location, target):
+    """
+    Wraps the `extractcode.extract_file` to execute the extraction and return errors.
+    """
+    errors = []
+
+    for event in extract_file(location, target):
+        if event.done:
+            errors.extend(event.errors)
+
+    return errors
+
+
 def get_resource_info(location):
     """
     Return a mapping suitable for the creation of a new CodebaseResource.