Implement multiple performance enhancements #70 (#110)

* Run celery worker with the "threads" pool implementation

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Implement parallelization with ProcessPoolExecutor for file and package scans #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Consistent queryset calls using project.relation in place of Relation.objects #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add settings configuration for the MAX_WORKERS #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add a SCANCODE_PROCESSES settings to control the multiprocessing CPUs #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Optimize "tag" type pipes using the update() API #70

in place of save() on the QuerySet iteration

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add entries in the CHANGELOG #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Add size=0 as a lookup in the .empty() queryset method #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Refactor make_codebase_resource to remove the extra get query #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

* Reduce the pagination by 10 in the ProjectListView #70

Signed-off-by: Thomas Druez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -3,6 +3,15 @@
 
 ### v1.1.1 (unreleased)
 
+- Run celery worker with the "threads" pool implementation.
+  Implement parallelization with ProcessPoolExecutor for file and package scans.
+  Add a SCANCODE_PROCESSES settings to control the multiprocessing CPUs count.
+  https://github.com/nexB/scancode.io/issues/70
+
+- Optimize "tag" type pipes using the update() API in place of save() on the QuerySet
+  iteration.
+  https://github.com/nexB/scancode.io/issues/70
+
 - Use the extractcode API for the Docker pipeline.
   This change helps with performance and results consistency between pipelines.
   https://github.com/nexB/scancode.io/issues/70

docker-compose.yml

@@ -15,8 +15,9 @@ services:
     build: .
     command: celery --app scancodeio worker
                     --loglevel=INFO
-                    --soft-time-limit=21600 --time-limit=22000
+                    --concurrency 1 --pool threads
                     --events -Ofair --prefetch-multiplier=1
+                    --soft-time-limit=21600 --time-limit=22000
     env_file:
       - docker.env
     volumes:
@@ -32,7 +33,7 @@ services:
     command: sh -c "
         ./manage.py migrate &&
         ./manage.py collectstatic --no-input &&
-        gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 --workers 3"
+        gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 --workers 2"
     env_file:
       - docker.env
     expose:

etc/nginx/conf.d/default.conf

@@ -10,8 +10,8 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $host;
         proxy_redirect off;
-        client_max_body_size 100M;
-	    proxy_read_timeout 600s;
+        client_max_body_size 10G;
+        proxy_read_timeout 600s;
     }
 
     location /static/ {

scancodeio/settings/base.py

@@ -45,6 +45,10 @@
 
 SCANCODE_DEFAULT_OPTIONS = env.list("SCANCODE_DEFAULT_OPTIONS", default=[])
 
+# Set the number of parallel processes to use for ScanCode related scan execution.
+# If the SCANCODE_PROCESSES argument is not set, defaults to the number of CPUs minus 1.
+SCANCODE_PROCESSES = env.int("SCANCODE_PROCESSES", default=None)
+
 # Application definition
 
 INSTALLED_APPS = (

scanpipe/api/serializers.py

@@ -21,10 +21,8 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from django.apps import apps
-from django.utils.functional import lazy
 
 from rest_framework import serializers
-from rest_framework.exceptions import ValidationError
 
 from scanpipe.api import ExcludeFromListViewMixin
 from scanpipe.models import CodebaseResource

scanpipe/api/views.py

@@ -21,6 +21,7 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from django.apps import apps
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
 
 from rest_framework import mixins
@@ -36,10 +37,7 @@
 from scanpipe.api.serializers import ProjectErrorSerializer
 from scanpipe.api.serializers import ProjectSerializer
 from scanpipe.api.serializers import RunSerializer
-from scanpipe.models import CodebaseResource
-from scanpipe.models import DiscoveredPackage
 from scanpipe.models import Project
-from scanpipe.models import ProjectError
 from scanpipe.models import Run
 from scanpipe.views import project_results_json_response
 
@@ -97,9 +95,7 @@ def pipelines(self, request, *args, **kwargs):
     @action(detail=True)
     def resources(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = CodebaseResource.objects.project(project).prefetch_related(
-            "discovered_packages"
-        )
+        queryset = project.codebaseresources.prefetch_related("discovered_packages")
 
         paginated_qs = self.paginate_queryset(queryset)
         serializer = CodebaseResourceSerializer(paginated_qs, many=True)
@@ -109,7 +105,7 @@ def resources(self, request, *args, **kwargs):
     @action(detail=True)
     def packages(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = DiscoveredPackage.objects.project(project)
+        queryset = project.discoveredpackages.all()
 
         paginated_qs = self.paginate_queryset(queryset)
         serializer = DiscoveredPackageSerializer(paginated_qs, many=True)
@@ -119,7 +115,7 @@ def packages(self, request, *args, **kwargs):
     @action(detail=True)
     def errors(self, request, *args, **kwargs):
         project = self.get_object()
-        queryset = ProjectError.objects.project(project)
+        queryset = project.projecterrors.all()
 
         paginated_qs = self.paginate_queryset(queryset)
         serializer = ProjectErrorSerializer(paginated_qs, many=True)
@@ -130,11 +126,11 @@ def errors(self, request, *args, **kwargs):
     def file_content(self, request, *args, **kwargs):
         project = self.get_object()
         path = request.query_params.get("path")
-        codebase_resources = CodebaseResource.objects.project(project)
+        codebase_resources = project.codebaseresources.all()
 
         try:
             codebase_resource = codebase_resources.get(path=path)
-        except CodebaseResource.DoesNotExist:
+        except ObjectDoesNotExist:
             message = {"status": "Resource not found. Use ?path=<resource_path>"}
             return Response(message, status=status.HTTP_400_BAD_REQUEST)
 

scanpipe/models.py

@@ -453,13 +453,16 @@ class SaveProjectErrorMixin:
     Use `SaveProjectErrorMixin` on a model to create a ProjectError entry
     from a raised exception during `save()` in place of stopping the analysis
     process.
+    The creation of ProjectError can be skipped providing False for the `save_error`
+    argument.
     """
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, save_error=True, **kwargs):
         try:
             super().save(*args, **kwargs)
         except Exception as error:
-            self.add_error(error)
+            if save_error:
+                self.add_error(error)
 
     @classmethod
     def check(cls, **kwargs):
@@ -611,6 +614,9 @@ def status(self, status=None):
     def no_status(self):
         return self.filter(status="")
 
+    def empty(self):
+        return self.filter(Q(size__isnull=True) | Q(size=0))
+
     def in_package(self):
         return self.filter(discovered_packages__isnull=False)
 

scanpipe/pipes/__init__.py

@@ -33,35 +33,37 @@
 
 def make_codebase_resource(project, location, rootfs_path=None):
     """
-    Get or create and return a CodebaseResource with `location` absolute path
-    for the `project` Project.
+    Create a CodebaseResource with the `location` absolute path for the `project`.
 
     The `location` of this Resource must be rooted in `project.codebase_path`.
 
     `rootfs_path` is an optional path relative to a rootfs root within an
     Image/VM filesystem context. e.g.: "/var/log/file.log"
 
     All paths use the POSIX separators.
+
+    If a CodebaseResource already exists in the `project` for with the same path,
+    the error raised on save() is not stored in the database and the creation is
+    skipped.
     """
-    location = location.rstrip("/")
-    codebase_path = str(project.codebase_path)
-    assert location.startswith(
-        codebase_path
-    ), f"Location: {location} is not under project/codebase: {codebase_path}"
+    resource_location = location.rstrip("/")
+    codebase_dir = str(project.codebase_path)
+
+    assert resource_location.startswith(
+        codebase_dir
+    ), f"Location: {resource_location} is not under project/codebase/: {codebase_dir}"
 
-    path = location.replace(codebase_path, "")
+    resource_data = scancode.get_resource_info(location=resource_location)
 
-    resource_defaults = {}
     if rootfs_path:
-        resource_defaults["rootfs_path"] = rootfs_path
-    resource_defaults.update(scancode.get_resource_info(location=location))
+        resource_data["rootfs_path"] = rootfs_path
 
-    codebase_resource, _created = CodebaseResource.objects.get_or_create(
+    codebase_resource = CodebaseResource(
         project=project,
-        path=path,
-        defaults=resource_defaults,
+        path=resource_location.replace(codebase_dir, ""),
+        **resource_data,
     )
-    return codebase_resource
+    codebase_resource.save(save_error=False)
 
 
 def update_or_create_package(project, package_data):
@@ -137,22 +139,21 @@ def analyze_scanned_files(project):
     """
     Set the status for CodebaseResource with unknown or no licenses.
     """
-    queryset = CodebaseResource.objects.project(project).files().status("scanned")
-    queryset.has_no_licenses().update(status="no-licenses")
+    scanned_files = project.codebaseresources.files().status("scanned")
+
+    scanned_files.has_no_licenses().update(status="no-licenses")
 
-    for codebase_resource in queryset:
+    for codebase_resource in scanned_files:
         if has_unknown_license(codebase_resource):
             codebase_resource.status = "unknown-license"
             codebase_resource.save()
 
 
 def tag_not_analyzed_codebase_resources(project):
     """
-    Flag as "not-analyzed" the `CodebaseResource` without a status of the
-    provided `project`
+    Flag as "not-analyzed" the `CodebaseResource` without a status of the `project`.
     """
-    no_status = CodebaseResource.objects.project(project).no_status()
-    no_status.update(status="not-analyzed")
+    project.codebaseresources.no_status().update(status="not-analyzed")
 
 
 def normalize_path(path):

scanpipe/pipes/compliance.py

@@ -20,7 +20,6 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from scanpipe.models import CodebaseResource
 from scanpipe.pipes import scancode
 
 """
@@ -56,7 +55,7 @@ def analyze_compliance_licenses(project):
     """
     Scan compliance licenses status for the provided `project`.
     """
-    qs = CodebaseResource.objects.project(project).status("compliance-licenses")
+    qs = project.codebaseresources.status("compliance-licenses")
 
     for codebase_resource in qs:
         scan_results, scan_errors = scancode.scan_file(codebase_resource.location)

scanpipe/pipes/docker.py

@@ -26,10 +26,8 @@
 from pathlib import Path
 
 from container_inspector.image import Image
-from container_inspector.rootfs import get_whiteout_marker_type
 
 from scanpipe import pipes
-from scanpipe.models import CodebaseResource
 from scanpipe.pipes import rootfs
 from scanpipe.pipes import scancode
 
@@ -133,7 +131,7 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
         missing_resources = created_package.missing_resources[:]
         modified_resources = created_package.modified_resources[:]
 
-        codebase_resources = CodebaseResource.objects.project(project)
+        codebase_resources = project.codebaseresources.all()
 
         for install_file in package.installed_files:
             install_file_path = pipes.normalize_path(install_file.path)
@@ -194,12 +192,10 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
 
 def tag_whiteout_codebase_resources(project):
     """
-    Mark overlayfs/AUFS whiteout special files CodebaseResource as "ignored".
+    Mark overlayfs/AUFS whiteout special files CodebaseResource as "ignored-whiteout".
+    See https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
+    for details.
     """
+    whiteout_prefix = ".wh."
     qs = project.codebaseresources.no_status()
-
-    for codebase_resource in qs:
-        filename = Path(codebase_resource.path).name
-        if get_whiteout_marker_type(filename):
-            codebase_resource.status = "ignored-whiteout"
-            codebase_resource.save()
+    qs.filter(name__startswith=whiteout_prefix).update(status="ignored-whiteout")