feat!: Switch development workflow from local venv to Docker Compose (#2145)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

Makefile

@@ -20,11 +20,66 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+########################################################################################
+# Docker dev commands
+########################################################################################
+
+IMAGE_NAME=scancodeio
+COMPOSE=docker compose -f docker-compose.yml -f docker-compose.dev.yml
+MANAGE=${COMPOSE} exec web ./manage.py
+
+run:
+	@echo "-> Run the Docker compose services in dev mode (hot reload on code changes)"
+	${COMPOSE} up
+
+bash:
+	# Open a bash session in the running web container
+	${COMPOSE} exec web bash
+
+shell:
+	# Open a bash session in a standalone container (no stack required)
+	docker run -it $(IMAGE_NAME) bash
+
+test:
+	@echo "-> Run the test suite"
+	${MANAGE} test --noinput
+
+fasttest:
+	@echo "-> Run the test suite without the PipelinesIntegrationTest"
+	${MANAGE} test --noinput --exclude-tag slow
+
+migrations:
+	@echo "-> Creates new database migrations"
+	${MANAGE} makemigrations
+
+migrate:
+	@echo "-> Apply database migrations"
+	${MANAGE} migrate
+
+restart-worker:
+	${COMPOSE} restart worker
+
+build:
+	# Build the dev Docker images
+	${COMPOSE} build
+
+build-full:
+	# Build the full production Docker image
+	docker build --target full -t $(IMAGE_NAME) .
+
+regen-fixtures:
+	@echo "-> Regenerate test fixtures from the running Docker stack"
+	${COMPOSE} exec -e SCANCODEIO_TEST_FIXTURES_REGEN=1 web ./manage.py test
+
+########################################################################################
+# Local venv commands (legacy)
+########################################################################################
+
 # Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
 PYTHON_EXE?=python3
 VENV_LOCATION=.venv
 ACTIVATE?=. ${VENV_LOCATION}/bin/activate;
-MANAGE=${VENV_LOCATION}/bin/python manage.py
+VENV_MANAGE=${VENV_LOCATION}/bin/python manage.py
 VIRTUALENV_PYZ=etc/thirdparty/virtualenv.pyz
 PIP_ARGS=--find-links=./etc/thirdparty/dummy_dist
 # Do not depend on Python to generate the SECRET_KEY
@@ -37,7 +92,6 @@ SCANCODEIO_DB_USER=scancodeio
 SCANCODEIO_DB_PASSWORD=scancodeio
 POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
 DATE=$(shell date +"%Y-%m-%d_%H%M")
-IMAGE_NAME=scancodeio
 
 # Use sudo for postgres, only on Linux
 UNAME := $(shell uname)
@@ -70,6 +124,13 @@ envfile:
 	@mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE}
 	@echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE}
 
+runserver:
+	DJANGO_RUNSERVER_HIDE_WARNING=true ${VENV_MANAGE} runserver 8001 --insecure
+
+check-deploy:
+	@echo "-> Check Django deployment settings"
+	${VENV_MANAGE} check --deploy
+
 doc8:
 	@echo "-> Run doc8 validation"
 	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/
@@ -89,24 +150,11 @@ check:
 	@echo "-> Run ABOUT files validation"
 	@${ACTIVATE} about check --exclude .venv/ --exclude scanpipe/tests/ .
 
-check-deploy:
-	@echo "-> Check Django deployment settings"
-	${MANAGE} check --deploy
-
 clean:
 	@echo "-> Clean the Python env"
 	rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/
 	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 
-migrate:
-	@echo "-> Apply database migrations"
-	${MANAGE} migrate
-
-upgrade:
-	@echo "-> Upgrade local git checkout"
-	@git pull
-	@$(MAKE) migrate
-
 postgresdb:
 	@echo "-> Configure PostgreSQL database"
 	@echo "-> Create database user ${SCANCODEIO_DB_NAME}"
@@ -127,37 +175,13 @@ sqlitedb:
 	@echo SCANCODEIO_DB_NAME=\"sqlite3.db\" >> ${ENV_FILE}
 	@$(MAKE) migrate
 
-run:
-	DJANGO_RUNSERVER_HIDE_WARNING=true ${MANAGE} runserver 8001 --insecure
-
-run-docker-dev:
-	@echo "-> Run the Docker compose services in dev mode (hot reload on code changes)"
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --watch
-
-test:
-	@echo "-> Run the test suite"
-	${MANAGE} test --noinput
-
-fasttest:
-	@echo "-> Run the test suite without the PipelinesIntegrationTest"
-	${MANAGE} test --noinput --exclude-tag slow
-
 worker:
 	${MANAGE} rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 2
 
 docs:
 	rm -rf docs/_build/
 	@${ACTIVATE} sphinx-build docs/ docs/_build/
 
-build:
-	docker build --target base -t $(IMAGE_NAME) .
-
-build-full:
-	docker build --target full -t $(IMAGE_NAME) .
-
-bash:
-	docker run -it $(IMAGE_NAME) bash
-
 docker-images:
 	@echo "-> Build Docker services"
 	docker compose build
@@ -174,4 +198,4 @@ offline-package: docker-images
 	@mkdir -p dist/
 	@tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/
 
-.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run run-docker-dev test fasttest docs build bash docker-images offline-package
+.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate makemigrations restart-worker postgresdb sqlitedb backupdb run test fasttest regen-fixtures docs build bash shell docker-images offline-package

docker-compose.dev.yml

@@ -1,16 +1,12 @@
-# Dev mode, https://docs.docker.com/compose/how-tos/file-watch/
-
-# Mount the local scanpipe/ directory in the containers
-# This can be used to refresh fixtures from the docker container:
-# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml up
-# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm web bash
-# $ SCANCODEIO_TEST_FIXTURES_REGEN=1 ./manage.py test
+# Dev mode: overlay for local development with hot reload.
+# Usage: $ make run
+# See the Makefile for all available commands (test, migrate, bash, etc.)
 
 name: scancodeio
 
 x-dev-env: &dev-env
   SCANCODEIO_DEBUG: "True"
-  GUNICORN_RELOAD_FLAG: "--reload"
+  DJANGO_RUNSERVER_HIDE_WARNING: "true"
 
 x-dev-build: &dev-build
   context: .
@@ -21,15 +17,29 @@ services:
     build: *dev-build
     environment:
       <<: *dev-env
+    command: ./manage.py runserver --skip-checks 0.0.0.0:8001
     volumes:
       - ./scanpipe:/opt/scancodeio/scanpipe
+    ports:
+      - "8001:8001"
 
+  # Volume mount keeps code in sync. Restart manually with: make restart-worker
   worker:
     build: *dev-build
     environment:
       <<: *dev-env
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./scanpipe
-          target: /opt/scancodeio/scanpipe
+    command: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker
+                                  --queue-class scancodeio.worker.ScanCodeIOQueue
+                                  --verbosity 1
+    volumes:
+      - ./scanpipe:/opt/scancodeio/scanpipe
+
+  # Disable nginx in dev mode, the runserver serves requests directly.
+  nginx:
+    profiles:
+      - production
+
+  # Disable clamav in dev mode, not needed for local development.
+  clamav:
+    profiles:
+      - production

docker-compose.yml

@@ -12,7 +12,7 @@ services:
   #
   # For fresh installations (no existing data), this check passes automatically.
   db-check:
-    image: docker.io/library/postgres:17
+    image: docker.io/library/postgres:17.9
     volumes:
       - db_data:/var/lib/postgresql/data/:ro
     entrypoint: [ "/bin/bash", "-c" ]
@@ -46,7 +46,7 @@ services:
     restart: "no"
 
   db:
-    image: docker.io/library/postgres:17
+    image: docker.io/library/postgres:17.9
     depends_on:
       db-check:
         condition: service_completed_successfully
@@ -63,7 +63,7 @@ services:
       retries: 5
 
   redis:
-    image: docker.io/library/redis:latest
+    image: docker.io/library/redis:8.6-alpine
     # Enable redis data persistence using the "Append Only File" with the
     # default policy of fsync every second. See https://redis.io/topics/persistence
     command: redis-server --appendonly yes
@@ -112,7 +112,7 @@ services:
       - web
 
   nginx:
-    image: docker.io/library/nginx:alpine
+    image: docker.io/library/nginx:1.29-alpine
     ports:
       - "${NGINX_PUBLISHED_HTTP_PORT:-80}:80"
       - "${NGINX_PUBLISHED_HTTPS_PORT:-443}:443"
@@ -125,7 +125,7 @@ services:
     restart: always
 
   clamav:
-    image: docker.io/clamav/clamav:latest
+    image: docker.io/clamav/clamav:1.5_base
     volumes:
       - clamav_data:/var/lib/clamav
       - workspace:/var/scancodeio/workspace/

scanpipe/tests/test_pipelines.py

@@ -265,7 +265,10 @@ def test_scanpipe_pipeline_class_download_fetch_exception(self, mock_fetch):
         self.assertIn("https://download.url/file.zip could not be fetched.", run.log)
 
     @mock.patch("git.repo.base.Repo.clone_from")
-    def test_scanpipe_pipeline_class_download_missing_inputs_git_repo(self, mock_clone):
+    @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True)
+    def test_scanpipe_pipeline_class_download_missing_inputs_git_repo(
+        self, mock_check_url, mock_clone
+    ):
         project1 = make_project()
         run = project1.add_pipeline("do_nothing")
         pipeline = run.make_pipeline_instance()
@@ -820,7 +823,10 @@ def test_scanpipe_scan_package_single_file(self):
         self.assertPipelineResultEqual(expected_file, scancode_file)
 
     @mock.patch("git.repo.base.Repo.clone_from")
-    def test_scanpipe_scan_package_single_package_git_repo(self, mock_clone):
+    @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True)
+    def test_scanpipe_scan_package_single_package_git_repo(
+        self, mock_check_url, mock_clone
+    ):
         pipeline_name = "scan_single_package"
         project1 = make_project()