feat!: Switch development workflow from local venv to Docker Compose #2145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

tdruez merged 4 commits into main from docker-dev-mode

Apr 10, 2026

Makefile

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -20,11 +20,66 @@
  
    # ScanCode.io is a free software code scanning tool from nexB Inc. and others.

    # Visit https://github.com/aboutcode-org/scancode.io for support and download.

    ########################################################################################

    # Docker dev commands

    ########################################################################################

    IMAGE_NAME=scancodeio

    COMPOSE=docker compose -f docker-compose.yml -f docker-compose.dev.yml

    MANAGE=${COMPOSE} exec web ./manage.py

    run:

    	@echo "-> Run the Docker compose services in dev mode (hot reload on code changes)"

    	${COMPOSE} up

    bash:

    	# Open a bash session in the running web container

    	${COMPOSE} exec web bash

    shell:

    	# Open a bash session in a standalone container (no stack required)

    	docker run -it $(IMAGE_NAME) bash

    test:

    	@echo "-> Run the test suite"

    	${MANAGE} test --noinput

    fasttest:

    	@echo "-> Run the test suite without the PipelinesIntegrationTest"

    	${MANAGE} test --noinput --exclude-tag slow

    migrations:

    	@echo "-> Creates new database migrations"

    	${MANAGE} makemigrations

    migrate:

    	@echo "-> Apply database migrations"

    	${MANAGE} migrate

    restart-worker:

    	${COMPOSE} restart worker

    build:

    	# Build the dev Docker images

    	${COMPOSE} build

    build-full:

    	# Build the full production Docker image

    	docker build --target full -t $(IMAGE_NAME) .

    regen-fixtures:

    	@echo "-> Regenerate test fixtures from the running Docker stack"

    	${COMPOSE} exec -e SCANCODEIO_TEST_FIXTURES_REGEN=1 web ./manage.py test

    ########################################################################################

    # Local venv commands (legacy)

    ########################################################################################

    # Python version can be specified with `$ PYTHON_EXE=python3.x make conf`

    PYTHON_EXE?=python3

    VENV_LOCATION=.venv

    ACTIVATE?=. ${VENV_LOCATION}/bin/activate;

    MANAGE=${VENV_LOCATION}/bin/python manage.py

    VENV_MANAGE=${VENV_LOCATION}/bin/python manage.py

    VIRTUALENV_PYZ=etc/thirdparty/virtualenv.pyz

    PIP_ARGS=--find-links=./etc/thirdparty/dummy_dist

    # Do not depend on Python to generate the SECRET_KEY

    @@ -37,7 +92,6 @@ SCANCODEIO_DB_USER=scancodeio
  
    SCANCODEIO_DB_PASSWORD=scancodeio

    POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8

    DATE=$(shell date +"%Y-%m-%d_%H%M")

    IMAGE_NAME=scancodeio

    # Use sudo for postgres, only on Linux

    UNAME := $(shell uname)

    @@ -70,6 +124,13 @@ envfile:
  
    	@mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE}

    	@echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE}

    runserver:

    	DJANGO_RUNSERVER_HIDE_WARNING=true ${VENV_MANAGE} runserver 8001 --insecure

    check-deploy:

    	@echo "-> Check Django deployment settings"

    	${VENV_MANAGE} check --deploy

    doc8:

    	@echo "-> Run doc8 validation"

    	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/

    @@ -89,24 +150,11 @@ check:
  
    	@echo "-> Run ABOUT files validation"

    	@${ACTIVATE} about check --exclude .venv/ --exclude scanpipe/tests/ .

    check-deploy:

    	@echo "-> Check Django deployment settings"

    	${MANAGE} check --deploy

    clean:

    	@echo "-> Clean the Python env"

    	rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/

    	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete

    migrate:

    	@echo "-> Apply database migrations"

    	${MANAGE} migrate

    upgrade:

    	@echo "-> Upgrade local git checkout"

    	@git pull

    	@$(MAKE) migrate

    postgresdb:

    	@echo "-> Configure PostgreSQL database"

    	@echo "-> Create database user ${SCANCODEIO_DB_NAME}"

    @@ -127,37 +175,13 @@ sqlitedb:
  
    	@echo SCANCODEIO_DB_NAME=\"sqlite3.db\" >> ${ENV_FILE}

    	@$(MAKE) migrate

    run:

    	DJANGO_RUNSERVER_HIDE_WARNING=true ${MANAGE} runserver 8001 --insecure

    run-docker-dev:

    	@echo "-> Run the Docker compose services in dev mode (hot reload on code changes)"

    	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --watch

    test:

    	@echo "-> Run the test suite"

    	${MANAGE} test --noinput

    fasttest:

    	@echo "-> Run the test suite without the PipelinesIntegrationTest"

    	${MANAGE} test --noinput --exclude-tag slow

    worker:

    	${MANAGE} rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 2

    docs:

    	rm -rf docs/_build/

    	@${ACTIVATE} sphinx-build docs/ docs/_build/

    build:

    	docker build --target base -t $(IMAGE_NAME) .

    build-full:

    	docker build --target full -t $(IMAGE_NAME) .

    bash:

    	docker run -it $(IMAGE_NAME) bash

    docker-images:

    	@echo "-> Build Docker services"

    	docker compose build

    @@ -174,4 +198,4 @@ offline-package: docker-images
  
    	@mkdir -p dist/

    	@tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/

    .PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run run-docker-dev test fasttest docs build bash docker-images offline-package

    .PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate makemigrations restart-worker postgresdb sqlitedb backupdb run test fasttest regen-fixtures docs build bash shell docker-images offline-package

docker-compose.dev.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,16 +1,12 @@
  
    # Dev mode, https://docs.docker.com/compose/how-tos/file-watch/

    # Mount the local scanpipe/ directory in the containers

    # This can be used to refresh fixtures from the docker container:

    # $ docker compose -f docker-compose.yml -f docker-compose.dev.yml up

    # $ docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm web bash

    # $ SCANCODEIO_TEST_FIXTURES_REGEN=1 ./manage.py test

    # Dev mode: overlay for local development with hot reload.

    # Usage: $ make run

    # See the Makefile for all available commands (test, migrate, bash, etc.)

    name: scancodeio

    x-dev-env: &dev-env

      SCANCODEIO_DEBUG: "True"

      GUNICORN_RELOAD_FLAG: "--reload"

      DJANGO_RUNSERVER_HIDE_WARNING: "true"

    x-dev-build: &dev-build

      context: .

    @@ -21,15 +17,29 @@ services:
  
        build: *dev-build

        environment:

          <<: *dev-env

        command: ./manage.py runserver --skip-checks 0.0.0.0:8001

        volumes:

          - ./scanpipe:/opt/scancodeio/scanpipe

        ports:

          - "8001:8001"

      # Volume mount keeps code in sync. Restart manually with: make restart-worker

      worker:

        build: *dev-build

        environment:

          <<: *dev-env

        develop:

          watch:

            - action: sync+restart

              path: ./scanpipe

              target: /opt/scancodeio/scanpipe

        command: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker

                                      --queue-class scancodeio.worker.ScanCodeIOQueue

                                      --verbosity 1

        volumes:

          - ./scanpipe:/opt/scancodeio/scanpipe

      # Disable nginx in dev mode, the runserver serves requests directly.

      nginx:

        profiles:

          - production

      # Disable clamav in dev mode, not needed for local development.

      clamav:

        profiles:

          - production

docker-compose.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -12,7 +12,7 @@ services:
  
      #

      # For fresh installations (no existing data), this check passes automatically.

      db-check:

        image: docker.io/library/postgres:17

        image: docker.io/library/postgres:17.9

        volumes:

          - db_data:/var/lib/postgresql/data/:ro

        entrypoint: [ "/bin/bash", "-c" ]

    @@ -46,7 +46,7 @@ services:
  
        restart: "no"

      db:

        image: docker.io/library/postgres:17

        image: docker.io/library/postgres:17.9

        depends_on:

          db-check:

            condition: service_completed_successfully

    @@ -63,7 +63,7 @@ services:
  
          retries: 5

      redis:

        image: docker.io/library/redis:latest

        image: docker.io/library/redis:8.6-alpine

        # Enable redis data persistence using the "Append Only File" with the

        # default policy of fsync every second. See https://redis.io/topics/persistence

        command: redis-server --appendonly yes

    @@ -112,7 +112,7 @@ services:
  
          - web

      nginx:

        image: docker.io/library/nginx:alpine

        image: docker.io/library/nginx:1.29-alpine

        ports:

          - "${NGINX_PUBLISHED_HTTP_PORT:-80}:80"

          - "${NGINX_PUBLISHED_HTTPS_PORT:-443}:443"

    @@ -125,7 +125,7 @@ services:
  
        restart: always

      clamav:

        image: docker.io/clamav/clamav:latest

        image: docker.io/clamav/clamav:1.5_base

        volumes:

          - clamav_data:/var/lib/clamav

          - workspace:/var/scancodeio/workspace/

scanpipe/tests/test_pipelines.py

-Original file line number
+Diff line change
@@ Expand Up @@
             self.assertIn("https://download.url/file.zip could not be fetched.", run.log)
         @mock.patch("git.repo.base.Repo.clone_from")
-        def test_scanpipe_pipeline_class_download_missing_inputs_git_repo(self, mock_clone):
+        @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True)
+        def test_scanpipe_pipeline_class_download_missing_inputs_git_repo(
+            self, mock_check_url, mock_clone
+        ):
             project1 = make_project()
             run = project1.add_pipeline("do_nothing")
             pipeline = run.make_pipeline_instance()
@@ Expand Down Expand Up / @@ -818,7 +821,10 @@ def test_scanpipe_scan_package_single_file(self): @@
             self.assertPipelineResultEqual(expected_file, scancode_file)
         @mock.patch("git.repo.base.Repo.clone_from")
-        def test_scanpipe_scan_package_single_package_git_repo(self, mock_clone):
+        @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True)
+        def test_scanpipe_scan_package_single_package_git_repo(
+            self, mock_check_url, mock_clone
+        ):
             pipeline_name = "scan_single_package"
             project1 = make_project()
@@ Expand Down @@

Navigation Menu

Uh oh!

Uh oh!

feat!: Switch development workflow from local venv to Docker Compose #2145

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!