diff --git a/Makefile b/Makefile index ca7ca26829..2863a98c6f 100644 --- a/Makefile +++ b/Makefile @@ -20,11 +20,66 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +######################################################################################## +# Docker dev commands +######################################################################################## + +IMAGE_NAME=scancodeio +COMPOSE=docker compose -f docker-compose.yml -f docker-compose.dev.yml +MANAGE=${COMPOSE} exec web ./manage.py + +run: + @echo "-> Run the Docker compose services in dev mode (hot reload on code changes)" + ${COMPOSE} up + +bash: + # Open a bash session in the running web container + ${COMPOSE} exec web bash + +shell: + # Open a bash session in a standalone container (no stack required) + docker run -it $(IMAGE_NAME) bash + +test: + @echo "-> Run the test suite" + ${MANAGE} test --noinput + +fasttest: + @echo "-> Run the test suite without the PipelinesIntegrationTest" + ${MANAGE} test --noinput --exclude-tag slow + +migrations: + @echo "-> Creates new database migrations" + ${MANAGE} makemigrations + +migrate: + @echo "-> Apply database migrations" + ${MANAGE} migrate + +restart-worker: + ${COMPOSE} restart worker + +build: + # Build the dev Docker images + ${COMPOSE} build + +build-full: + # Build the full production Docker image + docker build --target full -t $(IMAGE_NAME) . + +regen-fixtures: + @echo "-> Regenerate test fixtures from the running Docker stack" + ${COMPOSE} exec -e SCANCODEIO_TEST_FIXTURES_REGEN=1 web ./manage.py test + +######################################################################################## +# Local venv commands (legacy) +######################################################################################## + # Python version can be specified with `$ PYTHON_EXE=python3.x make conf` PYTHON_EXE?=python3 VENV_LOCATION=.venv ACTIVATE?=. ${VENV_LOCATION}/bin/activate; -MANAGE=${VENV_LOCATION}/bin/python manage.py +VENV_MANAGE=${VENV_LOCATION}/bin/python manage.py VIRTUALENV_PYZ=etc/thirdparty/virtualenv.pyz PIP_ARGS=--find-links=./etc/thirdparty/dummy_dist # Do not depend on Python to generate the SECRET_KEY @@ -37,7 +92,6 @@ SCANCODEIO_DB_USER=scancodeio SCANCODEIO_DB_PASSWORD=scancodeio POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 DATE=$(shell date +"%Y-%m-%d_%H%M") -IMAGE_NAME=scancodeio # Use sudo for postgres, only on Linux UNAME := $(shell uname) @@ -70,6 +124,13 @@ envfile: @mkdir -p $(shell dirname ${ENV_FILE}) && touch ${ENV_FILE} @echo SECRET_KEY=\"${GET_SECRET_KEY}\" > ${ENV_FILE} +runserver: + DJANGO_RUNSERVER_HIDE_WARNING=true ${VENV_MANAGE} runserver 8001 --insecure + +check-deploy: + @echo "-> Check Django deployment settings" + ${VENV_MANAGE} check --deploy + doc8: @echo "-> Run doc8 validation" @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ @@ -89,24 +150,11 @@ check: @echo "-> Run ABOUT files validation" @${ACTIVATE} about check --exclude .venv/ --exclude scanpipe/tests/ . -check-deploy: - @echo "-> Check Django deployment settings" - ${MANAGE} check --deploy - clean: @echo "-> Clean the Python env" rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/ find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -migrate: - @echo "-> Apply database migrations" - ${MANAGE} migrate - -upgrade: - @echo "-> Upgrade local git checkout" - @git pull - @$(MAKE) migrate - postgresdb: @echo "-> Configure PostgreSQL database" @echo "-> Create database user ${SCANCODEIO_DB_NAME}" @@ -127,21 +175,6 @@ sqlitedb: @echo SCANCODEIO_DB_NAME=\"sqlite3.db\" >> ${ENV_FILE} @$(MAKE) migrate -run: - DJANGO_RUNSERVER_HIDE_WARNING=true ${MANAGE} runserver 8001 --insecure - -run-docker-dev: - @echo "-> Run the Docker compose services in dev mode (hot reload on code changes)" - docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --watch - -test: - @echo "-> Run the test suite" - ${MANAGE} test --noinput - -fasttest: - @echo "-> Run the test suite without the PipelinesIntegrationTest" - ${MANAGE} test --noinput --exclude-tag slow - worker: ${MANAGE} rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 2 @@ -149,15 +182,6 @@ docs: rm -rf docs/_build/ @${ACTIVATE} sphinx-build docs/ docs/_build/ -build: - docker build --target base -t $(IMAGE_NAME) . - -build-full: - docker build --target full -t $(IMAGE_NAME) . - -bash: - docker run -it $(IMAGE_NAME) bash - docker-images: @echo "-> Build Docker services" docker compose build @@ -174,4 +198,4 @@ offline-package: docker-images @mkdir -p dist/ @tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/ -.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run run-docker-dev test fasttest docs build bash docker-images offline-package +.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate makemigrations restart-worker postgresdb sqlitedb backupdb run test fasttest regen-fixtures docs build bash shell docker-images offline-package diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1d83e672d6..b5f2fa3c55 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,16 +1,12 @@ -# Dev mode, https://docs.docker.com/compose/how-tos/file-watch/ - -# Mount the local scanpipe/ directory in the containers -# This can be used to refresh fixtures from the docker container: -# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml up -# $ docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm web bash -# $ SCANCODEIO_TEST_FIXTURES_REGEN=1 ./manage.py test +# Dev mode: overlay for local development with hot reload. +# Usage: $ make run +# See the Makefile for all available commands (test, migrate, bash, etc.) name: scancodeio x-dev-env: &dev-env SCANCODEIO_DEBUG: "True" - GUNICORN_RELOAD_FLAG: "--reload" + DJANGO_RUNSERVER_HIDE_WARNING: "true" x-dev-build: &dev-build context: . @@ -21,15 +17,29 @@ services: build: *dev-build environment: <<: *dev-env + command: ./manage.py runserver --skip-checks 0.0.0.0:8001 volumes: - ./scanpipe:/opt/scancodeio/scanpipe + ports: + - "8001:8001" + # Volume mount keeps code in sync. Restart manually with: make restart-worker worker: build: *dev-build environment: <<: *dev-env - develop: - watch: - - action: sync+restart - path: ./scanpipe - target: /opt/scancodeio/scanpipe + command: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker + --queue-class scancodeio.worker.ScanCodeIOQueue + --verbosity 1 + volumes: + - ./scanpipe:/opt/scancodeio/scanpipe + + # Disable nginx in dev mode, the runserver serves requests directly. + nginx: + profiles: + - production + + # Disable clamav in dev mode, not needed for local development. + clamav: + profiles: + - production diff --git a/docker-compose.yml b/docker-compose.yml index 426ae80bfe..18b82588b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: # # For fresh installations (no existing data), this check passes automatically. db-check: - image: docker.io/library/postgres:17 + image: docker.io/library/postgres:17.9 volumes: - db_data:/var/lib/postgresql/data/:ro entrypoint: [ "/bin/bash", "-c" ] @@ -46,7 +46,7 @@ services: restart: "no" db: - image: docker.io/library/postgres:17 + image: docker.io/library/postgres:17.9 depends_on: db-check: condition: service_completed_successfully @@ -63,7 +63,7 @@ services: retries: 5 redis: - image: docker.io/library/redis:latest + image: docker.io/library/redis:8.6-alpine # Enable redis data persistence using the "Append Only File" with the # default policy of fsync every second. See https://redis.io/topics/persistence command: redis-server --appendonly yes @@ -112,7 +112,7 @@ services: - web nginx: - image: docker.io/library/nginx:alpine + image: docker.io/library/nginx:1.29-alpine ports: - "${NGINX_PUBLISHED_HTTP_PORT:-80}:80" - "${NGINX_PUBLISHED_HTTPS_PORT:-443}:443" @@ -125,7 +125,7 @@ services: restart: always clamav: - image: docker.io/clamav/clamav:latest + image: docker.io/clamav/clamav:1.5_base volumes: - clamav_data:/var/lib/clamav - workspace:/var/scancodeio/workspace/ diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index d68c732747..7468bc4822 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -265,7 +265,10 @@ def test_scanpipe_pipeline_class_download_fetch_exception(self, mock_fetch): self.assertIn("https://download.url/file.zip could not be fetched.", run.log) @mock.patch("git.repo.base.Repo.clone_from") - def test_scanpipe_pipeline_class_download_missing_inputs_git_repo(self, mock_clone): + @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True) + def test_scanpipe_pipeline_class_download_missing_inputs_git_repo( + self, mock_check_url, mock_clone + ): project1 = make_project() run = project1.add_pipeline("do_nothing") pipeline = run.make_pipeline_instance() @@ -818,7 +821,10 @@ def test_scanpipe_scan_package_single_file(self): self.assertPipelineResultEqual(expected_file, scancode_file) @mock.patch("git.repo.base.Repo.clone_from") - def test_scanpipe_scan_package_single_package_git_repo(self, mock_clone): + @mock.patch("scanpipe.pipes.fetch.check_url", return_value=True) + def test_scanpipe_scan_package_single_package_git_repo( + self, mock_check_url, mock_clone + ): pipeline_name = "scan_single_package" project1 = make_project()