diff --git a/Dockerfile b/Dockerfile index c6216aa69a..452cf37e66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,6 +145,7 @@ RUN apt-get update \ libguestfs-tools \ linux-image-amd64 \ openjdk-17-jre-headless \ + docker.io \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docker-compose.yml b/docker-compose.yml index 9da2c248eb..5a3bf408e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,16 @@ services: timeout: 5s retries: 5 + # Add the Docker-in-Docker daemon + dind: + image: docker.io/library/docker:dind + privileged: true + environment: + - DOCKER_TLS_CERTDIR= + volumes: + - dind_data:/var/lib/docker + - workspace:/var/scancodeio/workspace/ + web: build: . command: sh -c " @@ -38,6 +48,8 @@ services: ./manage.py collectstatic --no-input --verbosity 0 --clear && gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 \ --workers ${GUNICORN_WORKERS:-8} --worker-tmp-dir /dev/shm" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container env_file: - docker.env expose: @@ -61,6 +73,8 @@ services: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 1" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container env_file: - docker.env volumes: @@ -75,6 +89,8 @@ services: condition: service_healthy web: condition: service_started + dind: + condition: service_started nginx: image: docker.io/library/nginx:1.31.4-alpine @@ -104,3 +120,4 @@ volumes: static: workspace: webroot: + dind_data: diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 644f81ee03..4368ab4eb3 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -281,6 +281,12 @@ Scan Maven Package :members: :member-order: bysource +Scan Nix Package +------------------- +.. autoclass:: scanpipe.pipelines.scan_nix_package.ScanNixPackage() + :members: + :member-order: bysource + Fetch Scores (addon) -------------------- .. warning:: diff --git a/pyproject.toml b/pyproject.toml index 6256fa28f3..f8ba8bb755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage" +scan_nix_package = "scanpipe.pipelines.scan_nix_package:ScanNixPackage" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" [tool.setuptools.packages.find] diff --git a/scanpipe/pipelines/__init__.py b/scanpipe/pipelines/__init__.py index 53f145d50c..267433ec72 100644 --- a/scanpipe/pipelines/__init__.py +++ b/scanpipe/pipelines/__init__.py @@ -107,14 +107,14 @@ def extract_archive(self, location, target): details=details, ) - def extract_archives(self, location=None): + def extract_archives(self, location=None, recurse=True): """Extract archives located in the codebase/ directory with extractcode.""" from scanpipe.pipes import scancode if not location: location = self.project.codebase_path - extract_errors = scancode.extract_archives(location=location, recurse=True) + extract_errors = scancode.extract_archives(location=location, recurse=recurse) for resource_path, errors in extract_errors.items(): self.project.add_error( diff --git a/scanpipe/pipelines/scan_nix_package.py b/scanpipe/pipelines/scan_nix_package.py new file mode 100644 index 0000000000..3c7e143898 --- /dev/null +++ b/scanpipe/pipelines/scan_nix_package.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import shutil +from pathlib import Path + +from scanpipe.pipelines.deploy_to_develop import DeployToDevelop +from scanpipe.pipelines.scan_codebase import ScanCodebase +from scanpipe.pipelines.scan_single_package import ScanSinglePackage +from scanpipe.pipes import d2d +from scanpipe.pipes import flag +from scanpipe.pipes import nix +from scanpipe.pipes import utils +from scanpipe.pipes.nix import check_input_and_return_purl +from scanpipe.pipes.nix import cleanup_docker_volumes +from scanpipe.pipes.nix import fetch_inputs + + +class ScanNixPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): + """ + Download the nix source and binary, and run a deployment to development + scan between the binary and the source to detect any discrepancies. + + Scan the sources and confirm that the detected license aligns with + the declared license that is detected from the nix package. + """ + + download_inputs = False + + @classmethod + def steps(cls): + return ( + cls.check_input_and_return_purl, + cls.check_docker_command, + cls.fetch_inputs, + cls.collect_input_info, + cls.extract_input_to_codebase_directory, + cls.extract_codebase_archives, + cls.collect_and_create_codebase_resources, + cls.scan_for_application_packages, + cls.scan_for_files, + cls.clear_to_codebase_status, + cls.collect_and_create_license_detections, + cls.add_from_to_tag, + cls.d2d_steps, + cls.validate_package_license_integrity, + cls.cleanup_docker_volumes, + ) + + def check_input_and_return_purl(self): + """Validate the input is a PURL string and return the PURL object.""" + self.purl = check_input_and_return_purl(self.project) + + def check_docker_command(self): + """Check if the Docker command is available.""" + if not utils.check_docker_command(): + raise Exception("Docker is required and its daemon must be running.") + nix.ensure_multiarch_emulation() + + def fetch_inputs(self): + """Fetch the binary and source of the given PURL.""" + from_file = "" + to_file = "" + output_format = "" + from_file, to_file, output_format, error_messages, warning_messages = ( + fetch_inputs(self.purl, self.project.codebase_path) + ) + self.from_file = from_file + self.to_file = to_file + self.output_format = output_format + + self.d2d_enable = bool(self.from_file and self.to_file) + + if error_messages: + self.project.add_error(error_messages) + if warning_messages: + self.project.add_warning(warning_messages) + + def collect_input_info(self): + """Collect information about the input.""" + self.input_path = "" + if self.to_file: + self.input_path = Path(self.to_file) + self.collect_input_information() + + def extract_input_to_codebase_directory(self): + """Extract input to project codebase/ directory.""" + if self.input_path: + extracted_path = nix.extract_nar_archive( + self.input_path, self.project.codebase_path, self.output_format + ) + + to_dir = Path(self.project.codebase_path) / "to" + # If the extraction failed (returned "") or we found it was empty + if not extracted_path or (to_dir.exists() and not list(to_dir.rglob("*"))): + if to_dir.exists(): + shutil.rmtree(to_dir) + self.d2d_enable = False + self.project.add_error( + "Failed to extract NAR archive, D2D scan disabled." + ) + + self.env = self.project.get_env() + + def extract_codebase_archives(self): + """Perform extraction of the codebase resources""" + self.extract_archives(recurse=True) + + def clear_to_codebase_status(self): + """ + Clear the status of the to codebase resources in the project as + having status will prevent D2D from running. + """ + flag.clear_status(self.project.codebaseresources.to_codebase()) + + def add_from_to_tag(self): + """Update 'from' and 'to' tag to resources based on their path.""" + if self.d2d_enable: + d2d.update_from_to_tag(self.project) + + def d2d_steps(self): + """ + Run the deployment to development scan if both the source and + binary are available. + """ + if self.d2d_enable: + self.flag_empty_files() + self.flag_whitespace_files() + self.flag_ignored_resources() + self.map_about_files() + self.map_checksum() + self.match_archives_to_purldb() + self.load_ecosystem_config() + self.d2d_java() + self.d2d_scala() + self.d2d_kotlin() + self.d2d_grammar() + self.d2d_groovy() + self.d2d_aspectj() + self.d2d_clojure() + self.d2d_xtend() + self.d2d_javascript() + self.d2d_process() + + def d2d_java(self): + self.find_java_packages() + self.map_java_to_class() + self.map_jar_to_java_source() + + def d2d_scala(self): + self.find_scala_packages() + self.map_scala_to_class() + self.map_jar_to_scala_source() + + def d2d_kotlin(self): + self.find_kotlin_packages() + self.map_kotlin_to_class() + self.map_jar_to_kotlin_source() + + def d2d_grammar(self): + self.find_grammar_packages() + self.map_grammar_to_class() + self.map_jar_to_grammar_source() + + def d2d_groovy(self): + self.find_groovy_packages() + self.map_groovy_to_class() + self.map_jar_to_groovy_source() + + def d2d_aspectj(self): + self.find_aspectj_packages() + self.map_aspectj_to_class() + self.map_jar_to_aspectj_source() + + def d2d_clojure(self): + self.find_clojure_packages() + self.map_clojure_to_class() + self.map_jar_to_clojure_source() + + def d2d_xtend(self): + self.find_xtend_packages() + self.map_xtend_to_class() + + def d2d_javascript(self): + self.map_javascript() + self.map_javascript_symbols() + self.map_javascript_strings() + + def d2d_process(self): + self.get_symbols_from_binaries() + self.map_elf() + self.map_macho() + self.map_winpe() + self.map_go() + self.map_rust() + self.map_python() + self.match_directories_to_purldb() + self.match_resources_to_purldb() + self.map_javascript_post_purldb_match() + self.map_javascript_path() + self.map_javascript_colocation() + self.map_thirdparty_npm_packages() + self.map_path() + self.flag_mapped_resources_archives_and_ignored_directories() + self.perform_house_keeping_tasks() + self.match_purldb_resources_post_process() + self.remove_packages_without_resources() + self.flag_deployed_from_resources_with_missing_license() + self.create_local_files_packages() + + def validate_package_license_integrity(self): + """ + Validate the correctness of the package license compare with the + detected license from the codebase. + """ + utils.validate_package_license_integrity(self.project) + + def flag_mapped_status(self): + """Flag the from codebase resources that were mapped.""" + if self.d2d_enable: + flag.flag_mapped_resources(self.project) + + def cleanup_docker_volumes(self): + """Cleanup the Docker volumes used for Nix.""" + cleanup_docker_volumes() diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index eda7703e38..51e152103f 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -141,10 +141,14 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field): def map_checksum(project, checksum_field, logger=None): """Map using checksum.""" - project_files = project.codebaseresources.files().no_status() - from_resources = project_files.from_codebase().has_value(checksum_field) + from_resources = ( + project.codebaseresources.files().from_codebase().has_value(checksum_field) + ) to_resources = ( - project_files.to_codebase().has_value(checksum_field).has_no_relation() + project.codebaseresources.files() + .to_codebase() + .has_value(checksum_field) + .has_no_relation() ) resource_count = to_resources.count() @@ -271,7 +275,7 @@ def find_jvm_packages(project, jvm_lang: jvm.JvmLanguage, logger=None): Note: we use the same API as the ScanCode scans by design """ - resources = project.codebaseresources.files().no_status().from_codebase() + resources = project.codebaseresources.files().from_codebase() from_jvm_resources = resources.filter(extension__in=jvm_lang.source_extensions) @@ -413,9 +417,8 @@ def _map_path_resource( def map_path(project, logger=None): """Map using path suffix similarities.""" - project_files = project.codebaseresources.files().no_status() - from_resources = project_files.from_codebase() - to_resources = project_files.to_codebase().has_no_relation() + from_resources = project.codebaseresources.files().from_codebase() + to_resources = project.codebaseresources.files().to_codebase().has_no_relation() resource_count = to_resources.count() if logger: @@ -1347,6 +1350,16 @@ def flag_processed_archives(project): for archive_resource in to_resources.archives(): extract_path = archive_resource.path + EXTRACT_SUFFIX + + # Skip archives that were not actually extracted to prevent getting + # flagged as "processed" (archives that's not supported by + # extractcode). + extracted_exists = project.codebaseresources.filter( + path__startswith=extract_path + ).exists() + if not extracted_exists: + continue + archive_unmapped_resources = to_resources.filter(path__startswith=extract_path) # Check if all resources in the archive "-extract" directory have been mapped. # Flag the archive resource as processed only when all resources are mapped. @@ -1767,20 +1780,25 @@ def map_paths_resource( relations_to_create[rel_key] = relation if paths_not_mapped: to_resource.status = flag.REQUIRES_REVIEW - logger( - f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT mapped for: " - f"{to_resource.path!r}" - ) + if logger: + logger( + f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT " + f" mapped for: {to_resource.path!r}" + ) to_resource.save() if relations_to_create: rels = CodebaseRelation.objects.bulk_create(relations_to_create.values()) - logger( - f"Created {len(rels)} mappings using " - f"{', '.join(map_types)} for: {to_resource.path!r}" - ) + if logger: + logger( + f"Created {len(rels)} mappings using " + f"{', '.join(map_types)} for: {to_resource.path!r}" + ) else: - logger(f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}") + if logger: + logger( + f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}" + ) def process_paths_in_binary( @@ -1944,9 +1962,17 @@ def map_elfs_with_dwarf_paths(project, logger=None): f"with {from_resources.count():,d} from/ resources." ) - from_resources_index = pathmap.build_index( - from_resources.values_list("id", "path"), with_subpaths=True - ) + # Build the path index, adding virtual aliases for .in template files + from_paths = [] + for res_id, path in from_resources.values_list("id", "path"): + from_paths.append((res_id, path)) + # If the source file is a template ending in '.in', also index its + # target name + if path.endswith(".in"): + target_path = path[:-3] # Removes the trailing '.in' + from_paths.append((res_id, target_path)) + + from_resources_index = pathmap.build_index(from_paths, with_subpaths=True) if logger: logger("Done building from/ resources index.") @@ -2035,6 +2061,15 @@ def map_go_paths(project, logger=None): ) +def update_from_to_tag(project): + """Update 'from' or 'to' tag to resources based on their path.""" + for resource in project.codebaseresources.files(): + if resource.path.startswith("from/"): + resource.update(tag="from") + elif resource.path.startswith("to/"): + resource.update(tag="to") + + RUST_BINARY_OPTIONS = ["Rust"] ELF_BINARY_OPTIONS = ["Python", "Go", "Elf"] MACHO_BINARY_OPTIONS = ["Rust", "Go", "MacOS"] diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index 3cbbb13200..401824f76f 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -82,6 +82,13 @@ def get_request_session(uri): """Return a Requests session setup with authentication and headers.""" session = requests.Session() + + # Set a default User-Agent to avoid 403 Forbidden errors on strict + # registries that block default python-requests headers. + session.headers.update( + {"User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"} + ) + netloc = urlparse(uri).netloc if credentials := scanpipe_settings.FETCH_BASIC_AUTH.get(netloc): diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index ad366045a0..087ff81935 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -66,6 +66,7 @@ REQUIRES_REVIEW = "requires-review" REVIEW_DANGLING_LEGAL_FILE = "review-dangling-legal-file" NOT_DEPLOYED = "not-deployed" +LICENSE_ISSUE = "license-mismatch-declared-vs-detected" GENERATED = "generated-file" @@ -138,3 +139,8 @@ def flag_mapped_resources(project): """Flag all codebase resources that were mapped during the d2d pipeline.""" resources = project.codebaseresources.has_relation().no_status() return resources.update(status=MAPPED) + + +def clear_status(resource_qs): + """Clear the status of given codebase resources.""" + return resource_qs.update(status="") diff --git a/scanpipe/pipes/nix.py b/scanpipe/pipes/nix.py new file mode 100644 index 0000000000..045e2864a8 --- /dev/null +++ b/scanpipe/pipes/nix.py @@ -0,0 +1,732 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import atexit +import logging +import os +import shutil +import subprocess +from collections import namedtuple +from pathlib import Path + +import requests +from fetchcode import fetch_json_response +from packageurl import PackageURL + +from scanpipe.pipes import utils + +logger = logging.getLogger(__name__) + + +# Result of `get_patched_source_with_docker`: +# - `path`: extracted source tree, or "" when nothing could be produced +# - `used_fallback`: True when the patched-source build failed and we fell +# back to the raw upstream `pkg.src` (unpatched) +# - `fallback_reason`: reason for the fallback, or "" +PatchedSourceResult = namedtuple( + "PatchedSourceResult", ["path", "used_fallback", "fallback_reason"] +) + +FALLBACK_REASON_PREFIX = "PATCHED_SOURCE_FALLBACK_REASON=" + + +def check_input_and_return_purl(project): + """Validate the input and return a Nix PURL.""" + input_sources = project.inputsources.all() + if len(input_sources) != 1: + error_msg = "Only 1 nix purl is accepted." + raise ValueError(error_msg) + + project_input = str(input_sources[0]) + input_purl = PackageURL.from_string(project_input) + if input_purl.type != "nix": + error_msg = "Only nix purl is supported." + raise ValueError(error_msg) + + namespace = input_purl.namespace + if not namespace or namespace.lower() != "nixpkgs": + raise Exception( + "Only official nixpkgs repository is supported (i.e. namespace=nixpkgs)." + ) + + qualifiers = input_purl.qualifiers or {} + if not input_purl.version and "commit" not in qualifiers: + raise Exception("Version or a 'commit' qualifier is required.") + + if "system" not in qualifiers: + raise Exception( + "The 'system' qualifier is required to resolve system-specific binaries." + ) + + return input_purl + + +def fetch_inputs(purl, output_dir): + """ + Fetch the system specific binary and the exact source tree with the + patches and configurations applied for the given input purl. Return a + tuple of (source_path, binary_path, output_format, error_message, + warning_message). + """ + data = get_package_data(purl) + name = purl.name + version = purl.version + + commit_hash = purl.qualifiers.get("commit", "") + system = purl.qualifiers.get("system", "") + user_output = purl.qualifiers.get("output", "") + error_message = "" + warning_message = "" + + output_format, path, release_commit_hash = get_nix_store_path( + data, name, version, system, commit_hash, user_output + ) + + concluded_commit_hash = release_commit_hash or commit_hash + + bin_path = "" + nix_bin_download_url = get_nix_download_url(path) if path else "" + if nix_bin_download_url: + bin_path = utils.fetch_path(nix_bin_download_url) + + if bin_path: + logger.info(f"Downloaded binary for {purl} to {bin_path}") + else: + if concluded_commit_hash: + logger.info( + f"Binary not found in cache for {purl}. Attempting local Nix build..." + ) + bin_path = build_binary_with_docker( + name, output_dir, system, concluded_commit_hash, output_format + ) + if bin_path: + logger.info(f"Successfully built binary for {purl} to {bin_path}") + warning_message = ( + f"Binary not found in cache for {purl}. Built locally using " + f"commit {concluded_commit_hash} with a Linux-based Nix " + f"Docker container." + ) + logger.warning(warning_message) + else: + error_message = f"Failed to fetch or build the binary for {purl}" + logger.error(error_message) + + source_result = PatchedSourceResult("", False, "") + if concluded_commit_hash: + source_result = get_patched_source_with_docker( + name, output_dir, system, concluded_commit_hash + ) + + if source_result.used_fallback: + detail = ( + f" Reason: {source_result.fallback_reason}." + if source_result.fallback_reason + else "" + ) + fallback_warning = ( + f"The patched source build for {name} failed; D2D will run " + f"against the raw upstream source (pkg.src) without nixpkgs " + f"patches.{detail} Mismatches between the source and binary " + f"trees may include files that were only added or modified by " + f"patches." + ) + if warning_message: + warning_message = f"{warning_message}\n{fallback_warning}" + else: + warning_message = fallback_warning + logger.warning(fallback_warning) + + return ( + source_result.path, + bin_path, + output_format, + error_message, + warning_message, + ) + + +def build_binary_with_docker(name, output_dir, system, commit_hash, output_format): + """ + Fetch a Nix package and build its binary from source using Docker. + Exports the resulting store path as a .nar file for standard extraction. + + Return an empty string if build fails. + """ + nar_filename = f"{name}-bin.nar" + extracted_path = Path(output_dir) / nar_filename + absolute_out_dir = str(Path(output_dir).resolve()) + + # Handle architecture and system incompatibilities + target_os = system.split("-")[-1] if "-" in system else system + if target_os and target_os != "linux": + logger.warning( + f"SYSTEM BARRIER DETECTED: Target system '{system}' requires " + f"OS-specific SDKs that cannot be evaluated inside the " + f"Linux-based Nix Docker container. Defaulting the build to " + f"the container's native Linux architecture." + ) + system_config = "" + else: + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + nixpkgs_import = ( + f'import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/' + f'{commit_hash}.tar.gz") {{ {system_config} {config_str} }}' + ) + + # Defaulting to 'debug' if none is specified. + effective_output = output_format or "debug" + + # Fall back to the default target if the effective_output is not + # defined in the recipe for this package. + nix_expression = ( + f"let " + f" pkgs = {nixpkgs_import}; " + f" target = pkgs.{name}; " + f" hasIt = builtins.isAttrs target && " + f'builtins.hasAttr "{effective_output}" target; ' + f"in if hasIt then target.{effective_output} else target" + ) + + # Build the Nix package, verify it succeeded, and export the output as + # a .nar file. + container_script = f""" + OUT_PATH=$(nix-build --no-out-link -E '{nix_expression}') + if [ -z "$OUT_PATH" ] || [ ! -e "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + exit 1 + fi + nix-store --dump "$OUT_PATH" > /build_output/{nar_filename} + """ + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{absolute_out_dir}:/build_output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Building ({name} for {system})" + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=1800) # noqa: S603 + if extracted_path.exists(): + return str(extracted_path) + return "" + except subprocess.CalledProcessError as e: + logger.error(f"Failed: {task_description} with error: {e.stderr.strip()}") + except subprocess.TimeoutExpired: + logger.error(f"Failed: {task_description} with error: Process timed out") + return "" + + +def get_nix_store_path(data, name, version, system, commit_hash, user_output): + """Get the Nix store path and release commit hash.""" + outputs_to_try = [user_output] if user_output else ["debug", "out"] + path = "" + release_commit_hash = "" + output_format = "" + + for output in outputs_to_try: + if data: + release_commit_hash, path = get_commit_hash_nix_store_path( + data, system, output, version, commit_hash + ) + + if not data or not path: + if commit_hash: + path = get_nix_store_path_with_nix(name, system, output, commit_hash) + + if path: + output_format = output + break + + if not path: + if not commit_hash: + raise Exception( + "Please provide a 'commit' qualifier in the PURL " + "for Nix to determine the download URL or build it locally." + ) + output_format = user_output or "debug" + + return output_format, path, release_commit_hash + + +def get_commit_hash_nix_store_path(data, system, output, version, commit_hash=""): + """ + Find and return the commit_hash and store path (/nix/store/) + based on the qualifiers + """ + releases = data.get("releases") or [] + releases = [r for r in releases if r.get("version") == version] + + for release in releases: + release_version = release.get("version", "") + if version and release_version != version: + continue + for platform in release.get("platforms", []): + release_commit_hash = platform.get("commit_hash", "") + if platform.get("system") != system: + continue + if commit_hash and release_commit_hash != commit_hash: + continue + for out in platform.get("outputs", []): + if out.get("name") == output: + return release_commit_hash, out.get("path") + return "", "" + + +def get_package_data(purl): + """Fetch package data from https://search.devbox.sh/.""" + api_url = f"https://search.devbox.sh/v2/pkg?name={purl.name}" + try: + return fetch_json_response(api_url) + except Exception as e: + logger.warning(f"Failed to fetch package data for {purl}: {e}") + return None + + +def get_nix_store_path_with_nix(name, system, output, commit_hash): + """Find and return the store path using 'nix'""" + system_config = f'system = "{system}";' if system else "" + config_str = "config = { allowBroken = true; allowUnfree = true; };" + + nix_expression = ( + "let " + f" pkgs = import (fetchTarball " + f'"https://github.com/NixOS/nixpkgs/archive/{commit_hash}.tar.gz") ' + f"{{ {system_config} {config_str} }}; " + f" target = pkgs.{name}; " + f' hasIt = builtins.isAttrs target && builtins.hasAttr "{output}" target; ' + f'in if hasIt then target.{output}.outPath else ""' + ) + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "nixos/nix", + "nix-instantiate", + "--eval", + "--raw", + "-E", + nix_expression, + ] + + try: + result = subprocess.run( # noqa: S603 + cmd, capture_output=True, text=True, check=True, timeout=300 + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"Error evaluating attribute for package '{name}': {e.stderr}") + return "" + except subprocess.TimeoutExpired: + logger.error(f"Timeout evaluating attribute for package '{name}'") + return "" + + +def get_nix_download_url(path): + """Construct a download url from cache.nixos.org based on store path""" + base_name = path.rstrip("/").split("/")[-1] + narinfo_hash = base_name.split("-")[0] + + narinfo_url = f"https://cache.nixos.org/{narinfo_hash}.narinfo" + url_path = get_narinfo_url(narinfo_url) + + if not url_path: + logger.warning(f"{narinfo_url} is not accessible.") + return "" + + return f"https://cache.nixos.org/{url_path}" + + +def get_narinfo_url(narinfo_url): + """Visit the narinfo url, parse and return the URL value""" + try: + response = requests.get(narinfo_url, timeout=10) + response.raise_for_status() + except requests.exceptions.RequestException: + return "" + + for line in response.text.splitlines(): + if line.startswith("URL:"): + return line.split(":", 1)[1].strip() + + return "" + + +def cleanup_docker_volumes(): + """Cleanup the Docker volumes used for Nix.""" + if not shutil.which("docker"): + return + + cmd = ["docker", "volume", "rm", "-f", "nix-eval-cache"] + try: + subprocess.run(cmd, capture_output=True, check=False) # noqa: S603 + except Exception as e: + logger.debug(f"Failed to cleanup Docker volumes: {e}") + + +atexit.register(cleanup_docker_volumes) + + +def _get_decompress_cmd(archive_name): + """Return (compression_type, decompress_cmd) for the archive name.""" + if archive_name.endswith(".xz"): + return "xz", f"xzcat /input/{archive_name}" + if archive_name.endswith(".zst"): + return "zstd", f"zstdcat /input/{archive_name}" + if archive_name.endswith(".bz2"): + return "bzip2", f"bzcat /input/{archive_name}" + if archive_name.endswith(".gz"): + return "gzip", f"zcat /input/{archive_name}" + return None, f"cat /input/{archive_name}" + + +def _stage_archive(archive_path, output_dir): + """ + Ensure the archive lives inside output_dir so it is visible to the Docker + daemon that resolves the `-v` mount source. Return the staged path. + """ + target = output_dir / archive_path.name + if archive_path == target: + return target + + is_present = ( + target.exists() and target.stat().st_size == archive_path.stat().st_size + ) + if not is_present: + shutil.copy2(archive_path, target) + return target + + +def extract_nar_archive(archive_path, output_dir, output): + """Extract a compressed Nix NAR archive.""" + archive_path = Path(archive_path).resolve() + output_dir = Path(output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + # Docker mounts are resolved by the daemon, not the client. To make the + # archive visible to the daemon that runs the container, it must live + # in `output_dir` — the one path this project shares with that daemon. + archive_path = _stage_archive(archive_path, output_dir) + + archive_dir = str(archive_path.parent) + archive_name = archive_path.name + extracted_path = output_dir / "to" / output + + compression_type, decompress_cmd = _get_decompress_cmd(archive_name) + + if compression_type: + restore_pipeline = ( + f"nix-shell -p {compression_type} --run " + f"'{decompress_cmd} | nix-store --restore /output/to/{output}'" + ) + else: + restore_pipeline = f"{decompress_cmd} | nix-store --restore /output/to/{output}" + + # nix-store --restore runs as root inside the container and preserves the + # NAR's ownership metadata, so the extracted tree ends up owned by root. + # Chown it back to the calling user so ScanCode can extract nested archives, + # read the files, and clean up afterwards. + host_uid = os.getuid() + host_gid = os.getgid() + + container_script = ( + f"rm -rf /output/to/{output} " + f"&& mkdir -p /output/to " + f"&& {restore_pipeline} " + f"&& chown -R {host_uid}:{host_gid} /output/to " + f"&& chmod -R u+w /output/to" + ) + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{archive_dir}:/input:ro", + "-v", + f"{output_dir}:/output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=600) # noqa: S603 + return str(extracted_path) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to extract {archive_name} with error: {e.stderr.strip()}") + except subprocess.TimeoutExpired: + logger.error(f"Failed to extract {archive_name}: Process timed out") + finally: + if archive_path.parent == output_dir: + try: + archive_path.unlink(missing_ok=True) + except OSError as e: + logger.debug(f"Could not remove staged archive {archive_path}: {e}") + return "" + + +def get_patched_source_with_docker(name, output_dir, system, commit_hash): + """ + Fetch a Nix package source and apply its official patches, falling back + to raw archives if package source cannot be built. + """ + extracted_path = Path(output_dir) / "from" + extracted_path.mkdir(parents=True, exist_ok=True) + absolute_out_dir = str(extracted_path.resolve()) + + host_uid = os.getuid() + host_gid = os.getgid() + + # Get the OS part from the system string (e.g. 'aarch64-darwin' to 'darwin') + target_os = system.split("-")[-1] if "-" in system else system + + if target_os and target_os != "linux": + logger.warning( + f"SYSTEM BARRIER DETECTED: Target system '{system}' requires " + f"OS-specific SDKs that cannot be evaluated inside the " + f"Linux-based Nix Docker container." + ) + logger.warning( + f"FALLBACK IN EFFECT: Evaluating the source using the container's " + f"native Linux environment. The extracted source tree will " + f"contain Linux-specific patches instead of {system} patches. " + f"Impact on the deployment to development mapping is expected to " + f"be minimal: you may observe a small number of unmapped files " + f"due to missing OS-specific structural patches." + ) + system_config = "" + else: + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + nixpkgs_import = ( + f'import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/' + f'{commit_hash}.tar.gz") {{ {system_config} {config_str} }}' + ) + + # Build patched source, but first `cd` into the actual source root + # (`$sourceRoot`) so we copy only its contents, not the wrapper + # directory that Nix creates during unpacking. This prevents + # duplicate paths like `from/-source/src/...`. + nix_expression = f""" + let + pkgs = {nixpkgs_import}; + pkg = pkgs.{name}; + in + pkg.overrideAttrs (old: {{ + name = (old.name or "{name}") + "-patched-src"; + phases = [ "unpackPhase" "patchPhase" "installPhase" ]; + installPhase = '' + mkdir -p $out + rm -f env-vars + + if [ -n "$sourceRoot" ] && [ -d "$sourceRoot" ]; then + cd "$sourceRoot" + fi + + cp -a . $out/ + ''; + outputs = [ "out" ]; + separateDebugInfo = false; + doCheck = false; + doInstallCheck = false; + }})""" + + fallback_expression = f""" + let + pkgs = {nixpkgs_import}; + pkg = pkgs.{name}; + in + if pkg ? gemFile then pkg.gemFile + else if pkg ? src then pkg.src + else pkg + """ + + # This bash script must NOT be indented in Python. + # If EOF has spaces before it, bash will fail to parse it. + # The following script first attempts a standard patched build; if that + # fails (or yields no files), it falls back to fetching the raw source + # archive. The result is copied to the mounted `from/` directory with + # correct ownership so the host can extract and process it. + container_script = f""" +set -e + +cat << 'EOF' > /build_output/expr.nix +{nix_expression} +EOF + +cat << 'EOF' > /build_output/fallback.nix +{fallback_expression} +EOF + +# Try standard patched build +OUT_PATH=$(nix-build --no-out-link /build_output/expr.nix || true) + +# Check if output is empty or only contains env-vars which is generated by Nix +VALID_FILES=0 +if [ -n "$OUT_PATH" ] && [ -d "$OUT_PATH" ]; then + VALID_FILES=$(ls -A1 "$OUT_PATH" 2>/dev/null | grep -v "^env-vars$" | wc -l) +fi + +if [ -z "$OUT_PATH" ]; then + FALLBACK_REASON="primary nix-build returned no store path" +elif [ ! -d "$OUT_PATH" ]; then + FALLBACK_REASON="primary store path is not a directory" +elif [ "$VALID_FILES" -eq 0 ]; then + FALLBACK_REASON="primary output contained only env-vars" +fi + +# Use raw archive if standard build failed or was empty +if [ -n "$FALLBACK_REASON" ]; then + echo "PATCHED_SOURCE_FALLBACK_REASON=$FALLBACK_REASON" >&2 + OUT_PATH=$(nix-build --no-out-link /build_output/fallback.nix || true) +fi + +# If both completely failed, clean up and exit +if [ -z "$OUT_PATH" ] || [ ! -e "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + rm -f /build_output/expr.nix /build_output/fallback.nix + exit 1 +fi + +# Copy contents (if directory) or the single file (if archive) +if [ -d "$OUT_PATH" ]; then + cp -a "$OUT_PATH/." /build_output/ +else + cp -L "$OUT_PATH" /build_output/ +fi + +# Set ownership to the host user so Python can extract it +chown -R $HOST_UID:$HOST_GID /build_output/ +chmod -R u+w /build_output/ + +# Cleanup the temp nix files +rm -f /build_output/expr.nix /build_output/fallback.nix +""" + + cmd = [ + "docker", + "run", + "--rm", + "-e", + f"HOST_UID={host_uid}", + "-e", + f"HOST_GID={host_gid}", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{absolute_out_dir}:/build_output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + try: + result = subprocess.run( # noqa: S603 + cmd, capture_output=True, text=True, check=True, timeout=600 + ) + if any(extracted_path.iterdir()): + used_fallback = False + fallback_reason = "" + for line in result.stderr.splitlines(): + if line.startswith(FALLBACK_REASON_PREFIX): + used_fallback = True + fallback_reason = line[len(FALLBACK_REASON_PREFIX) :].strip() + break + if used_fallback: + logger.warning( + f"Primary patched-source build failed for {name}: {fallback_reason}" + ) + return PatchedSourceResult( + path=str(extracted_path), + used_fallback=used_fallback, + fallback_reason=fallback_reason, + ) + except subprocess.CalledProcessError as e: + logger.error(f"Failed: {e.stderr.strip()}") + except subprocess.TimeoutExpired: + logger.error("Process timed out") + + shutil.rmtree(extracted_path, ignore_errors=True) + return PatchedSourceResult(path="", used_fallback=False, fallback_reason="") + + +def ensure_multiarch_emulation(): + """ + Configure Docker host with binfmt emulators to support + multi-architecture execution and builds. + """ + cmd = [ + "docker", + "run", + "--privileged", + "--rm", + "tonistiigi/binfmt", + "--install", + "all", + ] + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=60) # noqa: S603 + return True + except subprocess.CalledProcessError as e: + logger.warning(f"Could not install binfmt multi-arch emulators: {e.stderr}") + return False + except subprocess.TimeoutExpired: + logger.warning("Timeout trying to setup binfmt emulators. Skipping.") + return False diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py new file mode 100644 index 0000000000..b2c00b51d0 --- /dev/null +++ b/scanpipe/pipes/utils.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import logging +import shutil +import subprocess +from fnmatch import fnmatch + +import requests +from license_expression import Licensing + +from scanpipe.pipes import fetch +from scanpipe.pipes import flag + +logger = logging.getLogger(__name__) + + +def validate_package_license_integrity(project): + """Validate the correctness of the package license.""" + # Patterns to ignore certain resources during license validation + ignore_patterns = [ + "*test*", + "*.sh", + ] + + for package in project.discoveredpackages.all(): + package_lic = package.get_declared_license_expression() + if package_lic: + if package.type == "cargo": + # A single cargo package only has one Cargo.toml file + # meaning only one package is defined. Therefore, we don't + # need to check for the package_uid + # In addition, the package_uid is not populated to source files: + # https://github.com/aboutcode-org/scancode.io/issues/2169 + # so we set package_uid to None to consider all resources + # in the codebase for license validation. + package_uid = None + else: + package_uid = package.package_uid + resources = project.codebaseresources.has_license_expression() + detected_lic_list = collect_detected_licenses( + resources, ignore_patterns, package_uid + ) + + if detected_lic_list: + lic_exp = " AND ".join(detected_lic_list) + detected_lic_exp = str(Licensing().dedup(lic_exp)) + + if detected_lic_exp != package_lic: + package_issues = package.extra_data.get("issues", []) + + package_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + package.update_extra_data({"issues": package_issues}) + + for datafile_path in package.datafile_paths: + if not datafile_path.startswith("https://"): + data_path = project.codebaseresources.get( + path=datafile_path + ) + data_path.update(status=flag.LICENSE_ISSUE) + + resource_issues = data_path.extra_data.get("issues", []) + resource_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + data_path.update_extra_data({"issues": resource_issues}) + + +def contains_ignore_pattern(resource_path, ignore_patterns): + """Check if the resource path matches any of the ignore patterns.""" + for pattern in ignore_patterns: + if fnmatch(resource_path, pattern): + return True + return False + + +def filter_ignored_licenses(license_expression, licensing): + """Filter out ignored licenses from a license expression.""" + # Some licenses are not useful for validating package license + # integrity, so we ignore them. + ignored_licenses = [ + "free-unknown", + "unknown", + "unknown-license-reference", + "unknown-spdx", + ] + + if license_expression is None: + return None + + if isinstance(license_expression, licensing.Symbol): + if ( + hasattr(license_expression, "key") + and license_expression.key in ignored_licenses + ): + return None + return license_expression + + # Handle AND operations + if isinstance(license_expression, licensing.AND): + return handle_operator_expression(license_expression, licensing, licensing.AND) + + # Handle OR operations + if isinstance(license_expression, licensing.OR): + return handle_operator_expression(license_expression, licensing, licensing.OR) + + return license_expression + + +def handle_operator_expression(expression, licensing, operator): + """ + Process AND/OR operations in a license expression, filtering out + ignored licenses. + """ + args = [] + for arg in expression.args: + filtered_arg = filter_ignored_licenses(arg, licensing) + if filtered_arg is not None: + args.append(filtered_arg) + if not args: + return None + if len(args) == 1: + return args[0] + + return operator(*args) + + +def collect_detected_licenses(resources, ignore_patterns, package_uid=None): + """Collect detected licenses from resources, ignoring defined patterns.""" + licensing = Licensing() + detected_lic_list = [] + + for resource in resources: + if contains_ignore_pattern(resource.path, ignore_patterns): + continue + + # If a package_uid is provided, only consider resources linked to it + if package_uid and package_uid not in resource.for_packages: + continue + + license_str = resource.detected_license_expression + if not license_str: + continue + try: + parsed_lic = licensing.parse(license_str) + + # Filter out the ignored keys + filtered_license = filter_ignored_licenses(parsed_lic, licensing) + + if filtered_license is not None: + final_lic = str(filtered_license) + + if final_lic not in detected_lic_list: + # Apply parentheses so that the 'OR' expression will + # not be filtered out when doing deduplication later. + detected_lic_list.append(f"({final_lic})") + + except Exception: + logger.warning( + "Failed to parse the license expression: %s at %s", + license_str, + resource.path, + ) + return detected_lic_list + + +def fetch_path(purl): + """Fetch the purl and return the location of the fetched tarball""" + try: + return fetch.fetch_url(url=purl).path + except (ValueError, requests.RequestException) as e: + logger.warning("Failed to fetch package: %s - %s", purl, e) + return None + + +def check_docker_command(): + """Check if the Docker command is available and the daemon is running.""" + docker_path = shutil.which("docker") + if not docker_path: + return False + + try: + subprocess.run([docker_path, "info"], capture_output=True, check=True) # noqa: S603 + return True + except (subprocess.SubprocessError, FileNotFoundError): + return False diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 90f917c245..206a66526e 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -34,6 +34,11 @@ {% endif %} + {% if package.extra_data.issues %} + + + + {% endif %} @@ -75,4 +80,4 @@ {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 5ef10243a3..5e5db04b7b 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -2591,3 +2591,13 @@ def test_scanpipe_pipes_d2d_flag_generated_file_by_bytecode(self, mock_read_byte "Google Protocol Buffers", resource.extra_data.get("Generated code"), ) + + def test_scanpipe_pipes_d2d_flag_processed_archives_never_extracted(self): + to_archive = make_resource_file( + self.project1, path="to/archive.rds", is_archive=True + ) + + d2d.flag_processed_archives(self.project1) + + to_archive.refresh_from_db() + self.assertEqual("", to_archive.status) diff --git a/scanpipe/tests/pipes/test_nix.py b/scanpipe/tests/pipes/test_nix.py new file mode 100644 index 0000000000..50858ab07d --- /dev/null +++ b/scanpipe/tests/pipes/test_nix.py @@ -0,0 +1,495 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import tempfile +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from packageurl import PackageURL + +from scanpipe.pipes import nix + + +class ScanPipeNixPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" + + def test_scanpipe_nix_check_input_and_return_purl(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ] + + expected = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + result = nix.check_input_and_return_purl(project) + self.assertEqual(result, expected) + + def test_scanpipe_nix_check_input_and_return_purl_no_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_multi_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux", + "pkg:nix/nixpkgs/world@2.40.0?system=x86_64-linux", + ] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_non_supported_type(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:npm/test@1.0"] + with self.assertRaisesMessage(ValueError, "Only nix purl is supported."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_invalid_namespace(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/namespace/hello@2.12.1?system=x86_64-linux" + ] + with self.assertRaisesMessage( + Exception, "Only official nixpkgs repository is supported" + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_version_and_commit(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello?system=x86_64-linux" + ] + with self.assertRaisesMessage( + Exception, "Version or a 'commit' qualifier is required." + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_system(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:nix/nixpkgs/hello@2.12.1"] + with self.assertRaisesMessage( + Exception, + "The 'system' qualifier is required to resolve system-specific binaries.", + ): + nix.check_input_and_return_purl(project) + + @mock.patch("scanpipe.pipes.nix.fetch_json_response") + def test_scanpipe_nix_get_package_data(self, mock_fetch_json): + mock_fetch_json.return_value = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + } + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux" + ) + + result = nix.get_package_data(purl) + self.assertEqual( + result, + { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + }, + ) + mock_fetch_json.assert_called_once_with( + "https://search.devbox.sh/v2/pkg?name=hello" + ) + + def test_scanpipe_nix_get_commit_hash_nix_store_path(self): + data = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + }, + { + "name": "debug", + "path": "/nix/store/aaaaaaa-hello-2.12.1-debug", + }, + ], + } + ], + } + ] + } + + commit, store_path = nix.get_commit_hash_nix_store_path( + data, "x86_64-linux", "out", "2.12.1", "1234abcd" + ) + self.assertEqual(commit, "1234abcd") + self.assertEqual(store_path, "/nix/store/aaaaaaa-hello-2.12.1") + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_get_nix_store_path_with_nix(self, mock_subprocess_run): + mock_result = mock.Mock() + mock_result.stdout = "/nix/store/evaluated-path-out" + mock_subprocess_run.return_value = mock_result + + path = nix.get_nix_store_path_with_nix( + "hello", "x86_64-linux", "out", "1234abcd" + ) + self.assertEqual(path, "/nix/store/evaluated-path-out") + + @mock.patch("scanpipe.pipes.nix.get_narinfo_url") + def test_scanpipe_nix_get_nix_download_url(self, mock_get_narinfo): + mock_get_narinfo.return_value = "nar/abc.nar.xz" + store_path = "/nix/store/aaaaaaaaaaaaa-hello-2.12.1" + + url = nix.get_nix_download_url(store_path) + self.assertEqual(url, "https://cache.nixos.org/nar/abc.nar.xz") + + @mock.patch("scanpipe.pipes.nix.requests.get") + def test_scanpipe_nix_get_narinfo_url(self, mock_requests_get): + mock_response = mock.Mock() + mock_response.text = "StorePath: /nix/store/xyz\nURL: nar/123.nar.xz" + mock_requests_get.return_value = mock_response + + url_path = nix.get_narinfo_url("https://cache.nixos.org/aaaaaaaaaaa.narinfo") + self.assertEqual(url_path, "nar/123.nar.xz") + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs( + self, + mock_fetch_path, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + mock_get_download_url.return_value = "https://cache.nixos.org/nar/hello.nar.xz" + mock_get_patched_source.return_value = nix.PatchedSourceResult( + path="/path/extracted/from", + used_fallback=False, + fallback_reason="", + ) + mock_fetch_path.return_value = "/path/debug/to" + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msg, warning_msg = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/debug/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msg, "") + self.assertEqual(warning_msg, "") + + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.nix.build_binary_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs_fallback_build( + self, + mock_fetch_path, + mock_build_binary, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + """Test that fetch_inputs falls back to local build if download fails.""" + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + # Simulate a missing/failed cache download + mock_get_download_url.return_value = "" + mock_fetch_path.return_value = "" + + # Simulate a successful local build and source extraction + mock_build_binary.return_value = "/path/built/locally/to" + mock_get_patched_source.return_value = nix.PatchedSourceResult( + path="/path/extracted/from", + used_fallback=False, + fallback_reason="", + ) + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msg, warning_msg = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/built/locally/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msg, "") + self.assertTrue("Built locally using commit" in warning_msg) + + mock_build_binary.assert_called_once() + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_commit_hash_nix_store_path") + def test_scanpipe_nix_get_nix_store_path_success( + self, mock_get_commit_hash_nix_store_path + ): + mock_get_commit_hash_nix_store_path.return_value = ( + "1234abcd", + "/nix/store/hello-path", + ) + + output_fmt, path, commit = nix.get_nix_store_path( + data={"releases": []}, + name="hello", + version="2.12.1", + system="x86_64-linux", + commit_hash="1234abcd", + user_output="", + ) + + self.assertEqual(output_fmt, "debug") + self.assertEqual(path, "/nix/store/hello-path") + self.assertEqual(commit, "1234abcd") + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_get_patched_source_with_docker_success( + self, mock_subprocess_run + ): + """Test successful fetching and patching of source using Docker.""" + mock_subprocess_run.return_value = mock.Mock(stderr="", returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + from_dir = Path(temp_dir) / "from" + from_dir.mkdir() + (from_dir / "somefile").touch() + + result = nix.get_patched_source_with_docker( + name="hello", + output_dir=temp_dir, + system="x86_64-linux", + commit_hash="1234abcd", + ) + + self.assertEqual(result.path, str(from_dir)) + self.assertFalse(result.used_fallback) + self.assertEqual(result.fallback_reason, "") + mock_subprocess_run.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_get_patched_source_with_docker_fallback( + self, mock_subprocess_run + ): + """The fallback line on stderr flips used_fallback and carries the reason.""" + mock_subprocess_run.return_value = mock.Mock( + stderr=( + "some nix output\n" + "PATCHED_SOURCE_FALLBACK_REASON=" + "primary output contained only env-vars\n" + ), + returncode=0, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + from_dir = Path(temp_dir) / "from" + from_dir.mkdir() + (from_dir / "somefile").touch() + + result = nix.get_patched_source_with_docker( + name="hello", + output_dir=temp_dir, + system="x86_64-linux", + commit_hash="1234abcd", + ) + + self.assertEqual(result.path, str(from_dir)) + self.assertTrue(result.used_fallback) + self.assertEqual( + result.fallback_reason, "primary output contained only env-vars" + ) + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_extract_nar_archive_success(self, mock_subprocess_run): + """Test extracting a .nar archive via Docker.""" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + # We don't actually need the file to exist for the mocked test + archive_path = Path(temp_dir) / "hello-bin.nar.xz" + + result = nix.extract_nar_archive( + archive_path=str(archive_path), output_dir=temp_dir, output="debug" + ) + + expected_extracted_path = str(Path(temp_dir).resolve() / "to" / "debug") + self.assertEqual(result, expected_extracted_path) + + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_extract_nar_archive_stages_from_tmp( + self, mock_subprocess_run, mock_copy2 + ): + """Archive outside output_dir is staged into it before docker run.""" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir) / "hello-bin.nar.xz" + + result = nix.extract_nar_archive( + archive_path=str(archive_path), output_dir=output_dir, output="debug" + ) + + expected_extracted_path = str(Path(output_dir).resolve() / "to" / "debug") + self.assertEqual(result, expected_extracted_path) + + # Staging must have happened exactly once + mock_copy2.assert_called_once() + src, dst = mock_copy2.call_args[0] + self.assertEqual(Path(src), Path(source_dir).resolve() / "hello-bin.nar.xz") + self.assertEqual(Path(dst), Path(output_dir).resolve() / "hello-bin.nar.xz") + + # The docker mount source must be output_dir, not the /tmp source + cmd = mock_subprocess_run.call_args[0][0] + volume_mounts = [cmd[i + 1] for i, a in enumerate(cmd) if a == "-v"] + self.assertTrue( + any(str(Path(output_dir).resolve()) in v for v in volume_mounts), + f"expected staged mount in {volume_mounts}", + ) + self.assertFalse( + any(str(Path(source_dir).resolve()) in v for v in volume_mounts), + f"unexpected source mount in {volume_mounts}", + ) + + def test_scanpipe_nix_get_decompress_cmd(self): + cases = [ + ("foo.nar.xz", "xz", "xzcat /input/foo.nar.xz"), + ("foo.nar.zst", "zstd", "zstdcat /input/foo.nar.zst"), + ("foo.nar.bz2", "bzip2", "bzcat /input/foo.nar.bz2"), + ("foo.nar.gz", "gzip", "zcat /input/foo.nar.gz"), + ("foo.nar", None, "cat /input/foo.nar"), + ] + for name, expected_type, expected_cmd in cases: + compression_type, cmd = nix._get_decompress_cmd(name) + self.assertEqual(compression_type, expected_type) + self.assertEqual(cmd, expected_cmd) + + def test_scanpipe_nix_stage_archive_already_in_output_dir(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir).resolve() + archive_path = output_dir / "hello-bin.nar.xz" + + with mock.patch("scanpipe.pipes.nix.shutil.copy2") as mock_copy2: + result = nix._stage_archive(archive_path, output_dir) + + self.assertEqual(result, archive_path) + mock_copy2.assert_not_called() + + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + def test_scanpipe_nix_stage_archive_from_elsewhere(self, mock_copy2): + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir).resolve() / "hello-bin.nar.xz" + output_path = Path(output_dir).resolve() + + result = nix._stage_archive(archive_path, output_path) + + self.assertEqual(result, output_path / "hello-bin.nar.xz") + mock_copy2.assert_called_once_with( + archive_path, output_path / "hello-bin.nar.xz" + ) + + @mock.patch("scanpipe.pipes.nix.shutil.copy2") + def test_scanpipe_nix_stage_archive_skips_copy_when_sizes_match(self, mock_copy2): + with ( + tempfile.TemporaryDirectory() as source_dir, + tempfile.TemporaryDirectory() as output_dir, + ): + archive_path = Path(source_dir).resolve() / "hello-bin.nar.xz" + archive_path.write_bytes(b"payload") + target = Path(output_dir).resolve() / "hello-bin.nar.xz" + target.write_bytes(b"payload") # same size + + result = nix._stage_archive(archive_path, Path(output_dir).resolve()) + + self.assertEqual(result, target) + mock_copy2.assert_not_called() diff --git a/scanpipe/tests/pipes/test_utils.py b/scanpipe/tests/pipes/test_utils.py new file mode 100644 index 0000000000..490f7a0997 --- /dev/null +++ b/scanpipe/tests/pipes/test_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + + +from unittest import mock + +from django.test import TestCase + +from license_expression import Licensing + +from scanpipe.pipes import flag +from scanpipe.pipes import utils + + +class ScanPipeUtilsTest(TestCase): + def setUp(self): + self.licensing = Licensing() + + @mock.patch("scanpipe.models.CodebaseResource") + @mock.patch("scanpipe.models.DiscoveredPackage") + @mock.patch("scanpipe.models.Project") + def test_validate_package_license_integrity_mismatch( + self, mock_project_class, mock_package_class, mock_resource_class + ): + mock_project = mock_project_class() + mock_package = mock_package_class() + + mock_package.type = "pypi" + mock_package.package_uid = "pkg:pypi/test@1.0" + mock_package.get_declared_license_expression.return_value = "mit" + mock_package.datafile_paths = ["src/main.py"] + mock_package.extra_data = {} + + mock_project.discoveredpackages.all.return_value = [mock_package] + + mock_resource = mock_resource_class() + mock_resource.path = "src/main.py" + mock_resource.for_packages = ["pkg:pypi/test@1.0"] + mock_resource.detected_license_expression = "gpl-3.0" + + mock_project.codebaseresources.has_license_expression.return_value = [ + mock_resource + ] + + mock_data_path = mock_resource_class() + mock_data_path.extra_data = {} + mock_project.codebaseresources.get.return_value = mock_data_path + + utils.validate_package_license_integrity(mock_project) + + package_update_args = mock_package.update_extra_data.call_args.args[0] + self.assertEqual( + package_update_args["issues"][0]["issue_type"], "License Mismatch" + ) + self.assertEqual( + package_update_args["issues"][0]["detected_codebase_license"], "gpl-3.0" + ) + + mock_data_path.update.assert_called_once_with(status=flag.LICENSE_ISSUE) + + def test_contains_ignore_pattern(self): + ignore_patterns = ["*test*", "*.sh"] + self.assertTrue( + utils.contains_ignore_pattern("src/test_main.py", ignore_patterns) + ) + self.assertTrue( + utils.contains_ignore_pattern("scripts/build.sh", ignore_patterns) + ) + self.assertFalse(utils.contains_ignore_pattern("src/main.py", ignore_patterns)) + + def test_filter_ignored_licenses(self): + exp1 = self.licensing.parse("mit") + self.assertEqual( + str(utils.filter_ignored_licenses(exp1, self.licensing)), "mit" + ) + + exp2 = self.licensing.parse("unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp2, self.licensing)) + + exp3 = self.licensing.parse("mit AND unknown") + self.assertEqual( + str(utils.filter_ignored_licenses(exp3, self.licensing)), "mit" + ) + + exp4 = self.licensing.parse("unknown-spdx OR free-unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp4, self.licensing)) + + def test_collect_detected_licenses(self): + mock_resource1 = mock.Mock() + mock_resource1.path = "src/main.py" + mock_resource1.for_packages = ["pkg:pypi/test@1.0"] + mock_resource1.detected_license_expression = "mit AND unknown" + + mock_resource2 = mock.Mock() + mock_resource2.path = "test/test_main.py" + mock_resource2.for_packages = ["pkg:pypi/test@1.0"] + mock_resource2.detected_license_expression = "gpl-3.0" + + mock_resource3 = mock.Mock() + mock_resource3.path = "src/other.py" + mock_resource3.for_packages = ["pkg:pypi/test@2.0"] + mock_resource3.detected_license_expression = "apache-2.0" + + resources = [mock_resource1, mock_resource2, mock_resource3] + ignore_patterns = ["*test*"] + + result = utils.collect_detected_licenses( + resources, ignore_patterns, package_uid="pkg:pypi/test@1.0" + ) + + self.assertEqual(result, ["(mit)"]) + + def test_handle_operator_expression_and(self): + expr = self.licensing.parse("mit AND apache-2.0") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit AND apache-2.0") + + def test_handle_operator_expression_or(self): + expr = self.licensing.parse("mit OR bsd-3-clause") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.OR + ) + self.assertEqual(str(result), "mit OR bsd-3-clause") + + def test_handle_operator_expression_filters_to_single_arg(self): + # 'unknown' gets filtered out to None, leaving only 'mit' (len == 1) + expr = self.licensing.parse("mit AND unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit") + + def test_handle_operator_expression_all_filtered_out(self): + # Both 'unknown' and 'free-unknown' get filtered out, leaving empty args + expr = self.licensing.parse("unknown AND free-unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertIsNone(result) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + @mock.patch("scanpipe.pipes.utils.subprocess.run") + def test_check_docker_command_success(self, mock_subprocess_run, mock_shutil_which): + mock_shutil_which.return_value = "/usr/bin/docker" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + self.assertTrue(utils.check_docker_command()) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + def test_check_docker_command_not_found(self, mock_shutil_which): + mock_shutil_which.return_value = None + + self.assertFalse(utils.check_docker_command()) diff --git a/scanpipe/views.py b/scanpipe/views.py index 7981f2a7d7..a22278220c 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1774,6 +1774,7 @@ def get_queryset(self): "compliance_alert", "copyright", "affected_by_vulnerabilities", + "extra_data", ) .with_resources_count() .order_by_package_url()