Evolve Rust pipeline (#1767)

 * Use Docker for building instead of Cargo
 * Accept only one PURL as input
 * Use the .rlib found in .d files for D2D mapping
 * Remove unnecessary code

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

scanpipe/pipelines/scan_rust_package.py

@@ -30,7 +30,9 @@
 from scanpipe.pipes import rust
 from scanpipe.pipes import utils
 
-# from scanpipe.pipes.maven import update_package_license_from_resource_if_missing
+from scanpipe.pipes.rust import check_input_and_return_purl, fetch_inputs
+
+import shutil
 
 
 class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase):
@@ -45,17 +47,19 @@ class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase):
     the license declared in Cargo.toml.
 
     Compare the crate’s source code against all other crates (MatchCode),
-    excluding itself, to detect any borrowed code from third‑party crates.
+    excluding itself, to detect any borrowed code from third-party crates.
     """
 
+    download_inputs = False
+
     @classmethod
     def steps(cls):
         return (
-            cls.get_input,
-            cls.get_package_input,
-            cls.collect_input_information,
-            cls.extract_inputs_to_codebase_directory,
-            cls.extract_archives,
+            cls.check_input_and_return_purl,
+            cls.fetch_inputs,
+            cls.collect_input_info,
+            cls.extract_input_to_codebase_directory,
+            cls.check_docker_command,
             cls.build_crates,
             cls.run_scan,
             cls.load_inventory_from_toolkit_scan,
@@ -66,30 +70,40 @@ def steps(cls):
             cls.make_summary_from_scan_results,
         )
 
-    def get_input(self):
-        """Get the input file for the Rust package scan pipeline."""
-        from_files = list(self.project.inputs("from*"))
-        from_files.extend([input.path for input in self.project.inputsources.all()])
-        self.from_files = from_files
-        self.to_files = list()
+    def check_input_and_return_purl(self):
+        """Validate the input is a PURL string and return the PURL object."""
+        self.purl = check_input_and_return_purl(self.project)
+
+    def fetch_inputs(self):
+        """Fetch the source of the given PURL."""
+        self.from_files = fetch_inputs(self.purl)
+
+    def collect_input_info(self):
+        """Collect information about the input."""
+        self.input_path = self.from_files
+        self.collect_input_information()
+
+    def check_docker_command(self):
+        self.have_docker = False
+        if shutil.which("docker"):
+            self.have_docker = True
 
     def build_crates(self):
         """
-        Build the Rust crate using Cargo and put the built files under the
+        Build the Rust crate using Docker and put the built files under the
         "to" directory.
         """
-        # Find the Cargo.toml file in the codebase directory
-        codebase_dir = Path(self.project.codebase_path)
-        cargo_toml_path = None
-        for path in codebase_dir.rglob("Cargo.toml"):
-            cargo_toml_path = path
-            break
-        if cargo_toml_path:
-            rust.build_crates(cargo_toml_path, self.project.codebase_path / "to/")
+        self.d2d_enable = False
+        if self.have_docker:
+            if rust.build_crates(self.project.codebase_path):
+                self.d2d_enable = True
+            else:
+                print("Docker command not found. Skipping crate build.")
 
     def add_from_to_tag(self):
         """Update 'from' or 'to' tag to resources based on their path."""
-        d2d.update_from_to_tag(self.project)
+        if self.d2d_enable:
+            d2d.update_from_to_tag(self.project)
 
     def validate_package_license_integrity(self):
         """
@@ -100,8 +114,10 @@ def validate_package_license_integrity(self):
 
     def identify_built_sources(self):
         """Identify the built sources from the '.d' file in the "to" directory."""
-        d2d.map_rust_paths(self.project)
+        if self.d2d_enable:
+            d2d.map_rust_paths(self.project)
 
     def flag_mapped_status(self):
         """Flag the from codebase resources that were mapped."""
-        flag.flag_mapped_resources(self.project)
+        if self.d2d_enable:
+            flag.flag_mapped_resources(self.project)

scanpipe/pipes/d2d.py

@@ -1948,10 +1948,10 @@ def map_go_paths(project, logger=None):
 def get_rust_file_paths(location):
     """Retrieve Rust file paths."""
     file_paths = {}
-    rust_file_paths = parse_d_file(location) or []
+    rust_lib_path, rust_file_paths = parse_d_file(location) or []
     if rust_file_paths:
         file_paths["rust_file_paths"] = rust_file_paths
-    return file_paths
+    return rust_lib_path, file_paths
 
 
 def parse_d_file(path):
@@ -1963,31 +1963,57 @@ def parse_d_file(path):
     if ":" not in cleaned_context:
         return []
 
-    _, dep_paths = cleaned_context.split(":", 1)
+    rust_lib, dep_paths = cleaned_context.split(":", 1)
+    rust_lib_path = rust_lib.strip()
 
     file_paths = []
     for file_path in dep_paths.split():
         file_path = file_path.strip()
         if file_path:
             file_paths.append(file_path)
 
-    return file_paths
+    return rust_lib_path, file_paths
 
 
 def map_rust_paths(project, logger=None):
     """Map the path listed in the .d file to the source in ``project``."""
     from_resources = project.codebaseresources.files().from_codebase()
-    to_resources = (
+    # Fetch the .d files to extract data from
+    data_resources = (
         project.codebaseresources.files()
         .to_codebase()
         .exclude(path__contains="/deps/")
         .exclude(path__contains="/build/")
         .filter(path__endswith=".d")
     )
-    for resource in to_resources:
+    target_rlib_ids = []
+    for resource in data_resources:
         try:
-            paths = get_rust_file_paths(resource.location_path)
-            resource.update_extra_data(paths)
+            rlib_path, paths = get_rust_file_paths(resource.location_path)
+            absolute_rlib_path = Path(rlib_path)
+            rlib_resource = None
+            try:
+                # Docker paths start with "/codebase". Host paths start with project.codebase_path.
+                if str(absolute_rlib_path).startswith("/codebase/"):
+                    clean_rlib_path = str(absolute_rlib_path.relative_to("/codebase"))
+                else:
+                    clean_rlib_path = str(absolute_rlib_path.relative_to(project.codebase_path))
+
+                # We can now safely do an exact path match
+                rlib_resource = (
+                    project.codebaseresources.files()
+                    .to_codebase()
+                    .filter(path=clean_rlib_path)
+                    .first()
+                )
+            except ValueError:
+                pass
+
+            if rlib_resource:
+                rlib_resource.update_extra_data(paths)
+                target_rlib_ids.append(rlib_resource.id)
+            elif logger:
+                logger(f"Warning: Could not find rlib file {absolute_rlib_path.name} in database.")
         except Exception as exception:
             project.add_warning(
                 exception=exception,
@@ -1997,6 +2023,8 @@ def map_rust_paths(project, logger=None):
                 details={"path": resource.path},
             )
 
+    to_resources = project.codebaseresources.filter(id__in=target_rlib_ids)
+
     if logger:
         logger(
             f"Mapping {to_resources.count():,d} to/ resources using paths "

scanpipe/pipes/rust.py

@@ -20,31 +20,117 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+import shutil
 import subprocess
+import logging
+import requests
 
+from packageurl import PackageURL
+from pathlib import Path
+from scanpipe.pipes import fetch
 from scanpipe.pipes import run_command_safely
 
 
-def build_crates(cargo_toml_path, build_dir):
+logger = logging.getLogger(__name__)
+
+
+def build_crates(codebase_dir):
     """
-    Build the Rust crate using Cargo to ensure that the source code compiles correctly.
+    Build the Rust crate from sources in an isolated Docker container.
+
+    Uses the official rust image to safely sandbox the build process and
+    injects RUSTFLAGS to force DWARF debug symbol generation (-C debuginfo=2)
+    required for binary-to-source mapping.
 
-    This step is crucial for validating the integrity of the source code and ensuring
-    that it can be successfully built. It also helps to identify any discrepancies
-    between the source code and the compiled binary, which can be further analyzed
-    in subsequent steps of the pipeline.
+    Return True if build successfully, False otherwise.
     """
+
+    # Find the Cargo.toml file in the codebase directory
+    codebase_dir = Path(codebase_dir)
+    cargo_toml_path = None
+    for path in codebase_dir.rglob("Cargo.toml"):
+        cargo_toml_path = path
+        break
+    if cargo_toml_path:
+        to_dir = codebase_dir / "to"
+    else:
+        return False
+
+    cargo_toml_path = Path(cargo_toml_path)
+    build_dir = Path(to_dir)
+
+    # Calculate paths relative to the container's mounted /codebase directory
+    rel_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix()
+    rel_build_dir = build_dir.relative_to(codebase_dir).as_posix()
+
+    container_cargo_toml = f"/codebase/{rel_cargo_toml}"
+    container_build_dir = f"/codebase/{rel_build_dir}"
+
     cmd = [
-        "cargo",
-        "build",
+        "docker", "run",
+        "--rm",  # Automatically remove the container when it exits
+        "--volume", f"{codebase_dir}:/codebase",
+        "--workdir", "/codebase",
+        "--env", "RUSTFLAGS=-C debuginfo=2",  # Force DWARF generation in release mode
+        "rust:latest",
+        "cargo", "build",
+        "--release",
         "--locked",
-        "--manifest-path",
-        str(cargo_toml_path),
-        "--target-dir",
-        str(build_dir),
+        "--manifest-path", container_cargo_toml,
+        "--target-dir", container_build_dir,
     ]
 
     try:
         run_command_safely(cmd)
     except subprocess.SubprocessError as error:
-        raise RuntimeError(f"Failed to build the Rust crate: {error}")
+        logger.warning(f"Failed to build the Rust crate in Docker: {error}")
+        return False
+
+    from_dir = codebase_dir / "from"
+    from_dir.mkdir(exist_ok=True)
+    for item in codebase_dir.iterdir():
+        if item != to_dir and item != from_dir:
+            shutil.move(str(item), str(from_dir / item.name))
+    return True
+
+
+def check_input_and_return_purl(project):
+    """Validate the input and return a cargo PURL."""
+    input_sources = project.inputsources.all()
+    if len(input_sources) != 1:
+        error_msg = "Only 1 cargo purl is accepted."
+        raise ValueError(error_msg)
+    # Strip the qualifiers as this is not needed.
+    project_input = str(input_sources[0]).split("?")[0]
+    input_purl = PackageURL.from_string(project_input)
+
+    if input_purl.type != "cargo":
+        error_msg = "Only cargo purl is supported."
+        raise ValueError(error_msg)
+    if not input_purl.version:
+        error_msg = "Version is required."
+        raise ValueError(error_msg)
+
+    return input_purl
+
+
+def fetch_inputs(purl):
+    """Fetch the source for the given input purl"""
+    purl_str = PackageURL.to_string(purl)
+
+    purl_src_path = fetch_path(purl_str)
+
+    if not purl_src_path:
+        err_msg = f"No source could be resolved for {purl}."
+        raise ValueError(err_msg)
+
+    return purl_src_path
+
+
+def fetch_path(purl):
+    """Fetch the purl and return the location of the fetched tarball"""
+    try:
+        return fetch.fetch_url(url=purl).path
+    except (ValueError, requests.RequestException) as e:
+        logger.warning("Failed to fetch package: %s - %s", purl, e)
+        return None