Implement the rust pipeline #1767

 * Add comparison logic
 * Add tests
 * Update extra_data fields
 * Better code organization
 * etc..

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

scanpipe/pipelines/scan_rust_package.py

@@ -20,19 +20,20 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+import shutil
+import tempfile
 from pathlib import Path
 
 from scanpipe.pipelines.deploy_to_develop import DeployToDevelop
 from scanpipe.pipelines.scan_codebase import ScanCodebase
 from scanpipe.pipelines.scan_single_package import ScanSinglePackage
 from scanpipe.pipes import d2d
 from scanpipe.pipes import flag
-from scanpipe.pipes import rust
 from scanpipe.pipes import utils
-
-from scanpipe.pipes.rust import check_input_and_return_purl, fetch_inputs
-
-import shutil
+from scanpipe.pipes.rust import build_crates
+from scanpipe.pipes.rust import check_input_and_return_purl
+from scanpipe.pipes.rust import get_cargo_toml_path
+from scanpipe.pipes.rust import get_repository_value_from_cargo_toml
 
 
 class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase):
@@ -60,13 +61,18 @@ def steps(cls):
             cls.collect_input_info,
             cls.extract_input_to_codebase_directory,
             cls.check_docker_command,
+            cls.get_cargo_toml,
             cls.build_crates,
             cls.run_scan,
             cls.load_inventory_from_toolkit_scan,
             cls.add_from_to_tag,
             cls.validate_package_license_integrity,
             cls.identify_built_sources,
             cls.flag_mapped_status,
+            cls.get_src_repo_download_url,
+            cls.download_src_repo,
+            cls.compare_src_repo_with_from_codebase,
+            cls.update_comparison_summary,
             cls.make_summary_from_scan_results,
         )
 
@@ -76,32 +82,49 @@ def check_input_and_return_purl(self):
 
     def fetch_inputs(self):
         """Fetch the source of the given PURL."""
-        self.from_files = fetch_inputs(self.purl)
+        self.from_files = utils.fetch_inputs(self.purl)
 
     def collect_input_info(self):
         """Collect information about the input."""
         self.input_path = self.from_files
         self.collect_input_information()
 
     def check_docker_command(self):
+        """Check if the Docker command is available."""
         self.have_docker = False
         if shutil.which("docker"):
             self.have_docker = True
 
+    def get_cargo_toml(self):
+        """Get the Cargo.toml path from the codebase directory."""
+        self.cargo_toml_path = None
+        self.devel_codebase_dir = None
+        if self.have_docker:
+            codebase_dir = Path(self.project.codebase_path)
+            self.devel_codebase_dir = codebase_dir
+            self.cargo_toml_path = get_cargo_toml_path(codebase_dir)
+
     def build_crates(self):
         """
         Build the Rust crate using Docker and put the built files under the
         "to" directory.
         """
         self.d2d_enable = False
-        if self.have_docker:
-            if rust.build_crates(self.project.codebase_path):
+        if self.cargo_toml_path:
+            codebase_dir = self.devel_codebase_dir
+            cargo_toml_path = self.cargo_toml_path
+            if build_crates(codebase_dir, cargo_toml_path):
                 self.d2d_enable = True
+                updated_path = cargo_toml_path.relative_to(codebase_dir)
+                self.cargo_toml_path = codebase_dir / "from" / updated_path
+                self.devel_codebase_dir = codebase_dir / "from"
             else:
                 print("Docker command not found. Skipping crate build.")
+        else:
+            print("Cargo.toml is not found.")
 
     def add_from_to_tag(self):
-        """Update 'from' or 'to' tag to resources based on their path."""
+        """Update 'from' and 'to' tag to resources based on their path."""
         if self.d2d_enable:
             d2d.update_from_to_tag(self.project)
 
@@ -121,3 +144,67 @@ def flag_mapped_status(self):
         """Flag the from codebase resources that were mapped."""
         if self.d2d_enable:
             flag.flag_mapped_resources(self.project)
+
+    def get_src_repo_download_url(self):
+        """
+        Get the source repository url from Cargo.toml and determine its
+        download url.
+        """
+        self.src_download_url = None
+        repository_url = get_repository_value_from_cargo_toml(self.cargo_toml_path)
+        if not repository_url:
+            self.project.add_warning(
+                description="No source repository URL found in Cargo.toml."
+            )
+        else:
+            self.src_download_url = utils.get_download_url(
+                repository_url, self.purl.version
+            )
+            if not self.src_download_url:
+                self.project.add_warning(
+                    description=(
+                        "Not able to determine the source repository download URL from "
+                        "Cargo.toml."
+                    )
+                )
+
+    def download_src_repo(self):
+        """Download the source from the source repo."""
+        self.src_repo_path = None
+        if self.src_download_url:
+            self.src_repo_path = utils.download_src_repo(self.src_download_url)
+            if not self.src_repo_path:
+                self.project.add_warning(
+                    description=(
+                        f"The source repository URL "
+                        f"{self.src_download_url} "
+                        f"could not be downloaded. Skipping the source "
+                        f"crate and source repository comparison."
+                    )
+                )
+
+    def compare_src_repo_with_from_codebase(self):
+        """Compare the downloaded source repo with the from codebase."""
+        self.matched_count = 0
+        self.mismatches = []
+        if self.src_repo_path:
+            with tempfile.TemporaryDirectory() as source_repo_path:
+                self.extract_archive(self.src_repo_path, source_repo_path)
+
+                self.matched_count, self.mismatches = utils.compare_directories(
+                    self.devel_codebase_dir, source_repo_path
+                )
+
+    def update_comparison_summary(self):
+        """Update the comparison summary in the discovered package."""
+        if self.src_repo_path:
+            utils.update_comparison_summary(
+                self.project,
+                self.purl,
+                self.devel_codebase_dir,
+                self.src_download_url,
+                self.purl.name,
+                self.purl.version,
+                self.matched_count,
+                self.mismatches,
+            )

scanpipe/pipes/d2d.py

@@ -1989,17 +1989,15 @@ def map_rust_paths(project, logger=None):
     target_rlib_ids = []
     for resource in data_resources:
         try:
-            rlib_path, paths = get_rust_file_paths(resource.location_path)
-            absolute_rlib_path = Path(rlib_path)
+            rlib_path_str, paths = get_rust_file_paths(resource.location_path)
+            rlib_path = Path(rlib_path_str)
             rlib_resource = None
             try:
-                # Docker paths start with "/codebase". Host paths start with project.codebase_path.
-                if str(absolute_rlib_path).startswith("/codebase/"):
-                    clean_rlib_path = str(absolute_rlib_path.relative_to("/codebase"))
+                if rlib_path_str.startswith("/codebase/"):
+                    clean_rlib_path = str(rlib_path.relative_to("/codebase"))
                 else:
-                    clean_rlib_path = str(absolute_rlib_path.relative_to(project.codebase_path))
+                    clean_rlib_path = str(rlib_path.relative_to(project.codebase_path))
 
-                # We can now safely do an exact path match
                 rlib_resource = (
                     project.codebaseresources.files()
                     .to_codebase()
@@ -2013,7 +2011,9 @@ def map_rust_paths(project, logger=None):
                 rlib_resource.update_extra_data(paths)
                 target_rlib_ids.append(rlib_resource.id)
             elif logger:
-                logger(f"Warning: Could not find rlib file {absolute_rlib_path.name} in database.")
+                logger(
+                    f"Warning: Could not find rlib file {rlib_path_str} in database."
+                )
         except Exception as exception:
             project.add_warning(
                 exception=exception,

scanpipe/pipes/fetch.py

@@ -68,9 +68,9 @@ def get_request_session(uri):
 
     # Set a default User-Agent to avoid 403 Forbidden errors on strict
     # registries like crates.io that block default python-requests headers.
-    session.headers.update({
-        "User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"
-    })
+    session.headers.update(
+        {"User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"}
+    )
 
     netloc = urlparse(uri).netloc
 

scanpipe/pipes/rust.py

@@ -20,64 +20,58 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+import logging
 import shutil
 import subprocess
-import logging
-import requests
+from pathlib import Path
 
+import tomllib
 from packageurl import PackageURL
-from pathlib import Path
-from scanpipe.pipes import fetch
-from scanpipe.pipes import run_command_safely
 
+from scanpipe.pipes import run_command_safely
 
 logger = logging.getLogger(__name__)
 
 
-def build_crates(codebase_dir):
+def build_crates(codebase_dir, cargo_toml_path):
     """
-    Build the Rust crate from sources in an isolated Docker container.
-
-    Uses the official rust image to safely sandbox the build process and
-    injects RUSTFLAGS to force DWARF debug symbol generation (-C debuginfo=2)
-    required for binary-to-source mapping.
-
-    Return True if build successfully, False otherwise.
+    Build the Rust crate from source in an isolated Docker container.
+    Use the official Rust image for the build process.
+    Return True if the build succeeds, False otherwise.
     """
-
-    # Find the Cargo.toml file in the codebase directory
-    codebase_dir = Path(codebase_dir)
-    cargo_toml_path = None
-    for path in codebase_dir.rglob("Cargo.toml"):
-        cargo_toml_path = path
-        break
-    if cargo_toml_path:
-        to_dir = codebase_dir / "to"
-    else:
-        return False
-
+    to_dir = codebase_dir / "to"
     cargo_toml_path = Path(cargo_toml_path)
     build_dir = Path(to_dir)
 
-    # Calculate paths relative to the container's mounted /codebase directory
-    rel_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix()
-    rel_build_dir = build_dir.relative_to(codebase_dir).as_posix()
+    # Get the relative paths
+    relative_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix()
+    relative_build_dir = build_dir.relative_to(codebase_dir).as_posix()
 
-    container_cargo_toml = f"/codebase/{rel_cargo_toml}"
-    container_build_dir = f"/codebase/{rel_build_dir}"
+    container_cargo_toml = f"/codebase/{relative_cargo_toml}"
+    container_build_dir = f"/codebase/{relative_build_dir}"
 
+    # Since we will use the .d file for deployment and development file
+    # mapping, we will not require building with DWARF debug symbols. If we
+    # later decide to include DWARF, we can add the following to the
+    # command:
+    # "--env", "RUSTFLAGS=-C debuginfo=2",
     cmd = [
-        "docker", "run",
-        "--rm",  # Automatically remove the container when it exits
-        "--volume", f"{codebase_dir}:/codebase",
-        "--workdir", "/codebase",
-        "--env", "RUSTFLAGS=-C debuginfo=2",  # Force DWARF generation in release mode
+        "docker",
+        "run",
+        "--rm",
+        "--volume",
+        f"{codebase_dir}:/codebase",
+        "--workdir",
+        "/codebase",
         "rust:latest",
-        "cargo", "build",
+        "cargo",
+        "build",
         "--release",
         "--locked",
-        "--manifest-path", container_cargo_toml,
-        "--target-dir", container_build_dir,
+        "--manifest-path",
+        container_cargo_toml,
+        "--target-dir",
+        container_build_dir,
     ]
 
     try:
@@ -86,6 +80,7 @@ def build_crates(codebase_dir):
         logger.warning(f"Failed to build the Rust crate in Docker: {error}")
         return False
 
+    # Move the development code under the /codebase/from/
     from_dir = codebase_dir / "from"
     from_dir.mkdir(exist_ok=True)
     for item in codebase_dir.iterdir():
@@ -100,7 +95,7 @@ def check_input_and_return_purl(project):
     if len(input_sources) != 1:
         error_msg = "Only 1 cargo purl is accepted."
         raise ValueError(error_msg)
-    # Strip the qualifiers as this is not needed.
+    # Strip the qualifiers if present as this is not needed
     project_input = str(input_sources[0]).split("?")[0]
     input_purl = PackageURL.from_string(project_input)
 
@@ -114,23 +109,23 @@ def check_input_and_return_purl(project):
     return input_purl
 
 
-def fetch_inputs(purl):
-    """Fetch the source for the given input purl"""
-    purl_str = PackageURL.to_string(purl)
-
-    purl_src_path = fetch_path(purl_str)
+def get_repository_value_from_cargo_toml(cargo_toml_path):
+    """Get the repository value from Cargo.toml."""
+    path = Path(cargo_toml_path)
+    if not path.exists():
+        raise FileNotFoundError(f"{cargo_toml_path} not found")
 
-    if not purl_src_path:
-        err_msg = f"No source could be resolved for {purl}."
-        raise ValueError(err_msg)
+    with path.open("rb") as f:
+        data = tomllib.load(f)
 
-    return purl_src_path
+    return data.get("package", {}).get("repository", "")
 
 
-def fetch_path(purl):
-    """Fetch the purl and return the location of the fetched tarball"""
-    try:
-        return fetch.fetch_url(url=purl).path
-    except (ValueError, requests.RequestException) as e:
-        logger.warning("Failed to fetch package: %s - %s", purl, e)
-        return None
+def get_cargo_toml_path(codebase_dir):
+    """Get the Cargo.toml path from the codebase directory."""
+    cargo_toml_path = None
+    # There is only one "Cargo.toml" per published package
+    for path in codebase_dir.rglob("Cargo.toml"):
+        cargo_toml_path = path
+        break
+    return cargo_toml_path