From e9ceab48d597a68f7149fdf83692bd80f26431b6 Mon Sep 17 00:00:00 2001 From: Alex Newman Date: Thu, 26 Mar 2026 23:10:19 +0000 Subject: [PATCH] fix(agent): fix registration bugs and upgrade VM scripts to libvirt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent fixes: - Fix API paths: /api/agents/ → /api/v1/agents/ (matched CP routes) - Fix register body: "quote" → "intel_ta_token" (matched CP request struct) - Add retry-based registration with exponential backoff (30 attempts) - Add TDX fallback: use dummy token on non-TDX hardware VM scripts: - Rewrite vm-launch.sh for libvirt (virsh define/start instead of raw QEMU) - Rewrite vm-stop.sh for libvirt (virsh shutdown/destroy/undefine) - Rewrite vm-status.sh for libvirt (virsh domstate) Co-Authored-By: Claude Opus 4.6 (1M context) --- agent/src/bin/dd-agent/main.rs | 86 ++++++----- infra/scripts/vm-launch.sh | 268 ++++++++++++++++++++++++--------- infra/scripts/vm-status.sh | 38 ++--- infra/scripts/vm-stop.sh | 42 +++--- 4 files changed, 289 insertions(+), 145 deletions(-) diff --git a/agent/src/bin/dd-agent/main.rs b/agent/src/bin/dd-agent/main.rs index 6b00e9b..b95bda9 100644 --- a/agent/src/bin/dd-agent/main.rs +++ b/agent/src/bin/dd-agent/main.rs @@ -37,7 +37,6 @@ async fn run_agent_mode(cfg: AgentRuntimeConfig) { } }; - // 1. Build an HTTP client. let http = match reqwest::Client::builder() .danger_accept_invalid_certs(false) .build() @@ -49,65 +48,73 @@ async fn run_agent_mode(cfg: AgentRuntimeConfig) { } }; - // 2. Obtain a challenge nonce from the control plane. - let challenge = match fetch_challenge(&http, &cp_url).await { - Ok(c) => c, - Err(e) => { - eprintln!("dd-agent: challenge failed: {e}"); - std::process::exit(1); - } - }; - - eprintln!( - "dd-agent: received nonce (expires in {}s)", - challenge.expires_in_seconds - ); - - // 3. Generate a TDX quote embedding the nonce as report data. - let quote_b64 = match dd_agent::attestation::tsm::generate_tdx_quote_base64() { - Ok(q) => q, + let registration = match register_with_retry(&http, &cp_url, &cfg).await { + Ok(r) => r, Err(e) => { - eprintln!("dd-agent: TDX quote generation failed: {e}"); + eprintln!("dd-agent: registration failed: {e}"); std::process::exit(1); } }; - // 4. Register with the control plane. - let registration = - match register_agent(&http, &cp_url, &challenge.nonce, "e_b64, &cfg).await { - Ok(r) => r, - Err(e) => { - eprintln!("dd-agent: registration failed: {e}"); - std::process::exit(1); - } - }; - eprintln!( "dd-agent: registered as {} at {}", registration.agent_id, registration.hostname ); - // 5. Start cloudflared tunnel. if let Err(e) = start_cloudflared(®istration.tunnel_token).await { eprintln!("dd-agent: cloudflared start failed: {e}"); - // Non-fatal: continue to workload. } - // 6. Run workload containers. if let Err(e) = run_workloads(&cfg).await { eprintln!("dd-agent: workload launch failed: {e}"); } - // 7. Heartbeat / reconciliation loop. let agent_id = registration.agent_id.clone(); heartbeat_loop(&http, &cp_url, &agent_id).await; } +async fn register_with_retry( + http: &reqwest::Client, + cp_url: &str, + cfg: &AgentRuntimeConfig, +) -> Result { + let max_retries = 30u32; + + for attempt in 1..=max_retries { + let challenge = match fetch_challenge(http, cp_url).await { + Ok(c) => c, + Err(e) => { + eprintln!("dd-agent: challenge failed (attempt {attempt}/{max_retries}): {e}"); + backoff_sleep(attempt).await; + continue; + } + }; + + let quote_b64 = match dd_agent::attestation::tsm::generate_tdx_quote_base64() { + Ok(q) => q, + Err(e) => { + eprintln!("dd-agent: TDX quote generation failed, using fallback: {e}"); + "no-tdx-available".to_string() + } + }; + + match register_agent(http, cp_url, &challenge.nonce, "e_b64, cfg).await { + Ok(r) => return Ok(r), + Err(e) => { + eprintln!("dd-agent: registration failed (attempt {attempt}/{max_retries}): {e}"); + backoff_sleep(attempt).await; + } + } + } + + Err(format!("failed after {max_retries} attempts")) +} + async fn fetch_challenge( http: &reqwest::Client, cp_url: &str, ) -> Result { - let url = format!("{cp_url}/api/agents/challenge"); + let url = format!("{cp_url}/api/v1/agents/challenge"); let resp = http .get(&url) .send() @@ -130,11 +137,11 @@ async fn register_agent( quote_b64: &str, cfg: &AgentRuntimeConfig, ) -> Result { - let url = format!("{cp_url}/api/agents/register"); + let url = format!("{cp_url}/api/v1/agents/register"); let body = serde_json::json!({ "nonce": nonce, - "quote": quote_b64, + "intel_ta_token": quote_b64, "vm_name": hostname(), "node_size": cfg.node_size, "datacenter": cfg.datacenter, @@ -223,7 +230,7 @@ async fn run_workloads(cfg: &AgentRuntimeConfig) -> Result<(), String> { } async fn heartbeat_loop(http: &reqwest::Client, cp_url: &str, agent_id: &str) { - let url = format!("{cp_url}/api/agents/{agent_id}/heartbeat"); + let url = format!("{cp_url}/api/v1/agents/{agent_id}/heartbeat"); let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); loop { @@ -310,3 +317,8 @@ fn hostname() -> String { .trim() .to_string() } + +async fn backoff_sleep(attempt: u32) { + let secs = std::cmp::min(5 * 2u64.saturating_pow(attempt.saturating_sub(1)), 60); + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; +} diff --git a/infra/scripts/vm-launch.sh b/infra/scripts/vm-launch.sh index 3bd459b..29f31b4 100755 --- a/infra/scripts/vm-launch.sh +++ b/infra/scripts/vm-launch.sh @@ -1,10 +1,5 @@ #!/usr/bin/env bash -# Launch a QEMU/KVM VM from a baked qcow2 image. -# -# Usage: -# ./vm-launch.sh --image /path/to/base.qcow2 --name dd-cp-staging \ -# --config /path/to/config.json --memory 8G --cpus 4 \ -# --port-forward 8080:8080 +# Launch a libvirt-managed VM from a baked qcow2 image. set -euo pipefail IMAGE="" @@ -14,8 +9,10 @@ MEMORY="4G" CPUS="2" PORT_FORWARDS=() VFIO_DEVICE="" +TDX="false" VM_DIR="/var/lib/devopsdefender/vms" -CONFIG_MODE="agent" # agent or control-plane +CONFIG_MODE="agent" +LIBVIRT_NETWORK="${LIBVIRT_NETWORK:-default}" usage() { cat </dev/null 2>&1 || { + echo "Error: required command not found: $1" >&2 + exit 1 + } +} + +detect_libvirt_qemu_owner() { + if [ -n "${LIBVIRT_QEMU_USER:-}" ]; then + LIBVIRT_QEMU_GROUP="${LIBVIRT_QEMU_GROUP:-$(id -gn "$LIBVIRT_QEMU_USER" 2>/dev/null || true)}" + return 0 + fi + + for candidate in libvirt-qemu qemu; do + if id -u "$candidate" >/dev/null 2>&1; then + LIBVIRT_QEMU_USER="$candidate" + LIBVIRT_QEMU_GROUP="$(id -gn "$candidate")" + return 0 + fi + done + + LIBVIRT_QEMU_USER="" + LIBVIRT_QEMU_GROUP="" +} + +prepare_runtime_permissions() { + if [ "$(id -u)" -ne 0 ]; then + return 0 + fi + + detect_libvirt_qemu_owner + if [ -z "$LIBVIRT_QEMU_USER" ] || [ -z "$LIBVIRT_QEMU_GROUP" ]; then + return 0 + fi + + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$VM_WORK_DIR" + chmod 0770 "$VM_WORK_DIR" + + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$OVERLAY" "$CIDATA_ISO" + chmod 0660 "$OVERLAY" "$CIDATA_ISO" + + touch "$SERIAL_LOG" + chown "$LIBVIRT_QEMU_USER:$LIBVIRT_QEMU_GROUP" "$SERIAL_LOG" + chmod 0660 "$SERIAL_LOG" +} + +to_mib() { + local value number unit + value="${1^^}" + if [[ "$value" =~ ^([0-9]+)([GM])I?B?$ ]]; then + number="${BASH_REMATCH[1]}" + unit="${BASH_REMATCH[2]}" + elif [[ "$value" =~ ^([0-9]+)$ ]]; then + echo "$value" + return 0 + else + echo "Error: unsupported memory value '$1' (use 4096, 4G, 8192M)" >&2 + exit 1 + fi + + if [ "$unit" = "G" ]; then + echo $((number * 1024)) + else + echo "$number" + fi +} + +escape_xml() { + sed \ + -e 's/&/\&/g' \ + -e 's//\>/g' \ + -e "s/'/\'/g" \ + -e 's/"/\"/g' +} + while [[ $# -gt 0 ]]; do case "$1" in --image) IMAGE="$2"; shift 2 ;; @@ -42,6 +116,7 @@ while [[ $# -gt 0 ]]; do --cpus) CPUS="$2"; shift 2 ;; --port-forward) PORT_FORWARDS+=("$2"); shift 2 ;; --vfio-device) VFIO_DEVICE="$2"; shift 2 ;; + --tdx) TDX="true"; shift ;; --help|-h) usage ;; *) echo "Unknown option: $1" >&2; usage ;; esac @@ -52,17 +127,12 @@ if [ -z "$IMAGE" ] || [ -z "$VM_NAME" ] || [ -z "$CONFIG_FILE" ]; then usage fi -if [ ! -f "$IMAGE" ]; then - echo "Error: base image not found: $IMAGE" >&2 - exit 1 -fi +require_cmd virsh +require_cmd qemu-img -if [ ! -f "$CONFIG_FILE" ]; then - echo "Error: config file not found: $CONFIG_FILE" >&2 - exit 1 -fi +[ -f "$IMAGE" ] || { echo "Error: base image not found: $IMAGE" >&2; exit 1; } +[ -f "$CONFIG_FILE" ] || { echo "Error: config file not found: $CONFIG_FILE" >&2; exit 1; } -# Create VM working directory. VM_WORK_DIR="${VM_DIR}/${VM_NAME}" mkdir -p "$VM_WORK_DIR" @@ -87,8 +157,6 @@ fi # Generate cloud-init ISO for config injection. CIDATA_DIR="${VM_WORK_DIR}/cidata" mkdir -p "$CIDATA_DIR" - -# Escape JSON for embedding in cloud-init write_files. CONFIG_CONTENT="$(cat "$CONFIG_FILE")" cat > "${CIDATA_DIR}/user-data" </dev/null; then +if command -v cloud-localds >/dev/null 2>&1; then cloud-localds "$CIDATA_ISO" "${CIDATA_DIR}/user-data" "${CIDATA_DIR}/meta-data" -elif command -v genisoimage &>/dev/null; then +elif command -v genisoimage >/dev/null 2>&1; then genisoimage -output "$CIDATA_ISO" -volid cidata -joliet -rock \ "${CIDATA_DIR}/user-data" "${CIDATA_DIR}/meta-data" else @@ -121,77 +189,139 @@ else exit 1 fi -# Build QEMU command. -QEMU_ARGS=( - qemu-system-x86_64 - -enable-kvm - -machine q35 - -cpu host - -m "$MEMORY" - -smp "$CPUS" - -drive "file=${OVERLAY},format=qcow2,if=virtio" - -drive "file=${CIDATA_ISO},format=raw,if=virtio,readonly=on" - -display none - -serial file:${VM_WORK_DIR}/${VM_NAME}.log - -daemonize - -pidfile "${VM_WORK_DIR}/${VM_NAME}.pid" -) - -# Pass through VFIO device (GPU) if requested. -if [ -n "$VFIO_DEVICE" ]; then - QEMU_ARGS+=(-device "vfio-pci,host=${VFIO_DEVICE}") +# Build libvirt domain XML. +MEMORY_MIB="$(to_mib "$MEMORY")" +DOMAIN_XML="${VM_WORK_DIR}/${VM_NAME}.xml" +SERIAL_LOG="${VM_WORK_DIR}/${VM_NAME}.log" + +prepare_runtime_permissions + +LAUNCH_SECURITY="" +FEATURES_EXTRA="" +CLOCK_XML=" \n" +PM_XML="" +MEMORY_BACKING_XML="" +OS_OPEN_TAG=" " +LOADER_XML="" +if [ "$TDX" = "true" ]; then + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + MEMORY_BACKING_XML+=" \n" + OS_OPEN_TAG=" " + LOADER_XML+=" /usr/share/qemu/OVMF.fd\n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" 0x10000000\n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + LAUNCH_SECURITY+=" \n" + FEATURES_EXTRA+=" \n" + CLOCK_XML=" \n" + CLOCK_XML+=" \n" + CLOCK_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" + PM_XML+=" \n" fi -# Build port forwarding netdev. -HOSTFWD_ARGS="" -for pf in "${PORT_FORWARDS[@]+"${PORT_FORWARDS[@]}"}"; do - host_port="${pf%%:*}" - guest_port="${pf##*:}" - HOSTFWD_ARGS="${HOSTFWD_ARGS},hostfwd=tcp::${host_port}-:${guest_port}" -done +HOSTDEV_XML="" +if [ -n "$VFIO_DEVICE" ]; then + domain_hex="${VFIO_DEVICE%%:*}" + remainder="${VFIO_DEVICE#*:}" + bus_hex="${remainder%%.*}" + function_hex="${remainder##*.}" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+="
\n" + HOSTDEV_XML+=" \n" + HOSTDEV_XML+=" \n" +fi -# Always forward SSH on a high port for debugging. -SSH_PORT=$((10000 + RANDOM % 50000)) -HOSTFWD_ARGS="${HOSTFWD_ARGS},hostfwd=tcp::${SSH_PORT}-:22" +cat > "$DOMAIN_XML" < + ${VM_NAME} + ${MEMORY_MIB} + ${MEMORY_MIB} +$(printf "%b" "$MEMORY_BACKING_XML") ${CPUS} +$(printf "%b" "$OS_OPEN_TAG") + hvm +$(printf "%b" "$LOADER_XML") + +$(printf "%b" "$LAUNCH_SECURITY") + + +$(printf "%b" "$FEATURES_EXTRA") + +$(printf "%b" "$CLOCK_XML") destroy + restart + destroy +$(printf "%b" "$PM_XML") + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + + + + + + + + + + /dev/urandom + +$(printf "%b" "$HOSTDEV_XML") + +EOF -QEMU_ARGS+=(-netdev "user,id=net0${HOSTFWD_ARGS}" -device "virtio-net-pci,netdev=net0") +if virsh dominfo "$VM_NAME" >/dev/null 2>&1; then + echo "Error: domain '$VM_NAME' already exists; stop it first" >&2 + exit 1 +fi -echo "==> Launching VM: ${VM_NAME}" +echo "==> Defining libvirt domain: ${VM_NAME}" echo " Memory: ${MEMORY}, CPUs: ${CPUS}" echo " Overlay: ${OVERLAY}" echo " Config mode: ${CONFIG_MODE}" -echo " SSH port: ${SSH_PORT}" +echo " Libvirt network: ${LIBVIRT_NETWORK}" +echo " TDX: ${TDX}" if [ -n "$VFIO_DEVICE" ]; then echo " VFIO device: ${VFIO_DEVICE}" fi -for pf in "${PORT_FORWARDS[@]+"${PORT_FORWARDS[@]}"}"; do - echo " Port forward: ${pf}" -done -"${QEMU_ARGS[@]}" /dev/null +virsh start "$VM_NAME" >/dev/null -PID_FILE="${VM_WORK_DIR}/${VM_NAME}.pid" -if [ -f "$PID_FILE" ]; then - PID="$(cat "$PID_FILE")" - echo "==> VM started with PID ${PID}" - echo " PID file: ${PID_FILE}" - echo " SSH: ssh -p ${SSH_PORT} ubuntu@localhost" -else - echo "Warning: PID file not created, VM may not have started" >&2 -fi +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +echo "==> VM started with libvirt state: ${STATE}" +echo " Inspect with: virsh list --all" -# Write metadata for vm-status.sh / vm-stop.sh. cat > "${VM_WORK_DIR}/vm-info.json" </dev/null 2>&1 || { + echo "Error: virsh is required" >&2 + exit 1 +} + if [ ! -d "$VM_DIR" ]; then echo "No VMs found (${VM_DIR} does not exist)" exit 0 fi -printf "%-25s %-8s %-15s %-6s %-6s %-8s %s\n" \ - "NAME" "PID" "STATUS" "MEM" "CPUS" "SSH" "STARTED" -printf "%s\n" "$(printf '%.0s-' {1..100})" +printf "%-25s %-12s %-6s %-6s %-12s %s\n" \ + "NAME" "STATE" "MEM" "CPUS" "NETWORK" "STARTED" +printf "%s\n" "$(printf '%.0s-' {1..90})" found=0 for vm_dir in "${VM_DIR}"/*/; do @@ -23,27 +27,25 @@ for vm_dir in "${VM_DIR}"/*/; do found=1 name="$(jq -r '.name // "unknown"' "$info_file")" - pid_file="$(jq -r '.pid_file // ""' "$info_file")" memory="$(jq -r '.memory // "?"' "$info_file")" cpus="$(jq -r '.cpus // "?"' "$info_file")" - ssh_port="$(jq -r '.ssh_port // "?"' "$info_file")" + network="$(jq -r '.libvirt_network // "?"' "$info_file")" started="$(jq -r '.started_at // "?"' "$info_file")" - status="stopped" - pid="-" - if [ -n "$pid_file" ] && [ -f "$pid_file" ]; then - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - status="running" - else - status="dead" - fi + state="undefined" + if virsh dominfo "$name" >/dev/null 2>&1; then + state="$(virsh domstate "$name" | tr -d '\r' | xargs)" fi - printf "%-25s %-8s %-15s %-6s %-6s %-8s %s\n" \ - "$name" "$pid" "$status" "$memory" "$cpus" "$ssh_port" "$started" + printf "%-25s %-12s %-6s %-6s %-12s %s\n" \ + "$name" "$state" "$memory" "$cpus" "$network" "$started" done if [ "$found" -eq 0 ]; then echo "No VMs found" + exit 0 fi + +echo +echo "virsh list --all" +virsh list --all diff --git a/infra/scripts/vm-stop.sh b/infra/scripts/vm-stop.sh index 5c67ff4..0fd0e95 100755 --- a/infra/scripts/vm-stop.sh +++ b/infra/scripts/vm-stop.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Stop a VM by name. +# Stop a libvirt-managed VM by name. # Usage: ./vm-stop.sh [--clean] set -euo pipefail @@ -22,39 +22,39 @@ if [ -z "$VM_NAME" ]; then fi VM_WORK_DIR="${VM_DIR}/${VM_NAME}" -PID_FILE="${VM_WORK_DIR}/${VM_NAME}.pid" -if [ ! -f "$PID_FILE" ]; then - echo "No PID file found for VM '${VM_NAME}' at ${PID_FILE}" >&2 +command -v virsh >/dev/null 2>&1 || { + echo "Error: virsh is required" >&2 exit 1 -fi +} -PID="$(cat "$PID_FILE")" +if ! virsh dominfo "$VM_NAME" >/dev/null 2>&1; then + echo "No libvirt domain found for VM '${VM_NAME}'" >&2 + exit 1 +fi -if kill -0 "$PID" 2>/dev/null; then - echo "==> Stopping VM '${VM_NAME}' (PID ${PID})" - kill "$PID" +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +echo "==> Stopping VM '${VM_NAME}' (state: ${STATE})" - # Wait for process to exit (up to 30s). +if [ "$STATE" = "running" ] || [ "$STATE" = "paused" ] || [ "$STATE" = "in shutdown" ]; then + virsh shutdown "$VM_NAME" >/dev/null || true for _ in $(seq 1 30); do - if ! kill -0 "$PID" 2>/dev/null; then + STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" + if [ "$STATE" = "shut off" ]; then break fi sleep 1 done +fi - # Force kill if still running. - if kill -0 "$PID" 2>/dev/null; then - echo " Force killing PID ${PID}" - kill -9 "$PID" 2>/dev/null || true - fi - - echo "==> VM '${VM_NAME}' stopped" -else - echo "VM '${VM_NAME}' is not running (PID ${PID})" +STATE="$(virsh domstate "$VM_NAME" | tr -d '\r' | xargs)" +if [ "$STATE" != "shut off" ]; then + echo " Force destroying domain ${VM_NAME}" + virsh destroy "$VM_NAME" >/dev/null || true fi -rm -f "$PID_FILE" +virsh undefine "$VM_NAME" --nvram >/dev/null 2>&1 || virsh undefine "$VM_NAME" >/dev/null +echo "==> VM '${VM_NAME}' undefined" if [ "$CLEAN" = true ]; then echo "==> Cleaning up VM directory: ${VM_WORK_DIR}"