From 656edbb7573478b6140b4bfba0e38af467a9ade1 Mon Sep 17 00:00:00 2001 From: "developing-today[bot]" <5454422+developing-today[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:11:11 -0600 Subject: [PATCH 1/3] atlas-search: fetch-model provisioning verb + baked ONNX Runtime default (step 5, #5355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetch-model provisions the pinned Snowflake snapshot (HF rev 95c2741480856aa9666782eb4afe11959938017f; tokenizer.json + onnx/model.onnx, sha256 recorded in-source per the Q1 ruling — state-dir fetch, not a nix FOD) into ~/.local/state/atlas-search/model//: streamed download with inline sha256, verify-before-install, staging dir + atomic rename promote, idempotent no-op on a verifying snapshot, loud refusal on hash mismatch, and a mkdir provisioning lock so concurrent runs cannot interleave writes into the shared staging dir (race observed live during step-5 verification; a killed run leaves the lock and the error says how to clear it). resolve_ort_lib gains a final compile-time fallback from ATLAS_SEARCH_ORT_LIB_DEFAULT so the nix package bakes its closure onnxruntime path in. Fixtures: make_fixture_db.py + smoke_check.py, the install-check harness (FTS/hybrid envelopes byte-compared to the Python recordings; ORT-load probe proves ort::init_from succeeds from the baked path). --- pkgs/atlas-search/Cargo.lock | 66 +++++ .../crates/atlas-search/Cargo.toml | 1 + .../crates/atlas-search/src/fetch_model.rs | 278 ++++++++++++++++++ .../crates/atlas-search/src/main.rs | 32 +- pkgs/atlas-search/fixtures/make_fixture_db.py | 49 +++ pkgs/atlas-search/fixtures/smoke_check.py | 123 ++++++++ 6 files changed, 543 insertions(+), 6 deletions(-) create mode 100644 pkgs/atlas-search/crates/atlas-search/src/fetch_model.rs create mode 100644 pkgs/atlas-search/fixtures/make_fixture_db.py create mode 100644 pkgs/atlas-search/fixtures/smoke_check.py diff --git a/pkgs/atlas-search/Cargo.lock b/pkgs/atlas-search/Cargo.lock index a1d255e62..a747ec017 100644 --- a/pkgs/atlas-search/Cargo.lock +++ b/pkgs/atlas-search/Cargo.lock @@ -124,6 +124,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2", "ureq", ] @@ -157,6 +158,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bytes" version = "1.12.1" @@ -249,6 +259,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.4" @@ -283,6 +302,16 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.20.11" @@ -358,6 +387,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.19.0" @@ -423,6 +462,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1030,6 +1079,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1172,6 +1232,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.27" diff --git a/pkgs/atlas-search/crates/atlas-search/Cargo.toml b/pkgs/atlas-search/crates/atlas-search/Cargo.toml index bf65a12f1..4749f51e5 100644 --- a/pkgs/atlas-search/crates/atlas-search/Cargo.toml +++ b/pkgs/atlas-search/crates/atlas-search/Cargo.toml @@ -17,6 +17,7 @@ atlas-embed = { path = "../atlas-embed" } clap = { version = "4", features = ["derive", "env"] } serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } +sha2 = "0.10" ureq = { version = "3", features = ["rustls"] } [dev-dependencies] diff --git a/pkgs/atlas-search/crates/atlas-search/src/fetch_model.rs b/pkgs/atlas-search/crates/atlas-search/src/fetch_model.rs new file mode 100644 index 000000000..7e740e6e3 --- /dev/null +++ b/pkgs/atlas-search/crates/atlas-search/src/fetch_model.rs @@ -0,0 +1,278 @@ +//! The `fetch-model` verb (step 5, code#5355; design section 4, +//! Q1 as ruled): provision the pinned Snowflake model snapshot +//! into the state dir, outside the nix store. +//! +//! The model is data, not code. `~/.local/state/atlas-search/ +//! model//` holds exactly the two files the embedder loads +//! (`tokenizer.json`, `onnx/model.onnx`), each verified against +//! the sha256 constants recorded in [`MODEL_FILES`] — the same +//! hashes the Gate A fixtures record (`fixtures/embed-probes.json` +//! provenance). Downloads land in a sibling temp dir and the +//! completed set renames into place in one move; a snapshot that +//! already verifies is a no-op, and a hash mismatch at any point +//! refuses the install loudly (this verb + the loud embedder-init +//! failure are the design's mitigation for the state-dir choice). + +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +/// The pinned HuggingFace repository (design section 4). +pub const MODEL_REPO: &str = "Snowflake/snowflake-arctic-embed-m-v2.0"; +/// The pinned HuggingFace revision (fixtures/README.md; Gate A). +pub const MODEL_REV: &str = "95c2741480856aa9666782eb4afe11959938017f"; +/// The pinned file set: (path within the snapshot, sha256). Only +/// the files atlas-embed loads are provisioned. +pub const MODEL_FILES: [(&str, &str); 2] = [ + ( + "tokenizer.json", + "f1cc44ad7faaeec47241864835473fd5403f2da94673f3f764a77ebcb0a803ec", + ), + ( + "onnx/model.onnx", + "c0c53d7f49a2db60761b92b7bbf5be87a7b3cf5d92dbbd7f1b5028bd5a40aa39", + ), +]; + +/// `~/.local/state/atlas-search` — the same state-root convention +/// `init`/`serve` use for `service.json`. +pub fn state_root() -> PathBuf { + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("~")); + home.join(".local/state/atlas-search") +} + +/// The provisioned snapshot dir for the pinned revision. +pub fn model_dir() -> PathBuf { + state_root().join("model").join(MODEL_REV) +} + +fn sha256_file(path: &Path) -> Result { + let mut f = fs::File::open(path).map_err(|e| format!("cannot open {}: {e}", path.display()))?; + let mut h = Sha256::new(); + let mut buf = [0u8; 1 << 20]; + loop { + let n = f + .read(&mut buf) + .map_err(|e| format!("cannot read {}: {e}", path.display()))?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + Ok(format!("{:x}", h.finalize())) +} + +/// Every file in `files` present in `dir` with a matching hash. +fn verify_dir_with(dir: &Path, files: &[(&str, &str)]) -> Result { + for (rel, want) in files { + let p = dir.join(rel); + if !p.is_file() { + return Ok(false); + } + if sha256_file(&p)? != *want { + return Ok(false); + } + } + Ok(true) +} + +/// The pinned snapshot in `dir` verifies, file for file. +fn verify_dir(dir: &Path) -> Result { + verify_dir_with(dir, &MODEL_FILES) +} + +fn download(url: &str, dest: &Path, want: &str, rel: &str) -> Result<(), String> { + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("cannot create {}: {e}", parent.display()))?; + } + let mut resp = ureq::get(url) + .call() + .map_err(|e| format!("download of {rel} failed ({url}): {e}"))?; + let status = resp.status().as_u16(); + if status != 200 { + return Err(format!( + "download of {rel} failed: HTTP {status} from {url}" + )); + } + let mut out = + fs::File::create(dest).map_err(|e| format!("cannot create {}: {e}", dest.display()))?; + let mut reader = resp.body_mut().as_reader(); + let mut h = Sha256::new(); + let mut buf = [0u8; 1 << 20]; + let mut total: u64 = 0; + loop { + let n = reader + .read(&mut buf) + .map_err(|e| format!("download of {rel} failed mid-body: {e}"))?; + if n == 0 { + break; + } + h.update(&buf[..n]); + out.write_all(&buf[..n]) + .map_err(|e| format!("cannot write {}: {e}", dest.display()))?; + total += n as u64; + } + out.flush() + .map_err(|e| format!("cannot write {}: {e}", dest.display()))?; + let got = format!("{:x}", h.finalize()); + if got != want { + return Err(format!( + "sha256 mismatch for {rel}: downloaded {got}, pinned {want} — refusing to install" + )); + } + println!("atlas-search: fetched {rel} ({total} bytes, sha256 verified)"); + Ok(()) +} + +/// Mutual exclusion for provisioning runs. Two `fetch-model` +/// processes sharing the staging dir interleave writes into the +/// same files (observed live during step-5 verification, where +/// orphaned runs raced): the mkdir is the atomic test-and-set, +/// and the guard removes it on exit. A killed run leaves the +/// lock behind; the error tells the operator how to clear it. +struct FetchLock(PathBuf); + +impl Drop for FetchLock { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn acquire_fetch_lock(model_root: &Path) -> Result { + let lock = model_root.join(".fetch-model.lock"); + match fs::create_dir(&lock) { + Ok(()) => Ok(FetchLock(lock)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Err(format!( + "another fetch-model run is in progress (lock {} exists); if no run is active, a previous run was killed mid-provision — remove the lock dir and retry", + lock.display() + )), + Err(e) => Err(format!( + "cannot take the fetch-model lock {}: {e}", + lock.display() + )), + } +} + +/// Provision the pinned model into the state dir. Idempotent: +/// a snapshot that verifies is left untouched. +pub fn fetch_model() -> Result<(), String> { + let target = model_dir(); + let model_root = state_root().join("model"); + fs::create_dir_all(&model_root) + .map_err(|e| format!("cannot create {}: {e}", model_root.display()))?; + let _lock = acquire_fetch_lock(&model_root)?; + if verify_dir(&target)? { + println!( + "atlas-search: model {MODEL_REV} already provisioned at {} (sha256 verified); nothing to do", + target.display() + ); + return Ok(()); + } + let tmp = model_root.join(format!(".fetch-{MODEL_REV}")); + if tmp.exists() { + fs::remove_dir_all(&tmp) + .map_err(|e| format!("cannot clear stale {}: {e}", tmp.display()))?; + } + fs::create_dir_all(&tmp).map_err(|e| format!("cannot create {}: {e}", tmp.display()))?; + let result = (|| { + for (rel, want) in MODEL_FILES { + let url = format!("https://huggingface.co/{MODEL_REPO}/resolve/{MODEL_REV}/{rel}"); + download(&url, &tmp.join(rel), want, rel)?; + } + Ok::<(), String>(()) + })(); + if let Err(e) = result { + let _ = fs::remove_dir_all(&tmp); + return Err(e); + } + // Atomic swap: the completed temp dir renames into place; a + // pre-existing (failing) snapshot is moved aside first and + // dropped once the new one is in. + let aside = model_root.join(format!(".previous-{MODEL_REV}")); + if aside.exists() { + fs::remove_dir_all(&aside).map_err(|e| format!("cannot clear {}: {e}", aside.display()))?; + } + let had_target = target.exists(); + if had_target { + fs::rename(&target, &aside) + .map_err(|e| format!("cannot move {} aside: {e}", target.display()))?; + } + if let Err(e) = fs::rename(&tmp, &target) { + if had_target { + let _ = fs::rename(&aside, &target); + } + return Err(format!("cannot install {}: {e}", target.display())); + } + if had_target { + let _ = fs::remove_dir_all(&aside); + } + println!( + "atlas-search: model {MODEL_REV} provisioned at {}", + target.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sha256_file_matches_known_vector() { + // FIPS 180-4 example: sha256("abc"). + let dir = std::env::temp_dir().join(format!("atlas-fetch-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let p = dir.join("abc"); + std::fs::write(&p, b"abc").unwrap(); + assert_eq!( + sha256_file(&p).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn verify_dir_with_toy_files() { + let dir = std::env::temp_dir().join(format!("atlas-fetch-test2-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let files: [(&str, &str); 1] = [( + "a.txt", + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + )]; + // Missing file: does not verify. + assert!(!verify_dir_with(&dir, &files).unwrap()); + // Wrong bytes: does not verify. + std::fs::write(dir.join("a.txt"), b"abd").unwrap(); + assert!(!verify_dir_with(&dir, &files).unwrap()); + // Exact bytes: verifies. + std::fs::write(dir.join("a.txt"), b"abc").unwrap(); + assert!(verify_dir_with(&dir, &files).unwrap()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn fetch_lock_excludes_second_holder() { + let dir = std::env::temp_dir().join(format!("atlas-fetch-test3-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let root = dir.join("model"); + std::fs::create_dir_all(&root).unwrap(); + let first = acquire_fetch_lock(&root).unwrap(); + // A second holder is refused while the first lives. + assert!(acquire_fetch_lock(&root).is_err()); + drop(first); + // And the lock frees on drop. + assert!(acquire_fetch_lock(&root).is_ok()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn model_dir_ends_with_pinned_rev() { + assert!(model_dir().ends_with(MODEL_REV)); + } +} diff --git a/pkgs/atlas-search/crates/atlas-search/src/main.rs b/pkgs/atlas-search/crates/atlas-search/src/main.rs index 7494dc8ad..b37deb3c1 100644 --- a/pkgs/atlas-search/crates/atlas-search/src/main.rs +++ b/pkgs/atlas-search/crates/atlas-search/src/main.rs @@ -16,6 +16,7 @@ //! backend/transport failures (Python tracebacks, also exit 1 — //! the binary prints the message instead of the traceback). +mod fetch_model; mod pyjson; mod render; mod server; @@ -62,6 +63,8 @@ enum Cmd { Serve(ServeArgs), /// generate the service config + v1 client table (0600) Init(InitArgs), + /// download + verify the pinned Snowflake model into the state dir (step 5) + FetchModel, } #[derive(Args)] @@ -301,10 +304,6 @@ fn pick_backend( // ------------------------------------------------------------ embedder -- -/// The pinned model revision (fixtures/README.md; step 5 fetches -/// it into the state dir — design section 4, Q1 as ruled). -const MODEL_REV: &str = "95c2741480856aa9666782eb4afe11959938017f"; - fn resolve_model_dir() -> Option { if let Ok(d) = std::env::var("ATLAS_MODEL_DIR") && !d.is_empty() @@ -313,12 +312,16 @@ fn resolve_model_dir() -> Option { return if p.is_dir() { Some(p) } else { None }; } let home = home_dir(); - let state = home.join(format!(".local/state/atlas-search/model/{MODEL_REV}")); + let state = home.join(format!( + ".local/state/atlas-search/model/{}", + fetch_model::MODEL_REV + )); if state.is_dir() { return Some(state); } let hf = home.join(format!( - ".cache/huggingface/hub/models--Snowflake--snowflake-arctic-embed-m-v2.0/snapshots/{MODEL_REV}" + ".cache/huggingface/hub/models--Snowflake--snowflake-arctic-embed-m-v2.0/snapshots/{}", + fetch_model::MODEL_REV )); if hf.is_dir() { return Some(hf); @@ -337,6 +340,18 @@ fn resolve_ort_lib() -> Option { if cand.is_file() { return Some(cand); } + // The packaged binary's closure path, baked in at build time + // (step 5, #5355): pkgs/atlas-search/default.nix compiles with + // ATLAS_SEARCH_ORT_LIB_DEFAULT set to its onnxruntime store + // path, so the packaged build resolves the runtime from its + // own closure — no env, no state-dir staging. Dev builds do + // not set it and never reach this branch. + if let Some(p) = option_env!("ATLAS_SEARCH_ORT_LIB_DEFAULT") { + let pb = PathBuf::from(p); + if pb.is_file() { + return Some(pb); + } + } None } @@ -684,6 +699,10 @@ fn cmd_serve(args: ServeArgs) -> Result<(), CliError> { server::serve_forever(settings, engine).map_err(|m| (1, m)) } +fn cmd_fetch_model() -> Result<(), CliError> { + fetch_model::fetch_model().map_err(|m| (2, m)) +} + fn cmd_init(args: InitArgs) -> Result<(), CliError> { let path = server::resolve_service_config_path(args.config.as_deref()); let summary = server::init_service_config(&path, args.force).map_err(|m| (2, m))?; @@ -706,6 +725,7 @@ fn run(cli: Cli) -> Result<(), CliError> { Cmd::GetChunk(args) => cmd_get_chunk(args), Cmd::Serve(args) => cmd_serve(args), Cmd::Init(args) => cmd_init(args), + Cmd::FetchModel => cmd_fetch_model(), } } diff --git a/pkgs/atlas-search/fixtures/make_fixture_db.py b/pkgs/atlas-search/fixtures/make_fixture_db.py new file mode 100644 index 000000000..1095864c8 --- /dev/null +++ b/pkgs/atlas-search/fixtures/make_fixture_db.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Build the atlas fixture DB (fixture_corpus.sql + the toy vectors +of fixture_spec.json) as a SQLite file. + +The Rust parity tests build the same DB in-process with rusqlite +(crates/atlas-core/tests/parity.rs); this stdlib-only twin exists +for the nix derivation's installCheck smoke +(pkgs/atlas-search/default.nix) and for manual packaged-binary +verification: F32 little-endian blobs keyed by chunk id, exactly as +the Rust builder inserts them. + +Usage: make_fixture_db.py [OUT.db] (default: ./fixture.db) +Also writes OUT.qvec.json (the fixture query vector, for +ATLAS_SEARCH_QVEC_FILE) next to the DB. +""" +import json +import sqlite3 +import struct +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def main() -> int: + out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("fixture.db") + sql = (HERE / "fixture_corpus.sql").read_text() + spec = json.loads((HERE / "fixture_spec.json").read_text()) + if out.exists(): + out.unlink() + con = sqlite3.connect(out) + con.executescript(sql) + for cid, vec in spec["vectors"].items(): + blob = struct.pack("<%df" % len(vec), *vec) + con.execute( + "INSERT INTO vec_snowflake_m_v2_768_f32 (chunk_id, vec) VALUES (?, ?)", + (int(cid), blob), + ) + con.commit() + con.close() + qvec = Path(str(out) + ".qvec.json") + qvec.write_text(json.dumps(spec["qvec_raw"])) + print(f"fixture db: {out}") + print(f"query vector: {qvec}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pkgs/atlas-search/fixtures/smoke_check.py b/pkgs/atlas-search/fixtures/smoke_check.py new file mode 100644 index 000000000..678a196cd --- /dev/null +++ b/pkgs/atlas-search/fixtures/smoke_check.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Packaged-binary smoke check (step 5, code#5355). + +Run by pkgs/atlas-search/default.nix's installCheckPhase against +the BUILT binary, and runnable by hand against any atlas-search +binary: + + python3 fixtures/smoke_check.py /path/to/atlas-search + +Three probes, all offline, in a scrubbed environment (temp HOME, +no ATLAS_* inherited): + +1. FTS-only search over the fixture DB (fixtures/make_fixture_db.py) + — stdout must equal the recorded Python envelope for the + `fts_default` case in fixtures/core-expectations.json. +2. Embedded (hybrid) search over the same DB via the sanctioned + ATLAS_SEARCH_QVEC_FILE fixture hook — the same monkeypatch the + Python fixture generator uses — byte-compared with the recorded + `hybrid_default` envelope. This is the model-free embedded + query: the ONNX session itself is probe 3's subject and the + packaged-artifact verification's (real model, production data). +3. The ONNX Runtime native binding loads from wherever the binary + resolves it (the packaged build: its nix closure, baked in at + build time): a vector search with an EMPTY model dir must fail + at the tokenizer — which atlas-embed loads only AFTER + ort::init_from succeeds — and must not report the runtime as + missing. If the dylib could not be found or loaded, the error + would name the runtime instead. + +Exits 0 with PASS lines, 1 with the first failure's detail. +""" +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +QUERY = "how do we back up the database" + + +def fail(msg: str) -> int: + print(f"smoke_check: FAIL: {msg}", file=sys.stderr) + return 1 + + +def run(binary: Path, args: list, env_extra: dict, home: Path) -> subprocess.CompletedProcess: + env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home)} + env.update(env_extra) + return subprocess.run( + [str(binary), *args], env=env, capture_output=True, text=True, timeout=120 + ) + + +def main() -> int: + if len(sys.argv) != 2: + return fail("usage: smoke_check.py /path/to/atlas-search") + binary = Path(sys.argv[1]) + if not binary.is_file(): + return fail(f"binary not found: {binary}") + + expectations = json.loads((HERE / "core-expectations.json").read_text()) + cases = {c["name"]: c for c in expectations["search_cases"]} + + with tempfile.TemporaryDirectory(prefix="atlas-smoke-") as td: + tmp = Path(td) + home = tmp / "home" + home.mkdir() + db = tmp / "fixture.db" + r = subprocess.run( + [sys.executable, str(HERE / "make_fixture_db.py"), str(db)], + capture_output=True, + text=True, + ) + if r.returncode != 0: + return fail(f"make_fixture_db failed: {r.stderr.strip()}") + qvec = Path(str(db) + ".qvec.json") + + tails = {"fts_default": ["--mode", "fts"], "hybrid_default": []} + for name, tail in tails.items(): + r = run( + binary, + ["search", QUERY, "--backend", "local", "--db", str(db), *tail], + {"ATLAS_SEARCH_QVEC_FILE": str(qvec)}, + home, + ) + if r.returncode != 0: + return fail(f"{name}: exit {r.returncode}, stderr: {r.stderr.strip()}") + try: + got = json.loads(r.stdout) + except json.JSONDecodeError as e: + return fail(f"{name}: stdout not JSON: {e}") + if got != cases[name]["envelope"]: + return fail(f"{name}: envelope differs from the recorded Python envelope") + print(f"smoke_check: PASS {name} (envelope byte-equal to the Python recording)") + + empty_model = tmp / "empty-model" + empty_model.mkdir() + r = run( + binary, + ["search", QUERY, "--backend", "local", "--db", str(db), "--mode", "vector"], + {"ATLAS_MODEL_DIR": str(empty_model)}, + home, + ) + if r.returncode == 0: + return fail("ort probe: vector search unexpectedly succeeded with an empty model dir") + if "ONNX Runtime library not found" in r.stderr: + return fail( + "ort probe: binary could not resolve the ONNX Runtime library: " + + r.stderr.strip() + ) + if "tokenizer:" not in r.stderr: + return fail( + "ort probe: expected failure at the tokenizer (after runtime init), got: " + + r.stderr.strip() + ) + print("smoke_check: PASS ort load (runtime initialized; failure landed on the tokenizer)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e9ba51f257137808ea0bde4ea95b0a7552040ddc Mon Sep 17 00:00:00 2001 From: "developing-today[bot]" <5454422+developing-today[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:11:13 -0600 Subject: [PATCH 2/3] pkgs/atlas-search: nix package + standalone flake (step 5, #5355) buildRustPackage on the repo-pinned rust-overlay toolchain (rust-toolchain.toml via the sub-flake and via the home wiring), cargoLock from the workspace lockfile. The closure carries onnxruntime (env ATLAS_SEARCH_ORT_LIB_DEFAULT points at its libonnxruntime.so; libstdc++/libz ride the stdenv closure and the ORT lib itself), so the packaged binary needs no host LD_LIBRARY_PATH. doInstallCheck runs fixtures/smoke_check.py against the built binary. flake.nix is a thin standalone wrapper (nixpkgs + rust-overlay + flake-utils) so the package also builds outside the fleet flake. --- pkgs/atlas-search/default.nix | 57 +++++++++++++++++++++++ pkgs/atlas-search/flake.lock | 85 +++++++++++++++++++++++++++++++++++ pkgs/atlas-search/flake.nix | 57 +++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 pkgs/atlas-search/default.nix create mode 100644 pkgs/atlas-search/flake.lock create mode 100644 pkgs/atlas-search/flake.nix diff --git a/pkgs/atlas-search/default.nix b/pkgs/atlas-search/default.nix new file mode 100644 index 000000000..092eaa78e --- /dev/null +++ b/pkgs/atlas-search/default.nix @@ -0,0 +1,57 @@ +# atlas-search — the Atlas access layer: hybrid FTS+vector search +# CLI + multi-client service (developing-today/code#5350, step 5: +# #5355). Build it through the package flake: +# +# nix build path:./pkgs/atlas-search +# +# or via home/common/default.nix, which callPackages this file +# with the same pinned toolchain (the pkgs/id pattern). +# +# ONNX Runtime is dynamically loaded (the ort crate's +# load-dynamic feature) and is NEVER downloaded at build or run +# time (design section 4, the loader lesson). The packaged binary +# resolves it from its own closure: ATLAS_SEARCH_ORT_LIB_DEFAULT +# is compiled in below, pointing at the onnxruntime store path — +# main.rs's resolve_ort_lib falls back to it after the env and +# state-dir candidates. Because the path is a store reference, +# onnxruntime — and with it libstdc++.so.6 and libz.so.1 — is +# part of this package's runtime closure (verify: +# `nix-store -qR `), and the installCheck proves the load +# works with no ATLAS_ORT_LIB set (fixtures/smoke_check.py). +{ + lib, + rustPlatform, + onnxruntime, + python3, +}: + +rustPlatform.buildRustPackage { + pname = "atlas-search"; + version = "0.1.0"; + src = ./.; + + cargoLock.lockFile = ./Cargo.lock; + + env.ATLAS_SEARCH_ORT_LIB_DEFAULT = "${onnxruntime}/lib/libonnxruntime.so"; + + nativeInstallCheckInputs = [ python3 ]; + doInstallCheck = true; + # The smoke: fixture FTS-only + embedded queries byte-compared + # with the recorded Python envelopes, plus the closure-ONNX + # load probe (an empty model dir must fail at the tokenizer, + # after runtime init — never at runtime resolution). + installCheckPhase = '' + runHook preInstallCheck + python3 ${./fixtures}/smoke_check.py $out/bin/atlas-search + runHook postInstallCheck + ''; + + meta = { + description = "Atlas corpus search: hybrid FTS+vector CLI and multi-client service (Snowflake arctic-embed-m-v2.0)"; + license = with lib.licenses; [ + mit + asl20 + ]; + mainProgram = "atlas-search"; + }; +} diff --git a/pkgs/atlas-search/flake.lock b/pkgs/atlas-search/flake.lock new file mode 100644 index 000000000..9b3792744 --- /dev/null +++ b/pkgs/atlas-search/flake.lock @@ -0,0 +1,85 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": [ + "systems" + ] + }, + "locked": { + "lastModified": 1791914995, + "narHash": "sha256-Zf/xfQEAasnwnmEfHqW/4hjDE/s44yn2hD4RuVIZRjA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b7a79d3fea1e214f6d668d521cecec0c01e26544", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1793540411, + "narHash": "sha256-fiPJ88izTVucjeo2EQ3jk4aWmLg03I1D4vxnX2QjEgE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "0bb2eedc20e4739f29e8045034f2828ad7054f8f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", + "systems": "systems" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1793782363, + "narHash": "sha256-5Vod7vKm10hOilQvEhTwXbhBE+V0ZqXH0jBQJhkX3KQ=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "7fcafdf6f0761462d68012aba6a627be3f008d68", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1791244623, + "narHash": "sha256-uuanbkFk8a313Zfrxdcz8Us/yQtaxNnKJtX8v0ZPpd8=", + "owner": "nix-systems", + "repo": "default", + "rev": "d36eeed142f88b5a636f2e3856a53c61654cf29b", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/pkgs/atlas-search/flake.nix b/pkgs/atlas-search/flake.nix new file mode 100644 index 000000000..e00ed08fd --- /dev/null +++ b/pkgs/atlas-search/flake.nix @@ -0,0 +1,57 @@ +{ + description = "atlas-search — Atlas corpus search CLI + service (developing-today/code#5350)"; + + inputs = { + # Track nixos-unstable, same as pkgs/id's sub-flake. + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + systems = { + url = "github:nix-systems/default"; + }; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-utils = { + url = "github:numtide/flake-utils"; + inputs.systems.follows = "systems"; + }; + }; + + outputs = + { + self, + nixpkgs, + rust-overlay, + flake-utils, + ... + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + # The pinned toolchain (rust-toolchain.toml, channel + # 1.99.0) — the same rustup toolchain the step 1-4 gates + # ran under on the laptop, so the packaged build compiles + # with the compiler the parity evidence belongs to. + rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + rustPlatform = pkgs.makeRustPlatform { + cargo = rustToolchain; + rustc = rustToolchain; + }; + atlas-search = pkgs.callPackage ./default.nix { inherit rustPlatform; }; + in + { + packages = { + inherit atlas-search; + default = atlas-search; + }; + checks.default = atlas-search; + devShells.default = pkgs.mkShell { + packages = [ rustToolchain ]; + }; + } + ); +} From 462ed22675813b860edfe8b2423ecac6ea29dfe2 Mon Sep 17 00:00:00 2001 From: "developing-today[bot]" <5454422+developing-today[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:11:16 -0600 Subject: [PATCH 3/3] home: atlas-search serve unit, landed disabled (step 5, #5355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit systemd.user.services.atlas-search runs the packaged binary (serve, default 127.0.0.1:18760 — inventoried free against the 18742-18752 proxy range and live listeners). Landed DISABLED: no Install section, so the unit cannot be enabled until step 6 (#5356) adds WantedBy, runs init for the real client tokens, and creates the tailscale serve mapping (recorded in the unit comment; serve state is imperative tailscaled state with no declarative precedent in the repo). The package is deliberately not in home.packages: the Python wrapper at ~/.local/bin/atlas-search remains the resolved atlas-search CLI until the step-6 cutover. --- home/common/default.nix | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/home/common/default.nix b/home/common/default.nix index 541724800..cde7a2b8c 100644 --- a/home/common/default.nix +++ b/home/common/default.nix @@ -21,6 +21,32 @@ let opencodeRouteProxyPort = 18750; claudeCodeProxyPort = 18749; antigravityDirectProxyPort = 18751; + # Atlas search service port (code#5350 step 5, #5355): the fleet + # inventory entry for 18760. Unlike the proxy lets above, this + # value is NOT plumbed into a unit's Environment — it lives in + # the service's own config (~/.local/state/atlas-search/ + # service.json, written by `atlas-search init`) and in the Rust + # defaults (server listen, CLI service URL), all 18760. The let + # exists so the reservation is recorded where the fleet looks + # for one; verified free against this file, the proxy-port + # guard (.github/scripts/check-proxy-port-fallbacks.sh), and + # the live listener table on 2026-11-04. + atlasSearchPort = 18760; + # The atlas-search package (pkgs/atlas-search), built with the + # toolchain pinned by its own rust-toolchain.toml (channel + # 1.99.0) through the fleet's rust-overlay lock — the pkgs/id + # pattern (cf. rustToolchain in nixos/environment/default.nix). + # Used ONLY as the atlas-search unit's ExecStart below: the + # package is deliberately not in home.packages, so the Python + # CLI stays the resolved `atlas-search` until the step-6 + # cutover (#5356) says otherwise. + atlasSearchToolchain = (pkgs.extend (import inputs.id-rust-overlay)).rust-bin.fromRustupToolchainFile ../../pkgs/atlas-search/rust-toolchain.toml; + atlasSearch = pkgs.callPackage ../../pkgs/atlas-search { + rustPlatform = pkgs.makeRustPlatform { + cargo = atlasSearchToolchain; + rustc = atlasSearchToolchain; + }; + }; # Upstream prebuilt release binaries for tools where nixpkgs trails upstream. # See pkgs/latest-cli/default.nix for the rationale, the tradeoffs, and the # list of things deliberately NOT bumped. @@ -846,6 +872,40 @@ in Install.WantedBy = [ "default.target" ]; }; + # Atlas search service (code#5350 step 5, #5355): the packaged + # atlas-search binary's serve mode — one resident Snowflake + # embedder shared by every client, the job protocol with its + # in-memory temp store + dedupe cache, and per-client ceiling + # caps enforced from ~/.local/state/atlas-search/service.json + # (0600, written by `atlas-search init`). Binds 127.0.0.1:18760 + # (atlasSearchPort above); the tailnet exposure is the OpenCode + # pattern — a `tailscale serve` mapping on the MagicDNS name in + # front of the loopback port — and is created at activation + # (step 6), not here: a serve mapping in front of a service + # that has not started would be a live exposure change, and + # tailscale serve state is imperative tailscaled state in this + # fleet (no declarative precedent to land it in). + # + # LANDED DISABLED — no Install section, so home-manager + # creates no wants symlink and the unit cannot auto-start. + # Step 6 (#5356, the cutover) adds + # Install.WantedBy = [ "default.target" ], issues the real + # client tokens via `atlas-search init`, and creates the + # tailscale serve mapping, in one change. + systemd.user.services.atlas-search = { + Unit = { + Description = "Atlas search service (hybrid corpus search, serve mode)"; + After = [ "network-online.target" ]; + Wants = [ "network-online.target" ]; + }; + Service = { + ExecStart = "${atlasSearch}/bin/atlas-search serve"; + Restart = "on-failure"; + RestartSec = 3; + }; + # No Install: landed disabled (step 5) — see the note above. + }; + systemd.user.services.sync-models = { Unit = { Description = "Sync AI models for OpenCode providers from models.dev and agy";