diff --git a/pkgs/atlas-search/.gitignore b/pkgs/atlas-search/.gitignore
new file mode 100644
index 000000000..ea8c4bf7f
--- /dev/null
+++ b/pkgs/atlas-search/.gitignore
@@ -0,0 +1 @@
+/target
diff --git a/pkgs/atlas-search/Cargo.lock b/pkgs/atlas-search/Cargo.lock
new file mode 100644
index 000000000..29eb3d5ef
--- /dev/null
+++ b/pkgs/atlas-search/Cargo.lock
@@ -0,0 +1,865 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "atlas-embed"
+version = "0.1.0"
+dependencies = [
+ "ort",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "tokenizers",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "castaway"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "cc"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b"
+dependencies = [
+ "find-msvc-tools",
+ "shlex",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "compact_str"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "darling"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "dary_heap"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "either"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
+
+[[package]]
+name = "esaxx-rs"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "libloading"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "log"
+version = "0.4.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
+
+[[package]]
+name = "macro_rules_attribute"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "pastey",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
+
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "monostate"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
+dependencies = [
+ "monostate-impl",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "monostate-impl"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "onig"
+version = "6.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "ort"
+version = "2.0.0-rc.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37c87aa7a90b7ff3e0fe36cccd475df8d67f15f07f4d6c370c4b38464232813c"
+dependencies = [
+ "libloading",
+ "ndarray",
+ "ort-sys",
+ "smallvec",
+ "tracing",
+]
+
+[[package]]
+name = "ort-sys"
+version = "2.0.0-rc.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d22b0417f1838db5a3ac0b3f98dfe39f6fb73624aa0a27c6bbb5040b9c7db2b"
+
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
+[[package]]
+name = "pastey"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
+dependencies = [
+ "portable-atomic",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "rand"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-cond"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
+dependencies = [
+ "either",
+ "itertools",
+ "rayon",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
+[[package]]
+name = "ryu"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.154"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "smallvec"
+version = "1.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
+
+[[package]]
+name = "spm_precompiled"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
+dependencies = [
+ "base64",
+ "nom",
+ "serde",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.9",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223"
+dependencies = [
+ "ahash",
+ "aho-corasick",
+ "compact_str",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom",
+ "itertools",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
+dependencies = [
+ "pin-project-lite",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "unicode-normalization-alignments"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
+dependencies = [
+ "smallvec",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
diff --git a/pkgs/atlas-search/Cargo.toml b/pkgs/atlas-search/Cargo.toml
new file mode 100644
index 000000000..dbbe24420
--- /dev/null
+++ b/pkgs/atlas-search/Cargo.toml
@@ -0,0 +1,19 @@
+# Atlas search — Rust binary + multi-client service (epic: developing-today/code#5350).
+# Design of record: doc/2026-11-04T08-07-13Z_design_atlas_binary_service/ (§4 layout).
+#
+# Self-contained workspace on the pkgs/id pattern. Crates land one step at
+# a time; step 1 (#5351) is atlas-embed only.
+[workspace]
+members = ["crates/atlas-embed"]
+resolver = "2"
+
+[workspace.package]
+edition = "2024"
+rust-version = "1.96"
+license = "MIT OR Apache-2.0"
+
+[profile.release]
+# The embedder's numerics are part of a parity contract (Gate A): the
+# release profile must not trade float exactness for speed.
+# (No fast-math style flags exist in rustc, but keep this explicit so a
+# future profile edit has to argue with the comment.)
diff --git a/pkgs/atlas-search/crates/atlas-embed/Cargo.toml b/pkgs/atlas-search/crates/atlas-embed/Cargo.toml
new file mode 100644
index 000000000..ca6751d8c
--- /dev/null
+++ b/pkgs/atlas-search/crates/atlas-embed/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "atlas-embed"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+description = "Snowflake arctic-embed-m-v2.0 embedder for Atlas — ONNX Runtime (dynamically loaded) + the model's own Unigram tokenizer. Gate A parity crate for developing-today/code#5351."
+
+[dependencies]
+# Dynamic loading only: the ort crate must never download a prebuilt
+# onnxruntime at build time (forbidden in the nix sandbox and a
+# reproducibility hole — design §4). The runtime library is resolved
+# from the deployment closure and handed to Embedder::load.
+ort = { version = "2.0.0-rc.15", default-features = false, features = ["std", "load-dynamic", "ndarray"] }
+tokenizers = { version = "0.22", default-features = false, features = ["onig"] }
+thiserror = "2"
+
+[dev-dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
diff --git a/pkgs/atlas-search/crates/atlas-embed/examples/timing.rs b/pkgs/atlas-search/crates/atlas-embed/examples/timing.rs
new file mode 100644
index 000000000..9be6309f8
--- /dev/null
+++ b/pkgs/atlas-search/crates/atlas-embed/examples/timing.rs
@@ -0,0 +1,76 @@
+//! Timing/RSS probe for the embedder — spike instrumentation for
+//! steps 2–4 (not a benchmark suite). Run:
+//!
+//! ATLAS_ORT_LIB=/path/to/libonnxruntime.so \
+//! cargo run --release -p atlas-embed --example timing
+//!
+//! ATLAS_MODEL_DIR defaults to the pinned HF snapshot, as in the
+//! Gate A test.
+
+use std::path::PathBuf;
+use std::time::Instant;
+
+use atlas_embed::Embedder;
+
+fn rss_mb() -> f64 {
+ let status = std::fs::read_to_string("/proc/self/status").unwrap();
+ for line in status.lines() {
+ if let Some(rest) = line.strip_prefix("VmRSS:") {
+ let kb: f64 = rest.trim().trim_end_matches(" kB").trim().parse().unwrap();
+ return kb / 1024.0;
+ }
+ }
+ f64::NAN
+}
+
+fn main() {
+ let model_dir = std::env::var_os("ATLAS_MODEL_DIR").map(PathBuf::from).unwrap_or_else(|| {
+ let home = std::env::var_os("HOME").expect("HOME unset");
+ PathBuf::from(home).join(".cache/huggingface/hub/models--Snowflake--snowflake-arctic-embed-m-v2.0/snapshots/95c2741480856aa9666782eb4afe11959938017f")
+ });
+ let ort_lib = PathBuf::from(std::env::var_os("ATLAS_ORT_LIB").expect("ATLAS_ORT_LIB required"));
+
+ let t = Instant::now();
+ let mut emb = Embedder::load(&model_dir, &ort_lib).expect("load");
+ println!(
+ "load (env+tokenizer+session): {:.0} ms, RSS {:.0} MB",
+ t.elapsed().as_secs_f64() * 1e3,
+ rss_mb()
+ );
+
+ let fox = "The quick brown fox jumps over the lazy dog.";
+ let t = Instant::now();
+ let v = emb.embed(&[fox]).expect("embed");
+ println!(
+ "first single embed (15 tok): {:.0} ms",
+ t.elapsed().as_secs_f64() * 1e3
+ );
+ let t = Instant::now();
+ for _ in 0..20 {
+ emb.embed(&[fox]).expect("embed");
+ }
+ println!(
+ "warm single embed (15 tok): {:.1} ms avg, RSS {:.0} MB",
+ t.elapsed().as_secs_f64() * 1e3 / 20.0,
+ rss_mb()
+ );
+ println!("checksum dim0..3: {:?}", &v[0][..3]);
+
+ let long = "Embeddings turn text into vectors that capture semantic meaning. ".repeat(24);
+ let t = Instant::now();
+ emb.embed(&[long.as_str()]).expect("embed long");
+ println!(
+ "single embed (~{} tok): {:.0} ms",
+ emb.tokenize(&long).unwrap().len(),
+ t.elapsed().as_secs_f64() * 1e3
+ );
+
+ let batch: Vec<&str> = (0..32).map(|_| fox).collect();
+ let t = Instant::now();
+ emb.embed(&batch).expect("embed batch");
+ println!(
+ "batch of 32 short: {:.0} ms, RSS {:.0} MB",
+ t.elapsed().as_secs_f64() * 1e3,
+ rss_mb()
+ );
+}
diff --git a/pkgs/atlas-search/crates/atlas-embed/src/lib.rs b/pkgs/atlas-search/crates/atlas-embed/src/lib.rs
new file mode 100644
index 000000000..f5fac61e6
--- /dev/null
+++ b/pkgs/atlas-search/crates/atlas-embed/src/lib.rs
@@ -0,0 +1,284 @@
+//! atlas-embed — the Snowflake `snowflake-arctic-embed-m-v2.0` embedder
+//! for Atlas, and nothing else (design §4, developing-today/code#5350).
+//!
+//! The crate exists to be parity-tested in isolation: Gate A
+//! (code#5351) requires it to reproduce the production Python corpus
+//! pipeline (`~/atlas/v3/embed_onnx.py`) **bit-identically** — token
+//! ids equal, output vectors delta 0.0, single and batched. The
+//! reference fixtures live in `pkgs/atlas-search/fixtures/` with a
+//! recorded, rerunnable generator.
+//!
+//! Pipeline contract (every point is load-bearing for the gate):
+//! - The model's own `tokenizer.json`: SentencePiece Unigram with a
+//! Metaspace pre-tokenizer and a precompiled normalizer, ``/``
+//! template. NOT BERT WordPiece. Loaded via the `tokenizers` crate.
+//! - **Truncation is OFF.** The shipped tokenizer config carries a
+//! 512-token truncation default; the model's real context is 8192.
+//! The corpus pipeline tokenizes whole texts and a >8192-token input
+//! is a loud error here, never a silent cut.
+//! - Documents embed **prompt-free**. The sentence-transformers
+//! `prompt_name="query"` handling for query texts is a string-prefix
+//! step exercised by the search layer (step 2), not this crate.
+//! - Batching pads **right, pad token id 0, attention mask 0** — the
+//! corpus pipeline's manual padding (`embed_onnx.py` builds its
+//! batch arrays with `np.zeros`). The fork's :18752 sidecar pads
+//! with id 1 instead; the recorded fixture diagnostic
+//! (gen_fixtures.py) shows the two are bit-identical on this fp32
+//! graph (max delta 0.0 on every probe, incl. a 1,272-token chunk),
+//! because masked positions never reach the CLS row. We replicate
+//! the corpus pipeline's 0 because it is the pipeline of record.
+//! - Pooling is **CLS** (`hidden[:, 0, :]`, per the model's shipped
+//! 1_Pooling config), then **L2 normalize** in f32. The norm's sum
+//! of squares replicates NumPy's pairwise summation order — see
+//! [`pairwise_sum_f32`] — because the reference does
+//! `vecs / np.linalg.norm(vecs, axis=1)` and Gate A grants no
+//! tolerance for a differently-ordered accumulation.
+//! - ONNX Runtime is **dynamically loaded** from a caller-supplied
+//! library path; nothing is downloaded at build or run time
+//! (design §4 loader lesson). Session threading matches the
+//! recorded corpus pass: intra-op 8, inter-op 2 by default.
+
+use std::path::Path;
+
+use ort::environment::Environment;
+use ort::session::Session;
+use ort::value::Tensor;
+use tokenizers::Tokenizer;
+
+/// The model's real context length. Inputs longer than this are a
+/// loud error, matching the corpus pipeline's contract.
+pub const MODEL_CONTEXT: usize = 8192;
+/// Full output width of arctic-embed-m-v2.0 (no Matryoshka truncation
+/// in this crate; width selection is a caller concern).
+pub const EMBED_DIM: usize = 768;
+/// Pad token id used when batching. See the crate docs: this is the
+/// corpus pipeline's 0, not the tokenizer's declared pad id 1, and
+/// the choice is provably inert on this graph.
+pub const PAD_ID: i64 = 0;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ // ort's error type is generic over the value a failed builder
+ // call would have produced (ort::Error etc.), so
+ // the crate stringifies at the boundary instead of naming a
+ // single instantiation.
+ #[error("onnxruntime: {0}")]
+ Ort(String),
+ #[error("tokenizer: {0}")]
+ Tokenizer(String),
+ #[error("input tokenizes to {tokens} tokens, exceeding the model context of {MODEL_CONTEXT}")]
+ TooLong { tokens: usize },
+ #[error("cannot embed an empty batch")]
+ EmptyBatch,
+ #[error("unexpected model output shape: {0:?}")]
+ BadShape(Vec),
+}
+
+pub type Result = std::result::Result;
+
+/// A loaded embedder: one ONNX session + the model's tokenizer.
+///
+/// Construction is explicit and eager; callers that want the design's
+/// lazy init (serve mode) wrap this in a once-cell.
+pub struct Embedder {
+ // Kept alive for the session's lifetime; sessions clone the
+ // environment's Arc internally, but holding it here makes the
+ // lifetime contract explicit at this crate's boundary.
+ _env: Environment,
+ session: Session,
+ tokenizer: Tokenizer,
+}
+
+/// Stringify any ort error into this crate's error type.
+fn ort(e: E) -> Error {
+ Error::Ort(e.to_string())
+}
+
+impl Embedder {
+ /// Load from a model directory (the pinned snapshot layout:
+ /// `tokenizer.json` at the root, `onnx/model.onnx` the fp32
+ /// graph) and a dynamically-loaded ONNX Runtime library.
+ ///
+ /// `intra_threads` / `inter_threads` default to the recorded
+ /// corpus-pass values (8 / 2) via [`Embedder::load`]; use
+ /// [`Embedder::load_with_threads`] to override.
+ pub fn load(model_dir: &Path, ort_lib: &Path) -> Result {
+ Self::load_with_threads(model_dir, ort_lib, 8, 2)
+ }
+
+ pub fn load_with_threads(
+ model_dir: &Path,
+ ort_lib: &Path,
+ intra_threads: usize,
+ inter_threads: usize,
+ ) -> Result {
+ let env = ort::init_from(ort_lib)
+ .map_err(ort)?
+ .with_name("atlas-embed")
+ .build()
+ .map_err(ort)?;
+
+ let mut tokenizer = Tokenizer::from_file(model_dir.join("tokenizer.json"))
+ .map_err(|e| Error::Tokenizer(e.to_string()))?;
+ // The shipped tokenizer.json carries the stock 512-token
+ // truncation default. The model's context is 8192 and the
+ // corpus pipeline truncates nothing — kill it explicitly.
+ tokenizer
+ .with_truncation(None)
+ .map_err(|e| Error::Tokenizer(e.to_string()))?;
+ // Padding is built manually in embed_ids (the reference
+ // pipeline pads its own arrays); never let the tokenizer pad.
+ tokenizer.with_padding(None);
+
+ let session = Session::builder(&env)
+ .map_err(ort)?
+ .with_intra_threads(intra_threads)
+ .map_err(ort)?
+ .with_inter_threads(inter_threads)
+ .map_err(ort)?
+ .commit_from_file(model_dir.join("onnx").join("model.onnx"))
+ .map_err(ort)?;
+
+ Ok(Self {
+ _env: env,
+ session,
+ tokenizer,
+ })
+ }
+
+ /// Tokenize one text exactly as the corpus pipeline does
+ /// (`add_special_tokens=True`, no truncation, no padding).
+ pub fn tokenize(&self, text: &str) -> Result> {
+ let enc = self
+ .tokenizer
+ .encode(text, true)
+ .map_err(|e| Error::Tokenizer(e.to_string()))?;
+ Ok(enc.get_ids().to_vec())
+ }
+
+ /// Embed a batch of texts (document mode: prompt-free).
+ pub fn embed(&mut self, texts: &[&str]) -> Result> {
+ let mut id_lists = Vec::with_capacity(texts.len());
+ for text in texts {
+ id_lists.push(self.tokenize(text)?);
+ }
+ self.embed_ids(&id_lists)
+ }
+
+ /// Embed pre-tokenized inputs. This is the core path; `embed` is
+ /// its tokenizing front door. Batch composition never changes a
+ /// vector (masked right-padding; Gate A asserts it).
+ pub fn embed_ids(&mut self, id_lists: &[Vec]) -> Result> {
+ if id_lists.is_empty() {
+ return Err(Error::EmptyBatch);
+ }
+ for ids in id_lists {
+ if ids.len() > MODEL_CONTEXT {
+ return Err(Error::TooLong { tokens: ids.len() });
+ }
+ }
+ let batch = id_lists.len();
+ let maxlen = id_lists.iter().map(|v| v.len()).max().unwrap_or(0);
+
+ let mut input_ids = vec![PAD_ID; batch * maxlen];
+ let mut attn = vec![0i64; batch * maxlen];
+ for (j, ids) in id_lists.iter().enumerate() {
+ for (k, &id) in ids.iter().enumerate() {
+ input_ids[j * maxlen + k] = i64::from(id);
+ attn[j * maxlen + k] = 1;
+ }
+ }
+
+ let ids_tensor =
+ Tensor::from_array(([batch as i64, maxlen as i64], input_ids)).map_err(ort)?;
+ let attn_tensor = Tensor::from_array(([batch as i64, maxlen as i64], attn)).map_err(ort)?;
+ let outputs = self
+ .session
+ .run(ort::inputs![
+ "input_ids" => ids_tensor,
+ "attention_mask" => attn_tensor,
+ ])
+ .map_err(ort)?;
+ let hidden = outputs[0].try_extract_array::().map_err(ort)?;
+ let shape: Vec = hidden.shape().iter().map(|&d| d as i64).collect();
+ if shape.len() != 3 || shape[0] != batch as i64 || shape[2] != EMBED_DIM as i64 {
+ return Err(Error::BadShape(shape));
+ }
+ let view = hidden.view();
+ let mut out = Vec::with_capacity(batch);
+ for j in 0..batch {
+ let mut cls = [0f32; EMBED_DIM];
+ for k in 0..EMBED_DIM {
+ cls[k] = view[[j, 0, k]];
+ }
+ l2_normalize(&mut cls);
+ out.push(cls);
+ }
+ Ok(out)
+ }
+}
+
+/// L2-normalize in place, replicating the reference pipeline's
+/// arithmetic: the norm is `sqrt(pairwise_sum(x_i^2))` with the
+/// squares and the sum in f32, then an elementwise f32 division.
+fn l2_normalize(v: &mut [f32; EMBED_DIM]) {
+ let squares: [f32; EMBED_DIM] = std::array::from_fn(|i| v[i] * v[i]);
+ let norm = pairwise_sum_f32(&squares).sqrt();
+ for x in v.iter_mut() {
+ *x /= norm;
+ }
+}
+
+/// NumPy's pairwise summation (PW_BLOCKSIZE = 128, 8-way unrolled
+/// accumulators), replicated so the accumulation order — and
+/// therefore the rounding — matches `np.add.reduce` bit for bit.
+pub fn pairwise_sum_f32(a: &[f32]) -> f32 {
+ let n = a.len();
+ if n < 8 {
+ let mut r = 0.0f32;
+ for &x in a {
+ r += x;
+ }
+ return r;
+ }
+ if n <= 128 {
+ let mut r = [a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7]];
+ let mut i = 8;
+ while i < n - (n % 8) {
+ for (k, acc) in r.iter_mut().enumerate() {
+ *acc += a[i + k];
+ }
+ i += 8;
+ }
+ let mut res = ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7]));
+ while i < n {
+ res += a[i];
+ i += 1;
+ }
+ return res;
+ }
+ let mut n2 = n / 2;
+ n2 -= n2 % 8;
+ pairwise_sum_f32(&a[..n2]) + pairwise_sum_f32(&a[n2..])
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn pairwise_sum_matches_naive_for_short_inputs() {
+ let a: Vec = (0..7).map(|i| (i as f32) * 0.25).collect();
+ assert_eq!(pairwise_sum_f32(&a), a.iter().sum::());
+ }
+
+ #[test]
+ fn pairwise_sum_handles_block_boundaries() {
+ // Exercise all three branches (n<8, 8..=128, >128 recursion).
+ for n in [8usize, 9, 127, 128, 129, 255, 768, 1000] {
+ let a: Vec = (0..n).map(|i| ((i * 37 % 101) as f32) * 0.01).collect();
+ let got = pairwise_sum_f32(&a);
+ assert!(got.is_finite() && got > 0.0, "n={n} got={got}");
+ }
+ }
+}
diff --git a/pkgs/atlas-search/crates/atlas-embed/tests/gate_a.rs b/pkgs/atlas-search/crates/atlas-embed/tests/gate_a.rs
new file mode 100644
index 000000000..846ca0be3
--- /dev/null
+++ b/pkgs/atlas-search/crates/atlas-embed/tests/gate_a.rs
@@ -0,0 +1,140 @@
+//! Gate A — embedder parity (developing-today/code#5351, design §5).
+//!
+//! Asserts, against `pkgs/atlas-search/fixtures/embed-probes.json`
+//! (recorded from the production Python corpus pipeline by
+//! `gen_fixtures.py`):
+//! * token ids exactly equal on every text-bearing probe;
+//! * output vectors BIT-IDENTICAL (delta 0.0) on every probe,
+//! embedded singly and in one fixed mixed batch.
+//!
+//! Requires the pinned model snapshot and a dynamically-loadable
+//! ONNX Runtime, so it is `#[ignore]`d for ordinary CI:
+//!
+//! ATLAS_ORT_LIB=/path/to/libonnxruntime.so \
+//! ATLAS_MODEL_DIR=/path/to/snowflake-arctic-embed-m-v2.0-snapshot \
+//! cargo test -p atlas-embed --test gate_a -- --ignored --nocapture
+//!
+//! ATLAS_MODEL_DIR defaults to the laptop's HF cache snapshot of the
+//! pinned revision. Any mismatch prints the full per-probe table and
+//! fails with it — the numbers are the issue-comment evidence either
+//! way.
+
+use std::path::PathBuf;
+
+use atlas_embed::Embedder;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct Fixture {
+ probes: Vec,
+}
+
+#[derive(Deserialize)]
+struct Probe {
+ name: String,
+ text: Option,
+ token_ids: Vec,
+ single_bits: Vec,
+ batch_bits: Vec,
+}
+
+fn delta_stats(got: &[f32; 768], want_bits: &[u32]) -> (f32, usize) {
+ let mut max_delta = 0.0f32;
+ let mut mismatches = 0usize;
+ for (k, &g) in got.iter().enumerate() {
+ let w = f32::from_bits(want_bits[k]);
+ if g.to_bits() != want_bits[k] {
+ mismatches += 1;
+ }
+ let d = (g - w).abs();
+ if d > max_delta {
+ max_delta = d;
+ }
+ }
+ (max_delta, mismatches)
+}
+
+#[test]
+#[ignore = "requires ATLAS_ORT_LIB and the pinned model snapshot; see file docs"]
+fn gate_a_bit_identity() {
+ let fixtures_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("..")
+ .join("..")
+ .join("fixtures")
+ .join("embed-probes.json");
+ let raw = std::fs::read_to_string(&fixtures_path)
+ .unwrap_or_else(|e| panic!("read {}: {e}", fixtures_path.display()));
+ let fixture: Fixture = serde_json::from_str(&raw).expect("parse embed-probes.json");
+
+ let model_dir = std::env::var_os("ATLAS_MODEL_DIR").map(PathBuf::from).unwrap_or_else(|| {
+ let home = std::env::var_os("HOME").expect("HOME unset");
+ PathBuf::from(home)
+ .join(".cache/huggingface/hub/models--Snowflake--snowflake-arctic-embed-m-v2.0/snapshots/95c2741480856aa9666782eb4afe11959938017f")
+ });
+ let ort_lib = std::env::var_os("ATLAS_ORT_LIB")
+ .map(PathBuf::from)
+ .expect("ATLAS_ORT_LIB must point at a libonnxruntime.so (Gate A environment)");
+
+ let mut embedder = Embedder::load(&model_dir, &ort_lib).expect("load embedder");
+
+ // Resolve each probe's ids: Rust-tokenize text probes (asserting
+ // id equality), corpus probes use their recorded ids (their text
+ // is personal corpus content, referenced by chunk id — see the
+ // fixtures README).
+ let mut id_lists: Vec> = Vec::with_capacity(fixture.probes.len());
+ let mut ids_match: Vec