From 555a972d7db010c2110f65e8b69aee3e61782e1a Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Mon, 5 Oct 2026 12:18:01 -0500 Subject: [PATCH] Remove model release preparation and the unused catalogue pin (#89) --- components.json | 10 - docs/MODEL_PACKAGE_HANDOFF.md | 241 -------- elastos/CHANGELOG.md | 11 + .../src/api/capsule_inventory.rs | 34 +- elastos/crates/elastos-server/src/carrier.rs | 25 +- elastos/crates/elastos-server/src/publish.rs | 504 +-------------- .../elastos-server/src/release_publication.rs | 326 ---------- elastos/crates/elastos-server/src/setup.rs | 89 +-- model-catalog.json | 1 - scripts/check-product-data.mjs | 47 -- scripts/check-product-data.test.mjs | 29 +- scripts/fetch/fetch-model-smoke.py | 535 ---------------- scripts/fetch/fetch-model.sh | 574 ------------------ scripts/local-carrier-setup-smoke.sh | 4 - scripts/model-package-handoff-test.py | 496 --------------- scripts/model-package-handoff.py | 555 ----------------- scripts/prepare-release-platform-test.py | 114 +--- scripts/prepare-release-platform.sh | 79 +-- scripts/publish-platform-artifacts-test.py | 39 -- scripts/publish-release.sh | 71 --- scripts/release-platform-input-test.py | 303 +-------- scripts/release-platform-input.py | 361 +---------- scripts/release-signer-test.py | 136 ----- scripts/release-signer.py | 158 +---- scripts/release-upstream-assets-test.py | 550 ----------------- scripts/release-upstream-assets.py | 427 +------------ scripts/setup-source-home.sh | 29 - 27 files changed, 94 insertions(+), 5654 deletions(-) delete mode 100644 docs/MODEL_PACKAGE_HANDOFF.md delete mode 100644 model-catalog.json delete mode 100755 scripts/fetch/fetch-model-smoke.py delete mode 100755 scripts/fetch/fetch-model.sh delete mode 100755 scripts/model-package-handoff-test.py delete mode 100755 scripts/model-package-handoff.py delete mode 100644 scripts/release-upstream-assets-test.py diff --git a/components.json b/components.json index fb8f3321c..804e2c052 100644 --- a/components.json +++ b/components.json @@ -1759,15 +1759,5 @@ "llama-server" ] } - }, - "model_catalog": { - "head_cid": "bafkreigidvbvotnjdljhqmao3jmtgdh56h6wveso6lv7mnsyvmhcmhjpfq", - "publisher_dids": [ - "did:key:z6Mkjg9duxEF2nskEPR9F38eSfrY6F1GyWUq5aDbgsMqgcjA" - ], - "local_use": { - "max_cache_bytes": 21474836480, - "max_model_memory_bytes": 8589934592 - } } } diff --git a/docs/MODEL_PACKAGE_HANDOFF.md b/docs/MODEL_PACKAGE_HANDOFF.md deleted file mode 100644 index b55333e85..000000000 --- a/docs/MODEL_PACKAGE_HANDOFF.md +++ /dev/null @@ -1,241 +0,0 @@ -# Model package handoff - -This document tells a sender operator and a receiver how to move one signed -model content package from the sender's Runtime-managed Kubo into the -receiver's Runtime-managed Kubo, and how the receiver pins the sender's signed -permanent catalog snapshot so their Runtime can run normal local Use. The -helper is `scripts/model-package-handoff.py` and needs only Python 3. Catalog -and package rules live in -[CONTENT_CAPSULE_DISTRIBUTION.md](CONTENT_CAPSULE_DISTRIBUTION.md). - -## Purpose and boundaries - -The package travels as a CAR file. A CAR is a portable copy of the exact block -closure under the package root CID, so the receiver's Kubo ends up with the -same blocks and the same root the sender holds. CAR transfer is package -portability between two operators. Cold Content or Carrier Get for models -remains a separate, unaccepted flow; this handoff does not stand in for it. - -Two pins exist and stay separate. The Content retention pin is a Kubo recursive -pin of the package root CID inside the receiver's Kubo; it keeps the bytes. The -catalog trust pin is `components.json.model_catalog.head_cid`, the raw CIDv1 -SHA-256 of the exact bytes of the signed `model-catalog.json` snapshot; it tells -the Runtime which publisher statement to trust. The helper writes each pin with -its own subcommand, removing one leaves the other untouched, and neither pin -implies the other. - -The Runtime verifies the catalog signature when it reads the snapshot. The -helper checks the head CID, the schema, the entry, the signer DID and the -permanent-snapshot rule. It does not verify the signature, and its receipt -says `"signature_verification": "pending_runtime_read"` until the Runtime has -read and verified the snapshot. A permanent snapshot omits `expires_at`. -Runtime readers older than the permanent-snapshot change require that field -and reject a permanent snapshot, so the receiver's Runtime includes that -change before the receiver pins one. - -Local Use also needs the platform llama.cpp engine. The Runtime derives -dispatch readiness from the current signed entry, the admission, the verified -`llama-server` engine receipt and one matching offer from the local model -provider. Current source has startup profiles for `darwin-arm64` and -`linux-amd64`; the latter uses a bounded CPU path. Linux readiness still needs -a compatible installed engine receipt and target proof. Other hosts stay not -ready until they have a verified profile. - -The helper streams every transfer in 1 MiB chunks and hashes as it goes, so a -6 GB package stays on disk and travels as raw bytes end to end. - -## Sender steps - -The sender holds the package in their Runtime's Kubo. The example package root -is `bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi` with a payload -of 6169366387 bytes; the CAR is a little larger than the payload because it -carries block headers and the DAG-PB directory nodes. - -1. Build the permanent catalog payload. It carries the schema, `published_at`, - and one to eight entries. Each entry `cid` is a package root, its - `capsule_manifest` is that package's `capsule.json` object, and its - `object_manifest` is its `_elastos_object.json` object. Package CIDs and - capsule names stay unique. Leave `expires_at` out. Serialize it with sorted - keys and no whitespace, which is what the Runtime signs over. The example - below shows one entry: - - ```bash - PAYLOAD=$(jq -cS -n \ - --arg cid bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi \ - --argjson capsule "$(cat package/capsule.json)" \ - --argjson object "$(cat package/_elastos_object.json)" \ - '{schema: "elastos.model.catalog/v1", - published_at: (now | floor), - entries: [{cid: $cid, capsule_manifest: $capsule, - object_manifest: $object}]}') - ``` - -2. Sign the payload with the publisher key and assemble the envelope. The - Runtime accepts the envelope keys `payload`, `signature` and `signer_did` - and nothing else: - - ```bash - SIGNED=$(printf '%s' "$PAYLOAD" | \ - elastos sign-payload --domain elastos.model.catalog.v1 --key "$KEY") - jq -cS -n --argjson payload "$PAYLOAD" \ - --arg signature "$(jq -r .signature <<<"$SIGNED")" \ - --arg signer_did "$(jq -r .signer_did <<<"$SIGNED")" \ - '{payload: $payload, signature: $signature, signer_did: $signer_did}' \ - > model-catalog.json - ``` - -3. Compute the head CID of the exact file bytes. The receiver pins this value, - so record it as `HEAD_CID` and hand it over as its own item: - - ```bash - python3 scripts/model-package-handoff.py head-cid \ - --catalog model-catalog.json - ``` - -4. Export the CAR from the Runtime's Kubo. The helper reads the Kubo API port - from `ipfs-coords.json` in the Runtime data dir, checks that Kubo is 0.40.1 - with the canonical import profile, streams the closure to - `qwen-package.car.partial`, renames it on success and writes the receipt: - - ```bash - python3 scripts/model-package-handoff.py export \ - --data-dir "${XDG_DATA_HOME:-$HOME/.local/share}/elastos" \ - --cid bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi \ - --output qwen-package.car - ``` - - The receipt `qwen-package.car.receipt.json` records the package CID, the - CAR SHA-256, the CAR size, the Kubo version and the export time. - -5. Hand over five things: `qwen-package.car`, `qwen-package.car.receipt.json`, - `model-catalog.json`, the exact `HEAD_CID` from step 3, and the publisher - DID printed by `sign-payload`. Send the head CID and the publisher DID in - text the receiver can compare against, not only inside the files. Any - durable file transport works, for example a USB drive, rsync or an object - store. The CAR is about 6.2 GB, so plan for a resumable transport; this path - moves the file as raw bytes and has no base64 or whole-file upload API. - -## Receiver steps - -The receiver runs these on the Home that will keep the package, with the -Runtime's Kubo running. `DATA_DIR` is the Runtime data dir, by default -`~/.local/share/elastos` on Linux and `~/Library/Application Support/elastos` -on macOS. - -1. Confirm the Runtime's Kubo is the pinned version with the canonical import - profile, and install the platform llama.cpp engine if the Runtime does not - already hold its verified receipt: - - ```bash - python3 scripts/model-package-handoff.py verify-kubo --data-dir "$DATA_DIR" - elastos setup --with llama-server - ``` - -2. Check the CAR against its receipt before Kubo sees it. This recomputes the - size and SHA-256 by streaming the file and refuses a truncated or altered - CAR or a receipt for a different package: - - ```bash - python3 scripts/model-package-handoff.py verify-car \ - --car qwen-package.car --receipt qwen-package.car.receipt.json \ - --cid bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi - ``` - -3. Import and pin. The command repeats the CAR check, verifies Kubo, reads the - Kubo repo path and refuses unless the free space left after the import stays - at or above the floor you name. The floor is your budget and the command - requires it. Kubo pins the root recursively during the import and the helper - confirms the pin afterwards. Kubo answers only after the whole CAR has - arrived and the root is pinned, so give the 6 GB import a timeout that - covers that wait: - - ```bash - python3 scripts/model-package-handoff.py import \ - --data-dir "$DATA_DIR" \ - --car qwen-package.car --receipt qwen-package.car.receipt.json \ - --cid bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi \ - --free-space-floor-bytes 21474836480 \ - --timeout-seconds 1800 - ``` - - The receipt reports `"pinned": true` with the block count and block bytes - Kubo stored. Running the same command again succeeds and leaves the pin in - place. - -4. Pin the catalog. First compare the handed-over `HEAD_CID` with the head of - the bytes you received; `pin-catalog` refuses a mismatch as well: - - ```bash - python3 scripts/model-package-handoff.py head-cid --catalog model-catalog.json - ``` - - The helper then checks the envelope, refuses a timed catalog, installs - `model-catalog.json` into `DATA_DIR` at mode 0600 and replaces only the - `model_catalog` key of `components.json`, leaving every other key as it - was. The local Use budgets are yours to set and the Runtime rejects zero. - Two floors apply to this package. The Qwen manifest declares - `minimum_memory_mb` 8192, so `--max-model-memory-bytes` is at least - 8589934592. The Runtime charges the cache three times the payload plus - 8 MiB and 192 KiB of index allowance, which is 18516684377 bytes for the - 6169366387-byte payload, so `--max-cache-bytes` is at least that; the - example uses 20 GiB: - - ```bash - python3 scripts/model-package-handoff.py pin-catalog \ - --data-dir "$DATA_DIR" \ - --catalog model-catalog.json \ - --head-cid "$HEAD_CID" \ - --publisher-did "$PUBLISHER_DID" \ - --package-cid bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi \ - --max-cache-bytes 21474836480 \ - --max-model-memory-bytes 8589934592 - ``` - - `DATA_DIR` has to be owned by you and closed to group and other writes, - which is how the Runtime reads it as well. - -5. Restart the Runtime with the platform restart helper from - [HOME_LOCAL_SETUP.md](HOME_LOCAL_SETUP.md), then check readiness. - `elastos content status --cid ` reports the package, and the - System view in Home shows the model catalog as verified with the sender as - publisher. Run local Use from Home as usual. That first successful Runtime - read is the signature verification the `pin-catalog` receipt left pending. - -If this Home already admitted the same package under a previous catalog head, -the admitted bytes and the original receipt stay in place, and readiness -follows the current head. Until you act, the Runtime reports the package as -admitted and not ready, and startup composition offers nothing for it. One -explicit same-CID Use under the new pin creates an alias record that reuses -the admitted bytes, so that Use completes without a second download and the -package becomes ready again. - -## Failure handling - -A truncated or corrupt CAR fails at `verify-car`, and `import` runs that same -check first, so Kubo receives no request for a bad file. If a transfer was -interrupted, copy the CAR again and rerun the same commands; every step -converges to the same end state, and a repeated `import` or `pin-catalog` is -safe. - -If `import` refuses on the free-space floor, free space in the Kubo repo's -filesystem or choose a smaller floor and rerun. If Kubo reports an import error -mid-stream, the helper prints Kubo's message and the root stays unpinned. -Blocks that Kubo already wrote before the failure stay in its blockstore as -unpinned garbage until its next collection; a rerun with a complete CAR -deduplicates them, pins the root, and converges to the same end state. On the -synthetic closure a truncated import committed nothing; the helper does not -rely on that for larger packages. - -The two pins come off independently. `kubo pin rm ` (with -`IPFS_PATH` set to the Runtime's Kubo repo) releases the retention pin and -leaves `components.json` alone. Removing the `model_catalog` key from -`components.json` and restarting the Runtime withdraws the catalog trust pin -and leaves the Kubo pin alone. - -## Verification scope - -`scripts/model-package-handoff-test.py` checks block-closure transfer, catalogue -trust and failure handling with isolated Kubo instances and test signing keys. -This operator utility does not replace installed Marketplace acquisition tests. -Current target verification and remaining work belong in [state.md](../state.md) -and [TASKS.md](../TASKS.md). diff --git a/elastos/CHANGELOG.md b/elastos/CHANGELOG.md index 8a8d1091b..6273133d7 100644 --- a/elastos/CHANGELOG.md +++ b/elastos/CHANGELOG.md @@ -4,8 +4,19 @@ All notable changes to the public ElastOS Runtime repository. ## [Unreleased] +### Removed + +- Releases no longer pin or carry a model catalogue; with no pin, Home shows no + published models. Removed the model CAR retention and import path from release + preparation, signing and publication, `model-package-handoff.py`, + `fetch-model.sh` and their tests. Models return later as a separate signed + publication. + ### Fixed +- Setup reads a pinned model catalogue over Carrier with its 128 KiB bound and + refuses a larger declared size before reading the body. + - Setup accepts passive content capsules from the signed release while retaining provider role and icon checks. It refuses URL-only downloads and blank release paths, including entries that also contain a CID. Explicit CID-only operator diff --git a/elastos/crates/elastos-server/src/api/capsule_inventory.rs b/elastos/crates/elastos-server/src/api/capsule_inventory.rs index d56db1422..64cd6d0a6 100644 --- a/elastos/crates/elastos-server/src/api/capsule_inventory.rs +++ b/elastos/crates/elastos-server/src/api/capsule_inventory.rs @@ -13,7 +13,7 @@ pub(in crate::api) mod preparation; const DEV_CAPSULES_ROOT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../capsules"); const MODEL_CATALOG_FILE: &str = "model-catalog.json"; const MODEL_CATALOG_DOMAIN: &str = "elastos.model.catalog.v1"; -const MAX_MODEL_CATALOG_BYTES: usize = 128 * 1024; +pub(crate) const MAX_MODEL_CATALOG_BYTES: usize = 128 * 1024; const MAX_MODEL_CATALOG_ENTRIES: usize = 8; pub(crate) struct VerifiedModelCatalogEntry { @@ -834,36 +834,4 @@ pub(crate) mod tests { BTreeSet::from(["object-provider".to_string()]) ); } - - #[test] - fn checkout_permanent_catalog_matches_components_pin() { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../"); - let components: crate::setup::ComponentsManifest = - serde_json::from_slice(&std::fs::read(root.join("components.json")).unwrap()).unwrap(); - let trust = components - .model_catalog - .expect("matching install pins a signed model catalog"); - let bytes = std::fs::read(root.join(MODEL_CATALOG_FILE)).unwrap(); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - let entries = verify_model_catalog(&trust, &bytes, now).unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!( - entries[0].cid, - "bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi" - ); - assert_eq!(entries[0].manifest.name, "qwen3-5-9b-q4-k-m-local"); - assert_eq!( - entries[1].cid, - "bafybeidy5kfvqwg6g6pfgdfwslmhijosbeskt5b2duqdqxnc7e6fwmr72y" - ); - assert_eq!(entries[1].manifest.name, "smollm2-135m-instruct-q8-0-local"); - assert_eq!( - entries[0].publisher_did, - "did:key:z6Mkjg9duxEF2nskEPR9F38eSfrY6F1GyWUq5aDbgsMqgcjA" - ); - assert_eq!(entries[1].publisher_did, entries[0].publisher_did); - } } diff --git a/elastos/crates/elastos-server/src/carrier.rs b/elastos/crates/elastos-server/src/carrier.rs index f5090df09..4e5d136f8 100644 --- a/elastos/crates/elastos-server/src/carrier.rs +++ b/elastos/crates/elastos-server/src/carrier.rs @@ -8275,14 +8275,22 @@ impl CarrierClient { } pub async fn fetch_file(&self, path: &str) -> Result> { + self.fetch_file_bounded(path, 200 * 1024 * 1024).await + } + + pub(crate) async fn fetch_file_bounded(&self, path: &str, max_bytes: usize) -> Result> { let (mut send, mut recv) = self.conn.open_bi().await?; let msg = serde_json::json!({"op":"file","path":path}); let mut bytes = serde_json::to_vec(&msg)?; bytes.push(b'\n'); send.write_all(&bytes).await?; send.finish()?; - read_carrier_len_prefixed_bytes(&mut recv, &format!("trusted source file fetch for {path}")) - .await + read_carrier_len_prefixed_bytes( + &mut recv, + &format!("trusted source file fetch for {path}"), + max_bytes, + ) + .await } pub async fn fetch_content(&self, cid: &str, path: Option<&str>) -> Result> { @@ -8298,7 +8306,7 @@ impl CarrierClient { bytes.push(b'\n'); send.write_all(&bytes).await?; send.finish()?; - read_carrier_len_prefixed_bytes(&mut recv, "content fetch").await + read_carrier_len_prefixed_bytes(&mut recv, "content fetch", 200 * 1024 * 1024).await } pub async fn invoke_provider( @@ -8427,11 +8435,12 @@ impl CarrierClient { async fn read_carrier_len_prefixed_bytes( recv: &mut iroh::endpoint::RecvStream, operation: &str, + max_bytes: usize, ) -> Result> { let mut len_buf = [0u8; 8]; recv.read_exact(&mut len_buf).await?; - let len = u64::from_be_bytes(len_buf) as usize; - if len > 200 * 1024 * 1024 { + let len = u64::from_be_bytes(len_buf); + if len_buf[0] == b'{' { let mut error_bytes = len_buf.to_vec(); let tail = recv.read_to_end(16 * 1024).await?; error_bytes.extend_from_slice(&tail); @@ -8447,7 +8456,11 @@ async fn read_carrier_len_prefixed_bytes( } anyhow::bail!("{operation} returned invalid byte reply ({len} bytes declared)"); } - let mut content = vec![0u8; len]; + anyhow::ensure!( + len <= max_bytes as u64, + "{operation} exceeds its {max_bytes}-byte bound ({len} bytes declared)" + ); + let mut content = vec![0u8; len as usize]; recv.read_exact(&mut content).await?; Ok(content) } diff --git a/elastos/crates/elastos-server/src/publish.rs b/elastos/crates/elastos-server/src/publish.rs index eef8bffb2..9e7b902da 100644 --- a/elastos/crates/elastos-server/src/publish.rs +++ b/elastos/crates/elastos-server/src/publish.rs @@ -448,16 +448,6 @@ async fn run_signed_publication( require_import_cid(&actual, &artifact.cid)?; } } - // Directory blocks, rather than just CAR file bytes, must be retained - // before this generation can become the advertised release head. - import_publication_models_with_session( - &mut preflight, - workspace_root, - &data_dir, - &snapshot.join("artifacts"), - frozen.model_retention(), - ) - .await?; let release_cid = preflight .import_file(&snapshot.join("release.json")) .await?; @@ -866,23 +856,6 @@ struct PublicationStorageBudget { repo_bytes: u64, } -fn model_storage_bytes( - records: &[elastos_server::release_publication::ModelRetention], -) -> anyhow::Result { - records.iter().try_fold(0_u64, |total, record| { - total - .checked_add( - record - .car - .size - .checked_mul(2) - .context("Model storage budget overflow")?, - ) - .and_then(|n| n.checked_add(1024 * 1024)) - .context("Model storage budget overflow") - }) -} - fn publication_storage_budget( publication: &elastos_server::release_publication::Publication, ) -> anyhow::Result { @@ -897,21 +870,13 @@ fn publication_storage_budget( .checked_add(artifact.size) .context("Publication storage budget overflow") })?; - storage_budget( - files, - publication.artifacts().len() + 3, - model_storage_bytes(publication.model_retention())?, - ) + storage_budget(files, publication.artifacts().len() + 3) } -fn storage_budget( - files: u64, - count: usize, - models: u64, -) -> anyhow::Result { +fn storage_budget(files: u64, count: usize) -> anyhow::Result { // Two full local copies (immutable snapshot and promotion stage), two - // payload budgets for UnixFS file blocks and filesystem overhead, then the - // retained model directory blocks. One MiB per file covers control files, + // payload budgets for UnixFS file blocks and filesystem overhead. One MiB + // per file covers control files, // allocation rounding and publication bookkeeping. Existing pins may make // the real write smaller; preflight reserves a cold repository. let overhead = u64::try_from(count)? @@ -926,7 +891,6 @@ fn storage_budget( .context("Publication storage budget overflow")?, repo_bytes: twice .checked_add(overhead) - .and_then(|n| n.checked_add(models)) .context("Publication storage budget overflow")?, }) } @@ -1037,201 +1001,6 @@ fn validate_publication_capacity( Ok(floor) } -fn model_import_space_guard( - repo: &Path, - records: &[elastos_server::release_publication::ModelRetention], -) -> anyhow::Result { - use std::os::fd::AsRawFd; - use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; - let directory = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) - .open(repo)?; - let metadata = directory.metadata()?; - anyhow::ensure!( - metadata.is_dir() - && metadata.uid() == unsafe { libc::geteuid() } - && metadata.mode() & 0o022 == 0, - "Model repository must be owned and protected" - ); - let mut observation = std::mem::MaybeUninit::::uninit(); - anyhow::ensure!( - unsafe { libc::fstatvfs(directory.as_raw_fd(), observation.as_mut_ptr()) } == 0, - "Model repository capacity observation failed" - ); - let observation = unsafe { observation.assume_init() }; - let capacity = u128::from(observation.f_blocks) * u128::from(observation.f_frsize); - let free = u128::from(observation.f_bavail) * u128::from(observation.f_frsize); - let floor = capacity.saturating_mul(15).div_ceil(100); - let car_bytes: u128 = records - .iter() - .map(|record| u128::from(record.car.size)) - .sum(); - // Reserve all CAR payload bytes plus an equal filesystem-overhead budget. - // Observe real free space again after each import as well. - let required = car_bytes * 2 + records.len() as u128 * 1024 * 1024; - anyhow::ensure!( - capacity > 0 && free <= capacity && free >= floor + required, - "Model directory import would cross the 15 percent free-space floor" - ); - u64::try_from(floor).context("Model repository floor exceeds its integer bound") -} - -async fn bounded_model_import( - script: &Path, - api: &elastos_server::local_http::LoopbackHttpBaseUrl, - artifacts: &Path, - record: &elastos_server::release_publication::ModelRetention, - floor: u64, -) -> anyhow::Result<()> { - use tokio::io::{AsyncRead, AsyncReadExt}; - async fn read_bounded(reader: impl AsyncRead + Unpin) -> anyhow::Result> { - let mut bytes = Vec::new(); - reader.take(64 * 1024 + 1).read_to_end(&mut bytes).await?; - anyhow::ensure!( - bytes.len() <= 64 * 1024, - "Model import process output exceeded its bound" - ); - Ok(bytes) - } - let mut child = tokio::process::Command::new("python3") - .arg(script) - .arg("import") - .arg("--kubo-api") - .arg(api.as_str()) - .arg("--cid") - .arg(&record.package_cid) - .arg("--car") - .arg(artifacts.join(&record.car.name)) - .arg("--receipt") - .arg(artifacts.join(&record.receipt.name)) - .arg("--free-space-floor-bytes") - .arg(floor.to_string()) - .arg("--timeout-seconds") - .arg("60") - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("Cannot run the model package import helper")?; - let stdout = child.stdout.take().context("Model import stdout missing")?; - let stderr = child.stderr.take().context("Model import stderr missing")?; - let result = tokio::time::timeout(std::time::Duration::from_secs(300), async { - let (stdout, _stderr, status) = - tokio::try_join!(read_bounded(stdout), read_bounded(stderr), async { - Ok::<_, anyhow::Error>(child.wait().await?) - })?; - anyhow::ensure!( - status.success(), - "Model CAR import failed before publication" - ); - let value: serde_json::Value = - serde_json::from_slice(&stdout).context("Model import receipt is invalid")?; - let fields = value - .as_object() - .context("Model import receipt must be an object")?; - anyhow::ensure!( - fields.len() == 6 - && fields.keys().all(|key| matches!( - key.as_str(), - "schema" | "package_cid" | "pinned" | "blocks" | "block_bytes" | "kubo_version" - )), - "Model import receipt fields refused" - ); - anyhow::ensure!( - value["schema"] == "elastos.model.package-import/v1" - && value["package_cid"] == record.package_cid - && value["pinned"] == true - && value["kubo_version"] == "0.40.1" - && value["blocks"] - .as_u64() - .is_some_and(|count| count > 0 && count <= record.car.size) - && value["block_bytes"] - .as_u64() - .is_some_and(|size| size > 0 && size <= record.car.size), - "Model import receipt differs from the signed retention root" - ); - Ok(()) - }) - .await; - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(error)) => { - let _ = child.kill().await; - Err(error) - } - Err(_) => { - let _ = child.kill().await; - anyhow::bail!("Model CAR import exceeded five minutes before publication") - } - } -} - -#[cfg(test)] -async fn import_publication_models( - provider: &Path, - workspace: &Path, - data_dir: &Path, - artifacts: &Path, - records: &[elastos_server::release_publication::ModelRetention], -) -> anyhow::Result<()> { - let mut session = PublicationProvider::start(provider, data_dir).await?; - let result = import_publication_models_with_session( - &mut session, - workspace, - data_dir, - artifacts, - records, - ) - .await; - let finished = session.finish().await; - if finished.is_err() { - let _ = session.child.kill().await; - } - result?; - finished -} - -async fn import_publication_models_with_session( - session: &mut PublicationProvider, - workspace: &Path, - data_dir: &Path, - artifacts: &Path, - records: &[elastos_server::release_publication::ModelRetention], -) -> anyhow::Result<()> { - if records.is_empty() { - return Ok(()); - } - let script = workspace.join("scripts/model-package-handoff.py"); - anyhow::ensure!(script.is_file(), "Model package import helper is required"); - async { - session - .request(serde_json::json!({"op":"runtime_prepare_backend"})) - .await?; - let repo = data_dir.join("ipfs-repo"); - let required = model_storage_bytes(records)?; - let (floor, api) = session.check_capacity(data_dir, required, 0).await?; - model_import_space_guard(&repo, records)?; - for record in records { - session - .request(serde_json::json!({"op":"runtime_prepare_backend"})) - .await?; - // Use the held provider's admitted endpoint, rather than resolving - // the mutable coordinate file again in the external helper. - bounded_model_import(&script, &api, artifacts, record, floor).await?; - let (_, after) = session.check_capacity(data_dir, 1, 0).await?; - anyhow::ensure!( - after.as_str() == api.as_str(), - "Model provider API changed during CAR import" - ); - model_import_space_guard(&repo, &[])?; - } - Ok(()) - } - .await -} - fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") } @@ -2842,22 +2611,13 @@ mod tests { } #[cfg(unix)] - fn model_import_fixture( - root: &Path, - mode: &str, - ) -> ( - PathBuf, - PathBuf, - PathBuf, - Vec, - ) { + fn provider_session_fixture(root: &Path, mode: &str) -> (PathBuf, PathBuf, PathBuf) { use std::os::unix::fs::PermissionsExt; let data = root.join("data"); let artifacts = root.join("artifacts"); std::fs::create_dir_all(data.join("ipfs-repo")).unwrap(); std::fs::create_dir(&artifacts).unwrap(); - std::fs::create_dir(root.join("scripts")).unwrap(); - let provider = root.join("model-provider-fixture"); + let provider = root.join("publication-provider-fixture"); let provider_program = r#"#!/usr/bin/env python3 import base64, hashlib, http.server, json, os, pathlib, sys, threading cid = 'b' + base64.b32encode(b'\x01\x55\x12\x20' + hashlib.sha256(b'public model provider fixture').digest()).decode().lower().rstrip('=') @@ -2878,22 +2638,13 @@ endpoint = 'http://127.0.0.1:' + str(server.server_port) pathlib.Path(__file__ + '.api').write_text(endpoint + '/') worker = threading.Thread(target=server.serve_forever, daemon=True) worker.start() -prepares = 0 try: for line in sys.stdin: with open(__file__ + '.requests', 'a') as sink: sink.write(line) request = json.loads(line) data = {} - if request['op'] == 'runtime_prepare_backend': - prepares += 1 - if __MODE__ == 'replaced-coords' and prepares > 1: - (repo.parent / 'ipfs-coords.json').write_text(json.dumps({'api_port':1})) if request['op'] == 'add_path': data = {'cid':cid} - if request['op'] == 'status': - api = endpoint - if __MODE__ == 'changed-api' and (repo.parent.parent / 'scripts/model-package-handoff.requests').exists(): - api = endpoint.replace('127.0.0.1', 'localhost') - data = {'api_endpoint':api} + if request['op'] == 'status': data = {'api_endpoint':endpoint} if request['op'] == 'runtime_check_capacity': stats = os.statvfs(repo) data = {'volume_id':os.stat(repo).st_dev, 'capacity_bytes':stats.f_blocks * stats.f_frsize, @@ -2907,62 +2658,15 @@ finally: "#.replace("__MODE__", &serde_json::to_string(mode).unwrap()); std::fs::write(&provider, provider_program).unwrap(); std::fs::set_permissions(&provider, std::fs::Permissions::from_mode(0o700)).unwrap(); - let script = root.join("scripts/model-package-handoff.py"); - let program = format!( - r#"import json, pathlib, sys, urllib.request -args = sys.argv[1:] -with pathlib.Path(__file__).with_suffix('.requests').open('a') as sink: - sink.write(json.dumps(args) + '\n') -if {mode:?} == 'replaced-coords': - root = pathlib.Path(__file__).parent.parent - assert json.loads((root / 'data/ipfs-coords.json').read_text())['api_port'] == 1 - assert '--data-dir' not in args - api = args[args.index('--kubo-api') + 1] - opener = urllib.request.build_opener(urllib.request.ProxyHandler({{}})) - with opener.open(urllib.request.Request(api.rstrip('/') + '/api/v0/repo/stat', data=b''), timeout=2) as response: - assert json.load(response)['RepoPath'] == str(root / 'data/ipfs-repo') -if {mode:?} == 'failure': - sys.exit(7) -if {mode:?} == 'oversized': - print('x' * 65537) - sys.exit(0) -cid = args[args.index('--cid') + 1] -print(json.dumps({{'schema':'elastos.model.package-import/v1','package_cid':cid, - 'pinned':{pin},'blocks':1,'block_bytes':5,'kubo_version':'0.40.1'}})) -"#, - pin = if mode == "unpinned" { "False" } else { "True" } - ); - std::fs::write(script, program).unwrap(); - let records = (0..4) - .map(|index| { - let bytes = format!("public model fixture {index}"); - let cid = import_fixture_raw_cid(bytes.as_bytes()); - let raw = cid::Cid::try_from(cid.as_str()).unwrap(); - let package_cid = cid::Cid::new_v1(0x70, *raw.hash()).to_string(); - let descriptor = - |name: String| elastos_server::release_publication::ArtifactDescriptor { - name, - cid: cid.clone(), - sha256: "a".repeat(64), - size: 32, - }; - elastos_server::release_publication::ModelRetention { - component: format!("model-fixture-{index}"), - package_cid, - car: descriptor(format!("model-fixture-{index}.car")), - receipt: descriptor(format!("model-fixture-{index}.car.receipt.json")), - } - }) - .collect(); - (provider, data, artifacts, records) + (provider, data, artifacts) } #[cfg(unix)] #[tokio::test] - async fn publication_files_raw_blocks_and_model_imports_share_one_owned_provider() { + async fn publication_files_and_raw_blocks_share_one_owned_provider() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); - let (provider, data, artifacts, records) = model_import_fixture(&root, "success"); + let (provider, data, artifacts) = provider_session_fixture(&root, "success"); let file = artifacts.join("public-file.tar"); std::fs::write(&file, b"public fixture").unwrap(); let mut session = super::PublicationProvider::start(&provider, &data) @@ -2977,15 +2681,6 @@ print(json.dumps({{'schema':'elastos.model.package-import/v1','package_cid':cid, .unwrap(), expected ); - super::import_publication_models_with_session( - &mut session, - &root, - &data, - &artifacts, - &records, - ) - .await - .unwrap(); session.finish().await.unwrap(); let starts = std::fs::read_to_string(format!("{}.starts", provider.display())).unwrap(); assert_eq!(starts.lines().count(), 1); @@ -3005,177 +2700,28 @@ print(json.dumps({{'schema':'elastos.model.package-import/v1','package_cid':cid, 1 ); assert_eq!(requests.last().unwrap()["op"], "shutdown"); - assert_eq!( - std::fs::read_to_string(root.join("scripts/model-package-handoff.requests")) - .unwrap() - .lines() - .count(), - 4 - ); - } - - #[cfg(unix)] - #[tokio::test] - async fn model_retention_batch_keeps_provider_ready_and_passes_exact_import_inputs() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - let (provider, data, artifacts, records) = model_import_fixture(&root, "success"); - tokio::time::timeout( - std::time::Duration::from_secs(15), - super::import_publication_models(&provider, &root, &data, &artifacts, &records), - ) - .await - .unwrap() - .unwrap(); - let requests = import_fixture_requests(&provider); - assert_eq!( - requests.first().unwrap(), - &serde_json::json!({"op":"init","config":{}}) - ); - assert_eq!( - requests - .iter() - .filter(|request| request["op"] == "runtime_prepare_backend") - .count(), - 5 - ); - assert_eq!( - requests.last().unwrap(), - &serde_json::json!({"op":"shutdown"}) - ); - let arguments: Vec> = - std::fs::read_to_string(root.join("scripts/model-package-handoff.requests")) - .unwrap() - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect(); - assert_eq!(arguments.len(), 4); - let api = std::fs::read_to_string(format!("{}.api", provider.display())).unwrap(); - for (args, record) in arguments.iter().zip(&records) { - assert_eq!(args[0], "import"); - assert!(!args.iter().any(|arg| arg == "--data-dir")); - for (flag, expected) in [ - ("--kubo-api", api.clone()), - ("--cid", record.package_cid.clone()), - ( - "--car", - artifacts - .join(&record.car.name) - .to_string_lossy() - .into_owned(), - ), - ( - "--receipt", - artifacts - .join(&record.receipt.name) - .to_string_lossy() - .into_owned(), - ), - ("--timeout-seconds", "60".to_owned()), - ] { - assert_eq!( - &args[args.iter().position(|arg| arg == flag).unwrap() + 1], - &expected - ); - } - assert!( - args[args - .iter() - .position(|arg| arg == "--free-space-floor-bytes") - .unwrap() - + 1] - .parse::() - .unwrap() - > 0 - ); - } - } - - #[cfg(unix)] - #[tokio::test] - async fn model_import_keeps_admitted_provider_api_when_coordinates_are_replaced() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - let (provider, data, artifacts, records) = model_import_fixture(&root, "replaced-coords"); - tokio::time::timeout( - std::time::Duration::from_secs(15), - super::import_publication_models(&provider, &root, &data, &artifacts, &records), - ) - .await - .unwrap() - .unwrap(); - let imports = - std::fs::read_to_string(root.join("scripts/model-package-handoff.requests")).unwrap(); - assert_eq!(imports.lines().count(), 4); - let coordinates: serde_json::Value = - serde_json::from_slice(&std::fs::read(data.join("ipfs-coords.json")).unwrap()).unwrap(); - assert_eq!(coordinates["api_port"], 1); - assert_eq!( - import_fixture_requests(&provider).last().unwrap()["op"], - "shutdown" - ); } #[cfg(unix)] #[tokio::test] - async fn model_import_failure_refuses_the_batch_and_preserves_previous_public_files() { - for mode in ["failure", "unpinned", "oversized", "changed-api"] { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - let (provider, data, artifacts, records) = model_import_fixture(&root, mode); - let previous = data.join("previous-public-head.json"); - std::fs::write(&previous, b"accepted previous public generation").unwrap(); - let error = tokio::time::timeout( - std::time::Duration::from_secs(15), - super::import_publication_models(&provider, &root, &data, &artifacts, &records), - ) - .await - .unwrap() - .unwrap_err(); - assert!(error.to_string().contains("Model"), "{error}"); - assert_eq!( - std::fs::read(&previous).unwrap(), - b"accepted previous public generation" - ); - let requests = import_fixture_requests(&provider); - assert_eq!( - requests.last().unwrap(), - &serde_json::json!({"op":"shutdown"}) - ); - let imports = - std::fs::read_to_string(root.join("scripts/model-package-handoff.requests")) - .unwrap(); - assert_eq!( - imports.lines().count(), - 1, - "failed import must stop the remaining batch" - ); - } - } - - #[cfg(unix)] - #[tokio::test] - async fn model_import_refuses_an_api_repository_outside_selected_runtime_before_writes() { + async fn publication_capacity_refuses_an_api_repository_outside_selected_runtime() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); - let (provider, data, artifacts, records) = model_import_fixture(&root, "wrong-repo"); - let error = super::import_publication_models(&provider, &root, &data, &artifacts, &records) + let (provider, data, _artifacts) = provider_session_fixture(&root, "wrong-repo"); + let mut session = super::PublicationProvider::start(&provider, &data) .await - .unwrap_err(); + .unwrap(); + let error = session.check_capacity(&data, 1, 0).await.unwrap_err(); assert!(error.to_string().contains("repository differs"), "{error}"); - assert!(!root.join("scripts/model-package-handoff.requests").exists()); - assert_eq!( - import_fixture_requests(&provider).last().unwrap()["op"], - "shutdown" - ); + session.finish().await.unwrap(); } #[test] fn publication_budget_reserves_copies_file_blocks_and_directory_blocks_before_writes() { - let budget = super::storage_budget(100, 2, 60).unwrap(); + let budget = super::storage_budget(100, 2).unwrap(); assert_eq!(budget.local_bytes, 200 + 2 * 1024 * 1024); - assert_eq!(budget.repo_bytes, budget.local_bytes + 60); - assert!(super::storage_budget(u64::MAX, 1, 1).is_err()); + assert_eq!(budget.repo_bytes, budget.local_bytes); + assert!(super::storage_budget(u64::MAX, 1).is_err()); let repo = super::StorageObservation { volume: 7, capacity: 1000, @@ -3195,18 +2741,6 @@ print(json.dumps({{'schema':'elastos.model.package-import/v1','package_cid':cid, assert!(super::validate_publication_capacity(&wrong, 150, repo, other, 150, 150).is_err()); } - #[cfg(unix)] - #[test] - fn model_storage_preflight_reserves_all_records_before_import() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - let (_provider, data, _artifacts, mut records) = model_import_fixture(&root, "success"); - for record in &mut records { - record.car.size = u64::MAX; - } - assert!(super::model_import_space_guard(&data.join("ipfs-repo"), &records).is_err()); - } - #[cfg(unix)] fn import_fixture_raw_cid(bytes: &[u8]) -> String { use sha2::Digest; diff --git a/elastos/crates/elastos-server/src/release_publication.rs b/elastos/crates/elastos-server/src/release_publication.rs index 4c21a57ad..09efa52aa 100644 --- a/elastos/crates/elastos-server/src/release_publication.rs +++ b/elastos/crates/elastos-server/src/release_publication.rs @@ -31,14 +31,6 @@ pub struct ArtifactDescriptor { pub size: u64, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ModelRetention { - pub component: String, - pub package_cid: String, - pub car: ArtifactDescriptor, - pub receipt: ArtifactDescriptor, -} - struct Artifact { descriptor: ArtifactDescriptor, file: File, @@ -68,7 +60,6 @@ pub struct Publication { installer: Vec, descriptors: Vec, artifacts: BTreeMap, - model_retention: Vec, } impl Publication { @@ -192,8 +183,6 @@ impl Publication { } } let mut visited = BTreeSet::new(); - let mut model_retention = None; - let mut retention_present = None; loop { let next = artifacts .keys() @@ -215,13 +204,6 @@ impl Publication { continue; } component_refs(&component, &artifact_directory, &names, &mut artifacts)?; - let present = component.get("model_retention").is_some(); - if let Some(prior) = retention_present.replace(present) { - ensure!( - prior == present, - "platform model retention contracts differ" - ); - } if let Some(catalog) = component.get("model_catalog") { let cid = text(catalog, "head_cid")?; ensure!( @@ -241,14 +223,6 @@ impl Publication { Some("model-catalog.json"), )?; } - if present { - let records = admit_model_retention(&component, &artifacts)?; - if let Some(prior) = &model_retention { - ensure!(prior == &records, "platform model retention records differ"); - } else { - model_retention = Some(records); - } - } } ensure!( !artifacts.is_empty() && artifacts.len() <= MAX_FILES, @@ -279,7 +253,6 @@ impl Publication { installer, descriptors, artifacts, - model_retention: model_retention.unwrap_or_default(), }; publication.unchanged()?; if !published { @@ -312,9 +285,6 @@ impl Publication { pub fn artifacts(&self) -> &[ArtifactDescriptor] { &self.descriptors } - pub fn model_retention(&self) -> &[ModelRetention] { - &self.model_retention - } /// Cached publication bytes belong to this admitted immutable generation. /// This check observes file identities and metadata without reading content. @@ -497,145 +467,6 @@ impl Publication { } } -fn admit_model_retention( - component: &Value, - artifacts: &BTreeMap, -) -> Result> { - let trust: crate::setup::ModelCatalogConfig = serde_json::from_value( - component - .get("model_catalog") - .context("model retention requires catalogue trust")? - .clone(), - )?; - let catalog = artifacts - .get("model-catalog.json") - .context("model retention requires the admitted catalogue")?; - let bytes = read_artifact(catalog, Some(128 * 1024))?; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_secs(); - let entries = crate::api::capsule_inventory::verify_model_catalog(&trust, &bytes, now)?; - let records = component["model_retention"] - .as_object() - .context("model retention map required")?; - ensure!( - records.len() == 4 && entries.len() == 4, - "model retention requires four catalogue roots" - ); - let external = component["external"] - .as_object() - .context("model retention requires external components")?; - let mut result = Vec::new(); - let mut owned_files = BTreeSet::new(); - for entry in entries { - let name = &entry.manifest.name; - ensure!( - external.contains_key(name), - "retained model is absent from components" - ); - let value = records - .get(name) - .context("catalogue model lacks its retention record")?; - let fields = value - .as_object() - .context("model retention record required")?; - ensure!( - fields.len() == 3 - && fields - .keys() - .all(|key| matches!(key.as_str(), "package_cid" | "car" | "receipt")), - "model retention fields refused" - ); - let package_cid = text(value, "package_cid")?; - let cid = checked_cid(package_cid)?; - ensure!( - cid.version() == cid::Version::V1 && cid.codec() == 0x70 && package_cid == entry.cid, - "model retention root differs from the signed catalogue" - ); - let mut descriptors = Vec::new(); - for kind in ["car", "receipt"] { - let descriptor = &value[kind]; - let fields = descriptor - .as_object() - .context("model retention file descriptor required")?; - ensure!( - fields.len() == 4 - && fields.keys().all(|key| matches!( - key.as_str(), - "release_path" | "cid" | "checksum" | "size" - )), - "model retention descriptor fields refused" - ); - let path = text(descriptor, "release_path")?; - safe_name(path)?; - ensure!( - owned_files.insert(path.to_owned()), - "model retention file is shared by multiple records" - ); - let artifact = artifacts - .get(path) - .context("model retention file outside admitted artifacts")?; - ensure!( - text(descriptor, "cid")? == artifact.descriptor.cid - && text(descriptor, "checksum")? - == format!("sha256:{}", artifact.descriptor.sha256) - && descriptor["size"].as_u64() == Some(artifact.descriptor.size), - "model retention descriptor differs from admitted bytes" - ); - descriptors.push(artifact.descriptor.clone()); - } - let car = descriptors.remove(0); - let receipt = descriptors.remove(0); - ensure!( - car.name.ends_with(".car") && receipt.name == format!("{}.receipt.json", car.name), - "model retention file names refused" - ); - ensure!( - car.size - <= entry - .size_bytes - .saturating_mul(2) - .saturating_add(1024 * 1024), - "model CAR exceeds the closure storage bound" - ); - let bytes = read_artifact(&artifacts[&receipt.name], Some(64 * 1024))?; - let receipt_json: Value = serde_json::from_slice(&bytes)?; - let fields = receipt_json - .as_object() - .context("model CAR receipt required")?; - ensure!( - fields.len() == 6 - && fields.keys().all(|key| matches!( - key.as_str(), - "schema" - | "package_cid" - | "car_sha256" - | "car_bytes" - | "kubo_version" - | "exported_at" - )), - "model CAR receipt fields refused" - ); - ensure!( - text(&receipt_json, "schema")? == "elastos.model.package-car/v1" - && text(&receipt_json, "package_cid")? == package_cid - && text(&receipt_json, "car_sha256")? == car.sha256 - && receipt_json["car_bytes"].as_u64() == Some(car.size) - && text(&receipt_json, "kubo_version")? == "0.40.1" - && receipt_json["exported_at"].as_u64().is_some(), - "model CAR receipt differs from its signed retention descriptor" - ); - result.push(ModelRetention { - component: name.clone(), - package_cid: package_cid.to_owned(), - car, - receipt, - }); - } - result.sort_by(|left, right| left.component.cmp(&right.component)); - Ok(result) -} - fn text<'a>(value: &'a Value, name: &str) -> Result<&'a str> { value .get(name) @@ -1262,163 +1093,6 @@ mod tests { fn open(&self) -> Result { Publication::open_flat(&self.input, &self.did) } - - fn write_components(&mut self, components: &Value) { - let bytes = serde_json::to_vec(components).unwrap(); - std::fs::write(self.input.join("components-aarch64-darwin.json"), &bytes).unwrap(); - self.release["platforms"]["aarch64-darwin"]["components"] = descriptor(&bytes); - self.write_signed(); - } - - fn with_model_retention() -> Self { - let mut fixture = Self::new(); - let mut components: Value = serde_json::from_slice( - &std::fs::read(fixture.input.join("components-aarch64-darwin.json")).unwrap(), - ) - .unwrap(); - let original = - crate::api::capsule_inventory::tests::model_catalog_fixture()["entries"][0].clone(); - let mut entries = Vec::new(); - let mut retention = serde_json::Map::new(); - for index in 0..4 { - let name = format!("model-fixture-{index}"); - let mut entry = original.clone(); - entry["capsule_manifest"]["name"] = json!(name); - let capsule = serde_json::to_vec(&entry["capsule_manifest"]).unwrap(); - let files = entry["object_manifest"]["files"].as_array_mut().unwrap(); - let mut hash = Sha256::new(); - for file in files { - if file["path"] == "capsule.json" { - file["size"] = json!(capsule.len()); - file["sha256"] = json!(digest(&capsule)); - } - for value in [ - file["path"].as_str().unwrap().to_owned(), - file["sha256"].as_str().unwrap().to_owned(), - file["size"].to_string(), - ] { - hash.update(value.as_bytes()); - hash.update(b"\0"); - } - } - entry["object_manifest"]["content_digest"] = - json!(format!("sha256:{}", hex::encode(hash.finalize()))); - // Synthetic metadata root shape, not a Kubo/Carrier proof. - let hash = - cid::multihash::Multihash::<64>::wrap(0x12, &Sha256::digest(&capsule)).unwrap(); - let package = cid::Cid::new_v1(0x70, hash).to_string(); - entry["cid"] = json!(package); - let car_name = format!("{name}.car"); - let receipt_name = format!("{car_name}.receipt.json"); - let car = format!("public model CAR fixture {index}").into_bytes(); - let receipt = serde_json::to_vec(&json!({"schema":"elastos.model.package-car/v1", - "package_cid":package,"car_sha256":digest(&car),"car_bytes":car.len(), - "kubo_version":"0.40.1","exported_at":1})) - .unwrap(); - let file_ref = |name: &str, bytes: &[u8]| json!({"release_path":name,"cid":raw_cid(bytes),"checksum":format!("sha256:{}",digest(bytes)),"size":bytes.len()}); - std::fs::write(fixture.input.join(&car_name), &car).unwrap(); - std::fs::write(fixture.input.join(&receipt_name), &receipt).unwrap(); - retention.insert(name.clone(), json!({"package_cid":package,"car":file_ref(&car_name,&car),"receipt":file_ref(&receipt_name,&receipt)})); - components["external"][&name] = components["external"]["home"].clone(); - entries.push(entry); - } - let catalog = envelope( - &fixture.key, - "elastos.model.catalog.v1", - &json!({ - "schema":"elastos.model.catalog/v1","published_at":1,"expires_at":null,"entries":entries}), - ); - std::fs::write(fixture.input.join("model-catalog.json"), &catalog).unwrap(); - components["model_catalog"] = - json!({"head_cid":raw_cid(&catalog),"publisher_dids":[fixture.did]}); - components["model_retention"] = Value::Object(retention); - fixture.write_components(&components); - fixture - } - } - - #[test] - fn signed_model_retention_binds_four_roots_and_receipts_across_snapshot() { - let fixture = PublicPublicationFixture::with_model_retention(); - let publication = fixture.open().unwrap(); - assert_eq!(publication.model_retention().len(), 4); - let output = fixture.parent.join("retained-snapshot"); - publication.snapshot_into(&output).unwrap(); - let copied = Publication::open_published(&output, &fixture.did).unwrap(); - assert_eq!(copied.model_retention(), publication.model_retention()); - } - - #[test] - fn model_retention_refuses_wrong_root_descriptor_fields_and_missing_models() { - for mutation in ["root", "descriptor", "missing", "catalog-signature"] { - let mut fixture = PublicPublicationFixture::with_model_retention(); - let path = fixture.input.join("components-aarch64-darwin.json"); - let mut components: Value = - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); - match mutation { - "root" => { - components["model_retention"]["model-fixture-0"]["package_cid"] = - components["model_retention"]["model-fixture-1"]["package_cid"].clone() - } - "descriptor" => { - components["model_retention"]["model-fixture-0"]["car"]["unexpected"] = - json!(true) - } - "missing" => { - components["model_retention"] - .as_object_mut() - .unwrap() - .remove("model-fixture-0"); - } - _ => { - let path = fixture.input.join("model-catalog.json"); - let mut catalog: Value = - serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); - catalog["signature"] = json!("0".repeat(128)); - let bytes = serde_json::to_vec(&catalog).unwrap(); - std::fs::write(path, &bytes).unwrap(); - components["model_catalog"]["head_cid"] = json!(raw_cid(&bytes)); - } - } - fixture.write_components(&components); - assert!(fixture.open().is_err(), "{mutation}"); - } - } - - #[test] - fn model_car_receipt_requires_the_admitted_hash_size_and_profile() { - for field in [ - "car_sha256", - "car_bytes", - "package_cid", - "kubo_version", - "extra", - ] { - let mut fixture = PublicPublicationFixture::with_model_retention(); - let component_path = fixture.input.join("components-aarch64-darwin.json"); - let mut components: Value = - serde_json::from_slice(&std::fs::read(&component_path).unwrap()).unwrap(); - let receipt_path = fixture.input.join("model-fixture-0.car.receipt.json"); - let mut receipt: Value = - serde_json::from_slice(&std::fs::read(&receipt_path).unwrap()).unwrap(); - receipt[field] = match field { - "car_bytes" => json!(1), - "extra" => json!(true), - "car_sha256" => json!("0".repeat(64)), - "package_cid" => { - components["model_retention"]["model-fixture-1"]["package_cid"].clone() - } - _ => json!("0.39.0"), - }; - let bytes = serde_json::to_vec(&receipt).unwrap(); - std::fs::write(&receipt_path, &bytes).unwrap(); - let descriptor = &mut components["model_retention"]["model-fixture-0"]["receipt"]; - descriptor["cid"] = json!(raw_cid(&bytes)); - descriptor["checksum"] = json!(format!("sha256:{}", digest(&bytes))); - descriptor["size"] = json!(bytes.len()); - fixture.write_components(&components); - assert!(fixture.open().is_err(), "{field}"); - } } #[test] diff --git a/elastos/crates/elastos-server/src/setup.rs b/elastos/crates/elastos-server/src/setup.rs index 0003446aa..05c45ab85 100644 --- a/elastos/crates/elastos-server/src/setup.rs +++ b/elastos/crates/elastos-server/src/setup.rs @@ -4,6 +4,7 @@ //! Assistant acquires its signed engine on demand; model directories use the //! bounded preparation path, not this archive installer. +use crate::api::capsule_inventory::MAX_MODEL_CATALOG_BYTES; use serde::{Deserialize, Serialize}; use sha2::Digest; use std::collections::HashMap; @@ -648,7 +649,7 @@ async fn admit_installed_setup_metadata( let catalog = if let Some(trust) = &manifest.model_catalog { let bytes = tokio::time::timeout( std::time::Duration::from_secs(30), - client.fetch_file(MODEL_CATALOG_FILE), + client.fetch_file_bounded(MODEL_CATALOG_FILE, MAX_MODEL_CATALOG_BYTES), ) .await .map_err(|_| anyhow::anyhow!("model catalogue Carrier fetch timed out"))??; @@ -2452,7 +2453,6 @@ pub fn write_installed_manifest( } const MODEL_CATALOG_FILE: &str = "model-catalog.json"; -const MAX_MODEL_CATALOG_BYTES: usize = 128 * 1024; pub(crate) fn catalog_head_cid(bytes: &[u8]) -> anyhow::Result { let hash = cid::multihash::Multihash::<64>::wrap(0x12, &sha2::Sha256::digest(bytes)) @@ -4255,12 +4255,6 @@ pub(crate) mod tests { let on_demand = resolve_components(&manifest, None, &["llama-server".to_string()], &[]).unwrap(); assert_eq!(on_demand, ["llama-server"]); - let catalog = manifest - .model_catalog - .as_ref() - .expect("home matching install pins a signed model catalog"); - assert!(!catalog.publisher_dids.is_empty()); - assert!(catalog.local_use.is_some()); for profile_name in ["home", "demo", "agent-local-ai", "public-gateway", "full"] { let profile = manifest .profiles @@ -5272,60 +5266,6 @@ pub(crate) mod tests { fs::set_permissions(&bundle, fs::Permissions::from_mode(0o700)).unwrap(); } - #[cfg(target_os = "macos")] - #[test] - fn runtime_setup_llama_bundle_is_reused_by_fetch() { - use std::os::unix::fs::PermissionsExt; - - let tmp = tempfile::tempdir().unwrap(); - let (manifest, data, bundle) = llama_bundle_fixture(tmp.path()); - let component = &manifest.external["llama-server"]; - write_cache_metadata( - &manifest, - resolve_platform_info(component, "darwin-arm64"), - "darwin-arm64", - "llama-server", - &bundle, - ) - .unwrap(); - let manifest_path = tmp.path().join("components.json"); - fs::write(&manifest_path, serde_json::to_vec(&manifest).unwrap()).unwrap(); - let tools = tmp.path().join("tools"); - fs::create_dir(&tools).unwrap(); - let uname = tools.join("uname"); - fs::write( - &uname, - b"#!/bin/sh\n[ \"$1\" = \"-s\" ] && echo Darwin || echo arm64\n", - ) - .unwrap(); - fs::set_permissions(&uname, fs::Permissions::from_mode(0o700)).unwrap(); - let output = Command::new("bash") - .arg( - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../../scripts/fetch/fetch-model.sh"), - ) - .arg("stable") - .env( - "PATH", - format!( - "{}:{}", - tools.display(), - std::env::var("PATH").unwrap_or_default() - ), - ) - .env("ELASTOS_COMPONENTS_MANIFEST", manifest_path) - .env("ELASTOS_DATA_DIR", &data) - .output() - .unwrap(); - fs::set_permissions(&bundle, fs::Permissions::from_mode(0o700)).unwrap(); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - assert!(!bundle.join(CACHED_ARTIFACT_SHA_FILE).exists()); - } - #[test] fn test_component_install_state_detects_stale_extracted_bundle_metadata() { let tmp = tempfile::tempdir().unwrap(); @@ -6506,6 +6446,13 @@ pub(crate) mod tests { } } + #[tokio::test] + async fn installed_setup_refuses_oversized_catalog_before_body_without_writes() { + for existing in [false, true] { + signed_setup_carrier_fixture("catalog size", existing).await; + } + } + #[tokio::test] async fn installed_setup_refuses_wrong_signers_without_writes() { for case in ["catalog signer", "head signer", "release signer"] { @@ -6915,6 +6862,7 @@ pub(crate) mod tests { let observed = requests.clone(); let endpoint = server.clone(); let writer_parent = data.clone(); + let oversized_catalog = case == "catalog size"; let serving = tokio::spawn(async move { while let Some(incoming) = endpoint.accept().await { let Ok(connection) = incoming.await else { @@ -6936,6 +6884,14 @@ pub(crate) mod tests { assert_eq!(signed_setup_snapshot(&writer_parent), before); } observed.fetch_add(1, Ordering::SeqCst); + if oversized_catalog && request["path"] == MODEL_CATALOG_FILE { + // Keep the body open and unsent: refusal must use the header alone. + send.write_all(&((MAX_MODEL_CATALOG_BYTES as u64) + 1).to_be_bytes()) + .await + .unwrap(); + let _ = send.stopped().await; + continue; + } let bytes = &files[request["path"].as_str().unwrap()]; send.write_all(&(bytes.len() as u64).to_be_bytes()) .await @@ -6963,9 +6919,16 @@ pub(crate) mod tests { } else { assert!(result.is_err(), "{case}"); assert_eq!(signed_setup_snapshot(&data), before, "{case}"); + if case == "catalog size" { + let error = format!("{:#}", result.unwrap_err()); + assert!( + error.contains(&format!("exceeds its {MAX_MODEL_CATALOG_BYTES}-byte bound")), + "{error}" + ); + } let expected_requests = match case { "component hash" => 1, - "catalog hash" | "catalog signer" => 2, + "catalog hash" | "catalog signer" | "catalog size" => 2, "head signer" | "release signer" | "pending journal" | "writer busy" => 0, _ => panic!("unknown refusal case"), }; diff --git a/model-catalog.json b/model-catalog.json deleted file mode 100644 index fd031950d..000000000 --- a/model-catalog.json +++ /dev/null @@ -1 +0,0 @@ -{"payload":{"entries":[{"capsule_manifest":{"entrypoint":"weights.gguf","model_content":{"consumer_interface":"elastos.provider.model","consumer_interface_version":"0.1.0","engine":"llama.cpp","format":"gguf","license":{"path":"LICENSE","spdx_id":"Apache-2.0"},"minimum_memory_mb":8192,"provenance":{"base_license":{"path":"LICENSE.base","spdx_id":"Apache-2.0"},"base_repository":"Qwen/Qwen3.5-9B","base_revision":"c202236235762e1c871ad0ccb60c8ee5ba337b9a","path":"PROVENANCE.md","quantized_repository":"bartowski/Qwen_Qwen3.5-9B-GGUF","quantized_revision":"bc658f7a1ae222ee901a565aafd2f5b8c839e176"},"quantization":"Q4_K_M"},"name":"qwen3-5-9b-q4-k-m-local","projections":["content"],"role":"content","schema":"elastos.capsule/v1","type":"data","version":"0.1.0"},"cid":"bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi","object_manifest":{"content_digest":"sha256:93a66d041debbef5daad88c5673666e38187ee64ec136d51b526ab34cdcce212","files":[{"path":"LICENSE","sha256":"bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a","size":11544},{"path":"LICENSE.base","sha256":"bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a","size":11544},{"path":"PROVENANCE.md","sha256":"e7f5d1b457faa5c42070106cea7c0de2f67aedddaae7bb848b2de4658b8ad57d","size":594},{"path":"capsule.json","sha256":"f95e193acd689473e36e9e962afaf57c3dd564815904ef39f29359c6ccdf521d","size":721},{"path":"weights.gguf","sha256":"d784ce9eda1a5a7b51e8f705a9e6310844bf4f173654d115823c775fdea56d43","size":6169341984}],"kind":"capsule","schema":"elastos.content.object.manifest/v1"}},{"capsule_manifest":{"entrypoint":"weights.gguf","model_content":{"consumer_interface":"elastos.provider.model","consumer_interface_version":"0.1.0","engine":"llama.cpp","format":"gguf","license":{"path":"LICENSE","spdx_id":"Apache-2.0"},"minimum_memory_mb":512,"provenance":{"base_license":{"path":"LICENSE.base","spdx_id":"Apache-2.0"},"base_repository":"HuggingFaceTB/SmolLM2-135M-Instruct","base_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","path":"PROVENANCE.md","quantized_repository":"unsloth/SmolLM2-135M-Instruct-GGUF","quantized_revision":"9e6855bc4be717fca1ef21360a1db4b29d5c559a"},"quantization":"Q8_0"},"name":"smollm2-135m-instruct-q8-0-local","projections":["content"],"role":"content","schema":"elastos.capsule/v1","type":"data","version":"0.1.0"},"cid":"bafybeidy5kfvqwg6g6pfgdfwslmhijosbeskt5b2duqdqxnc7e6fwmr72y","object_manifest":{"content_digest":"sha256:9599f2dcce56595e67c50c5f4434e04f14800504b45f78b2d7e895dc183cd619","files":[{"path":"LICENSE","sha256":"bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a","size":11544},{"path":"LICENSE.base","sha256":"bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a","size":11544},{"path":"PROVENANCE.md","sha256":"df5a337b439c3511487a4849ff658ecf65671e72a81a375e441ae200f535efcd","size":537},{"path":"capsule.json","sha256":"8f916fbe895beaa561631624f94f7db90a1ce5816cc1a193070629dc3e01d629","size":751},{"path":"weights.gguf","sha256":"c4a3dd037301b6ecea31d6da37f5cd793ead920dd5ddfe6d589294628d6ce66a","size":144811072}],"kind":"capsule","schema":"elastos.content.object.manifest/v1"}}],"published_at":1789592973,"schema":"elastos.model.catalog/v1"},"signature":"2d3177d67b83c36b2afb282728cc7ea76b3a571c80c5bc853975c879e8e9965c7cf973ba4c7bda26f746cb07f1590c9ee01d66547fa72a3affdadf2ae12bc504","signer_did":"did:key:z6Mkjg9duxEF2nskEPR9F38eSfrY6F1GyWUq5aDbgsMqgcjA"} \ No newline at end of file diff --git a/scripts/check-product-data.mjs b/scripts/check-product-data.mjs index 9657d3697..49fa36d42 100644 --- a/scripts/check-product-data.mjs +++ b/scripts/check-product-data.mjs @@ -2,7 +2,6 @@ // Validate shipped data. Runtime and UI behaviour belongs in executable tests. import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs"; import { dirname, isAbsolute, resolve, sep, win32 } from "node:path"; import { fileURLToPath } from "node:url"; @@ -295,50 +294,6 @@ export function validateComponents(components, manifests) { } } -export function validateModelCatalog(components, catalog) { - const policy = components.model_catalog; - assert(policy?.head_cid?.startsWith("bafkrei"), "Model catalog head CID"); - assert(Array.isArray(policy.publisher_dids) && policy.publisher_dids.length === 1, "Model catalog publisher DID"); - same(catalog.signer_did, policy.publisher_dids[0], "Model catalog trusted signer"); - same(catalog.payload?.schema, "elastos.model.catalog/v1", "Model catalog schema"); - assert(!Object.hasOwn(catalog.payload, "expires_at"), "Permanent model catalog expiry"); - assert(policy.local_use?.max_cache_bytes >= 18516684377 && policy.local_use?.max_model_memory_bytes >= 8589934592, "Model local-use floors"); - const entries = requireList(catalog.payload.entries, "Model catalog entries"); - assert(entries.length > 0, "Model catalog is empty"); - unique(entries.map((entry) => entry.cid), "Model catalog CIDs"); - for (const entry of entries) { - const capsule = entry.capsule_manifest; - same(capsule?.role, "content", "Model content role"); - same(capsule?.type, "data", "Model content type"); - const model = capsule.model_content; - same(model?.format, "gguf", "Model format"); - same(model?.engine, "llama.cpp", "Model engine"); - const files = requireList(entry.object_manifest?.files, "Model object files"); - unique(files.map((file) => file.path), "Model object file paths"); - for (const path of [capsule.entrypoint, model.license?.path, model.provenance?.path, model.provenance?.base_license?.path]) assert(files.some((file) => file.path === path), `Model object lacks ${path}`); - for (const file of files) { - safePath(file.path, "Model object file"); - assert(/^[a-f0-9]{64}$/.test(file.sha256) && Number.isSafeInteger(file.size) && file.size >= 0, `Model object ${file.path} integrity metadata`); - } - assert(Number.isSafeInteger(model.minimum_memory_mb) && model.minimum_memory_mb > 0, "Model minimum memory must be a positive integer"); - assert(model.minimum_memory_mb * 1024 * 1024 <= policy.local_use.max_model_memory_bytes, "Catalog model exceeds local memory policy"); - } -} - -export function validateCatalogBinding(components, bytes) { - // CIDv1, raw codec, SHA-256 multihash, lower-case base32. - const content = Buffer.concat([Buffer.from([1, 0x55, 0x12, 0x20]), createHash("sha256").update(bytes).digest()]); - const alphabet = "abcdefghijklmnopqrstuvwxyz234567"; - let encoded = "b", bits = 0, value = 0; - for (const byte of content) { - value = (value << 8) | byte; - bits += 8; - while (bits >= 5) { bits -= 5; encoded += alphabet[(value >>> bits) & 31]; } - } - if (bits) encoded += alphabet[(value << (5 - bits)) & 31]; - same(components.model_catalog.head_cid, encoded, "Model catalog bytes must match the pinned head CID"); -} - export function checkProductData(repoRoot = root) { const json = (path) => JSON.parse(readFileSync(resolve(repoRoot, path), "utf8")); const manifests = {}; @@ -376,8 +331,6 @@ export function checkProductData(repoRoot = root) { readFileSync(resolve(repoRoot, "scripts/publish-release.sh"), "utf8"), readFileSync(resolve(repoRoot, "elastos/crates/elastos-server/src/publish.rs"), "utf8"), )); - validateModelCatalog(components, json("model-catalog.json")); - validateCatalogBinding(components, readFileSync(resolve(repoRoot, "model-catalog.json"))); const pwa = json("capsules/home/browser/manifest.webmanifest"); same(pwa.display, "standalone", "Home PWA display"); for (const size of [192, 512]) { diff --git a/scripts/check-product-data.test.mjs b/scripts/check-product-data.test.mjs index 27395f3f0..e75ce20ad 100644 --- a/scripts/check-product-data.test.mjs +++ b/scripts/check-product-data.test.mjs @@ -5,7 +5,7 @@ import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { checkProductData, parsePublishData, validatePublishData, validateCapsule, validateCatalogBinding, validateComponents, validateContracts, validateModelCatalog } from "./check-product-data.mjs"; +import { checkProductData, parsePublishData, validatePublishData, validateCapsule, validateComponents, validateContracts } from "./check-product-data.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const json = (path) => JSON.parse(readFileSync(resolve(root, path), "utf8")); @@ -25,7 +25,6 @@ for (const base of ["capsules", "elastos/capsules"]) { } } const components = json("components.json"); -const catalog = json("model-catalog.json"); const clone = (value) => structuredClone(value); const shellPublish = readFileSync(resolve(root, "scripts/publish-release.sh"), "utf8"); const rustPublish = readFileSync(resolve(root, "elastos/crates/elastos-server/src/publish.rs"), "utf8"); @@ -77,14 +76,6 @@ test("shipped capsule, profile, catalog and icon data pass together", () => { assert(result.capsules > 0 && result.profiles > 0 && result.method_contracts > 0); }); -test("catalog head binding refuses changed signed bytes and a substituted CID", () => { - const bytes = readFileSync(resolve(root, "model-catalog.json")); - assert.throws(() => validateCatalogBinding(components, Buffer.concat([bytes, Buffer.from(" ")])), /pinned head CID/); - const value = clone(components); - value.model_catalog.head_cid = "bafkreiother"; - assert.throws(() => validateCatalogBinding(value, bytes), /pinned head CID/); -}); - for (const [name, mutate, message] of [ ["unknown role", (value) => value.role = "root", /unknown role/], ["provider metadata on an app", (value) => value.authority = {}, /provider owns authority/], @@ -177,22 +168,6 @@ for (const [name, mutate, message] of [ }); } -for (const [name, mutate, message] of [ - ["substituted signer", (value) => value.signer_did = "did:key:other", /trusted signer/], - ["missing model weights", (value) => value.payload.entries[0].object_manifest.files.pop(), /lacks weights/], - ["missing model provenance", (value) => value.payload.entries[0].capsule_manifest.model_content.provenance.path = "missing.md", /lacks missing/], - ["malformed file digest", (value) => value.payload.entries[0].object_manifest.files[0].sha256 = "00", /integrity metadata/], - ["model file traversal", (value) => value.payload.entries[0].object_manifest.files.push({ path: "../outside", sha256: "0".repeat(64), size: 1 }), /safe artifact path/], - ["absolute model file", (value) => value.payload.entries[0].object_manifest.files.push({ path: "/outside", sha256: "0".repeat(64), size: 1 }), /safe artifact path/], - ["negative minimum memory", (value) => value.payload.entries[0].capsule_manifest.model_content.minimum_memory_mb = -1, /positive integer/], -]) { - test(`model data refuse ${name}`, () => { - const value = clone(catalog); - mutate(value); - assert.throws(() => validateModelCatalog(components, value), message); - }); -} - function dataFixture() { const fixture = mkdtempSync(join(tmpdir(), "product-data-fixture-")); try { @@ -211,7 +186,7 @@ function dataFixture() { for (const size of [32, 64, 128, 256]) copyFileSync(resolve(root, dirname(path), value.icon, `icon-${size}.png`), resolve(directory, value.icon, `icon-${size}.png`)); } } - for (const path of ["components.json", "model-catalog.json", "capsules/home/browser/manifest.webmanifest", "capsules/home/browser/elastos-home-icon-192.png", "capsules/home/browser/elastos-home-icon-512.png", "scripts/publish-release.sh", "elastos/crates/elastos-server/src/publish.rs"]) { + for (const path of ["components.json", "capsules/home/browser/manifest.webmanifest", "capsules/home/browser/elastos-home-icon-192.png", "capsules/home/browser/elastos-home-icon-512.png", "scripts/publish-release.sh", "elastos/crates/elastos-server/src/publish.rs"]) { mkdirSync(dirname(resolve(fixture, path)), { recursive: true }); copyFileSync(resolve(root, path), resolve(fixture, path)); } diff --git a/scripts/fetch/fetch-model-smoke.py b/scripts/fetch/fetch-model-smoke.py deleted file mode 100755 index bc91fef8f..000000000 --- a/scripts/fetch/fetch-model-smoke.py +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import http.server -import io -import json -import os -import stat -import subprocess -import tarfile -import tempfile -import threading -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[2] -FETCH = ROOT / "scripts/fetch/fetch-model.sh" - - -def sha256(value): - return "sha256:" + hashlib.sha256(value).hexdigest() - - -def engine_archive(): - output = io.BytesIO() - with tarfile.open(fileobj=output, mode="w:gz") as archive: - payload = b"fixture llama-server\n" - dylib_payload = b"fixture dylib\n" - directory = tarfile.TarInfo("llama-fixture-v1") - directory.type = tarfile.DIRTYPE - directory.mode = 0o755 - archive.addfile(directory) - binary = tarfile.TarInfo("llama-fixture-v1/llama-server") - binary.mode = 0o755 - binary.size = len(payload) - archive.addfile(binary, io.BytesIO(payload)) - dylib = tarfile.TarInfo("llama-fixture-v1/libfixture.dylib") - dylib.mode = 0o644 - dylib.size = len(dylib_payload) - archive.addfile(dylib, io.BytesIO(dylib_payload)) - dylib_link = tarfile.TarInfo("llama-fixture-v1/libllama.dylib") - dylib_link.type = tarfile.SYMTYPE - dylib_link.linkname = "libfixture.dylib" - archive.addfile(dylib_link) - return output.getvalue() - - -class ArtifactServer: - def __init__(self, artifacts): - self.artifacts = artifacts - self.requests = [] - owner = self - - class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - owner.requests.append(self.path) - if self.path == "/interrupted.gguf": - payload = owner.artifacts[self.path] - self.send_response(200) - self.send_header("Content-Length", str(len(payload) + 64)) - self.end_headers() - self.wfile.write(payload) - self.wfile.flush() - self.connection.shutdown(1) - return - payload = owner.artifacts.get(self.path) - if payload is None: - self.send_error(404) - return - self.send_response(200) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def log_message(self, _format, *_args): - pass - - self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) - self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) - - def __enter__(self): - self.thread.start() - return self - - def __exit__(self, *_args): - self.server.shutdown() - self.thread.join() - self.server.server_close() - - @property - def origin(self): - return f"http://127.0.0.1:{self.server.server_port}" - - -def write_manifest(path, origin, engine, stable_url="/stable.gguf", stable_checksum=None): - path.parent.mkdir(parents=True, exist_ok=True) - stable = b"stable model\n" - bonsai = b"experimental model\n" - manifest = { - "external": { - "llama-server": { - "version": "fixture-v1", - "platforms": { - "darwin-arm64": { - "url": origin + "/llama.tar.gz", - "checksum": sha256(engine), - "extract_path": "llama-fixture-v1", - "install_path": "libexec/llama.cpp/fixture-v1/darwin-arm64", - "binary_path": "llama-server", - }, - "linux-amd64": { - "url": origin + "/llama.tar.gz", - "checksum": sha256(engine), - "extract_path": "llama-fixture-v1/llama-server", - "install_path": "bin/llama-server", - } - }, - }, - "model-qwen3.5-9b": { - "description": "stable fixture", - "platforms": { - "*": { - "url": origin + stable_url, - "checksum": stable_checksum or sha256(stable), - "install_path": "models/stable.gguf", - } - }, - }, - "model-bonsai-8b-q1": { - "description": "experimental fixture", - "platforms": { - "*": { - "url": origin + "/experimental.gguf", - "checksum": sha256(bonsai), - "install_path": "models/experimental.gguf", - } - }, - }, - } - } - path.write_text(json.dumps(manifest), encoding="utf-8") - return stable, bonsai, engine - - -def fixture_env(root, manifest, data, system="Darwin", machine="arm64"): - tool_dir = root / "tools" - tool_dir.mkdir(parents=True, mode=0o700) - uname = tool_dir / "uname" - uname.write_text( - f'#!/usr/bin/env bash\n[[ "$1" == "-s" ]] && echo {system} || echo {machine}\n', - encoding="utf-8", - ) - uname.chmod(0o700) - return { - **os.environ, - "PATH": f"{tool_dir}:{os.environ['PATH']}", - "ELASTOS_COMPONENTS_MANIFEST": str(manifest), - "ELASTOS_DATA_DIR": str(data), - "ELASTOS_MODEL_ALLOW_LOCAL_HTTP_FIXTURE": "1", - } - - -def run_fetch(env, candidate, expected=0): - result = subprocess.run( - ["bash", str(FETCH), candidate], - env=env, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False, - ) - if result.returncode != expected: - raise AssertionError( - f"fetch returned {result.returncode}, expected {expected}: " - f"stdout={result.stdout!r} stderr={result.stderr!r}" - ) - return result - - -def assert_reported_engine(result, expected): - lines = [ - line for line in result.stdout.splitlines() if line.startswith(" llama-server: ") - ] - if lines != [f" llama-server: {expected}"]: - raise AssertionError(f"unexpected reported llama-server path: {lines}") - - -def assert_no_partial_downloads(data): - leftovers = [path for path in data.rglob("*") if ".download." in path.name] - if leftovers: - raise AssertionError(f"partial downloads remain: {leftovers}") - - -def make_tree_removable(root): - for directory, _, _ in os.walk(root): - Path(directory).chmod(0o700) - - -def bundle_snapshot(root): - snapshot = [] - - def visit(directory): - for child in sorted(os.scandir(directory), key=lambda entry: entry.name): - metadata = child.stat(follow_symlinks=False) - relative = Path(child.path).relative_to(root).as_posix() - identity = ( - relative, - metadata.st_ino, - metadata.st_nlink, - stat.S_IMODE(metadata.st_mode), - ) - if stat.S_ISDIR(metadata.st_mode): - snapshot.append(identity + ("directory",)) - visit(child.path) - elif stat.S_ISREG(metadata.st_mode): - snapshot.append(identity + ("file", Path(child.path).read_bytes())) - elif stat.S_ISLNK(metadata.st_mode): - snapshot.append(identity + ("symlink", os.readlink(child.path))) - else: - snapshot.append(identity + ("special",)) - - visit(root) - return snapshot - - -def install_mac_bundle(root, server): - manifest = root / "components.json" - data = root / "data" - write_manifest(manifest, server.origin, server.artifacts["/llama.tar.gz"]) - env = fixture_env(root, manifest, data) - start = len(server.requests) - run_fetch(env, "stable") - if server.requests[start:] != ["/stable.gguf", "/llama.tar.gz"]: - raise AssertionError("macOS fixture did not fetch its initial artifacts") - bundle = data / "libexec/llama.cpp/fixture-v1/darwin-arm64" - return env, data, bundle - - -def assert_invalid_bundle_rejected(env, data, bundle, server): - before = bundle_snapshot(bundle) - start = len(server.requests) - result = run_fetch(env, "stable", expected=1) - if server.requests[start:]: - raise AssertionError("invalid bundle triggered an artifact download") - if "keep it for operator review" not in result.stderr: - raise AssertionError("invalid bundle failure did not preserve operator evidence") - if bundle_snapshot(bundle) != before: - raise AssertionError("invalid bundle changed after failed reuse validation") - assert_no_partial_downloads(data) - - -def test_platform_selection_and_verified_reuse(root, server): - manifest = root / "components.json" - data = root / "stable-data" - engine = server.artifacts["/llama.tar.gz"] - stable, _, _ = write_manifest(manifest, server.origin, engine) - env = fixture_env(root, manifest, data) - - first = run_fetch(env, "stable") - if "Platform: darwin-arm64" not in first.stdout: - raise AssertionError("macOS arm64 platform was not selected") - if server.requests != ["/stable.gguf", "/llama.tar.gz"]: - raise AssertionError(f"unexpected stable artifact requests: {server.requests}") - model = data / "models/stable.gguf" - archive = data / "artifacts/llama-fixture-v1-darwin-arm64.tar.gz" - bundle = data / "libexec/llama.cpp/fixture-v1/darwin-arm64" - binary = bundle / "llama-server" - dylib = bundle / "libfixture.dylib" - dylib_link = bundle / "libllama.dylib" - receipt = bundle / ".elastos-engine.json" - link = data / "bin/llama-server" - assert_reported_engine(first, binary.resolve()) - if model.read_bytes() != stable or archive.read_bytes() != engine: - raise AssertionError("verified artifacts were not installed exactly") - if dylib.read_bytes() != b"fixture dylib\n": - raise AssertionError("macOS dylib was not installed exactly") - if not dylib_link.is_symlink() or os.readlink(dylib_link) != "libfixture.dylib": - raise AssertionError("macOS dylib symlink was not installed exactly") - if not link.is_symlink() or link.resolve() != binary.resolve(): - raise AssertionError( - f"stable llama-server link is invalid: symlink={link.is_symlink()} " - f"target={os.readlink(link) if link.is_symlink() else None} " - f"resolved={link.resolve()} expected={binary.resolve()}" - ) - expected_receipt = { - "schema": "elastos.local-model-engine/v2", - "version": "fixture-v1", - "platform": "darwin-arm64", - "archive_sha256": sha256(engine), - "entries": [ - { - "path": "libfixture.dylib", - "sha256": sha256(b"fixture dylib\n"), - "type": "file", - }, - { - "path": "libllama.dylib", - "target": "libfixture.dylib", - "type": "symlink", - }, - { - "path": "llama-server", - "sha256": sha256(b"fixture llama-server\n"), - "type": "file", - }, - ], - } - if json.loads(receipt.read_text(encoding="utf-8")) != expected_receipt: - raise AssertionError("macOS bundle receipt does not contain the exact inventory") - expected_modes = { - bundle: 0o500, - binary: 0o500, - dylib: 0o400, - receipt: 0o400, - } - for path, expected_mode in expected_modes.items(): - actual_mode = stat.S_IMODE(path.stat().st_mode) - if actual_mode != expected_mode: - raise AssertionError( - f"unexpected protected mode for {path.name}: {actual_mode:o}" - ) - before = ( - model.stat().st_ino, - archive.stat().st_ino, - binary.stat().st_ino, - dylib.stat().st_ino, - dylib_link.lstat().st_ino, - receipt.stat().st_ino, - ) - - second = run_fetch(env, "stable") - after = ( - model.stat().st_ino, - archive.stat().st_ino, - binary.stat().st_ino, - dylib.stat().st_ino, - dylib_link.lstat().st_ino, - receipt.stat().st_ino, - ) - if server.requests != ["/stable.gguf", "/llama.tar.gz"] or before != after: - raise AssertionError("verified reuse downloaded or replaced an artifact") - if "already present and verified" not in second.stdout: - raise AssertionError("verified reuse was not reported") - assert_no_partial_downloads(data) - - -def test_changed_dylib_reuse_is_rejected(root, server): - env, data, bundle = install_mac_bundle(root, server) - dylib = bundle / "libfixture.dylib" - dylib.chmod(0o600) - dylib.write_bytes(b"changed dylib\n") - dylib.chmod(0o400) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_missing_bundle_file_reuse_is_rejected(root, server): - env, data, bundle = install_mac_bundle(root, server) - bundle.chmod(0o700) - (bundle / "libfixture.dylib").unlink() - bundle.chmod(0o500) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_added_bundle_file_reuse_is_rejected(root, server): - env, data, bundle = install_mac_bundle(root, server) - bundle.chmod(0o700) - added = bundle / "unexpected.dylib" - added.write_bytes(b"unexpected dylib\n") - added.chmod(0o400) - bundle.chmod(0o500) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_unsafe_bundle_symlink_reuse_is_rejected(root, server): - for name, target in ( - ("absolute", "/private/tmp/outside.dylib"), - ("escaping", "../outside.dylib"), - ): - env, data, bundle = install_mac_bundle(root / name, server) - dylib_link = bundle / "libllama.dylib" - bundle.chmod(0o700) - dylib_link.unlink() - dylib_link.symlink_to(target) - bundle.chmod(0o500) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_redirected_bundle_symlink_reuse_is_rejected(root, server): - env, data, bundle = install_mac_bundle(root, server) - dylib_link = bundle / "libllama.dylib" - bundle.chmod(0o700) - dylib_link.unlink() - dylib_link.symlink_to("llama-server") - bundle.chmod(0o500) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_bundle_protection_reuse_is_rejected(root, server): - for name, mutate in ( - ("directory-mode", lambda case_root, bundle: bundle.chmod(0o700)), - ( - "file-mode", - lambda case_root, bundle: (bundle / "libfixture.dylib").chmod(0o600), - ), - ( - "hardlink", - lambda case_root, bundle: os.link( - bundle / "libfixture.dylib", case_root / "outside-hardlink.dylib" - ), - ), - ): - case_root = root / name - env, data, bundle = install_mac_bundle(case_root, server) - mutate(case_root, bundle) - assert_invalid_bundle_rejected(env, data, bundle, server) - - -def test_experimental_selection(root, server): - manifest = root / "experimental-components.json" - data = root / "experimental-data" - _, bonsai, _ = write_manifest( - manifest, server.origin, server.artifacts["/llama.tar.gz"] - ) - env = fixture_env(root / "experimental-env", manifest, data) - start = len(server.requests) - run_fetch(env, "experimental") - if server.requests[start:] != ["/experimental.gguf", "/llama.tar.gz"]: - raise AssertionError("experimental candidate did not select its exact artifacts") - if (data / "models/experimental.gguf").read_bytes() != bonsai: - raise AssertionError("experimental model bytes differ") - - -def test_linux_file_install_shape(root, server): - manifest = root / "linux-components.json" - data = root / "linux-data" - stable, _, _ = write_manifest( - manifest, server.origin, server.artifacts["/llama.tar.gz"] - ) - env = fixture_env(root / "linux-env", manifest, data, "Linux", "x86_64") - start = len(server.requests) - first = run_fetch(env, "stable") - if "Platform: linux-amd64" not in first.stdout: - raise AssertionError("Linux amd64 platform was not selected") - if server.requests[start:] != ["/stable.gguf", "/llama.tar.gz"]: - raise AssertionError("Linux candidate did not select its exact artifacts") - model = data / "models/stable.gguf" - binary = data / "bin/llama-server" - assert_reported_engine(first, binary.resolve()) - if model.read_bytes() != stable or binary.read_bytes() != b"fixture llama-server\n": - raise AssertionError("Linux file-valued artifacts differ") - if binary.is_symlink(): - raise AssertionError("Linux file-valued install became a bundle link") - before = (model.stat().st_ino, binary.stat().st_ino) - run_fetch(env, "stable") - if server.requests[start:] != ["/stable.gguf", "/llama.tar.gz"]: - raise AssertionError("verified Linux reuse downloaded an artifact") - if before != (model.stat().st_ino, binary.stat().st_ino): - raise AssertionError("verified Linux reuse replaced an artifact") - - -def test_checksum_rejection_preserves_existing(root, server): - manifest = root / "bad-components.json" - data = root / "bad-data" - stable, _, _ = write_manifest( - manifest, - server.origin, - server.artifacts["/llama.tar.gz"], - stable_checksum=sha256(b"expected different bytes"), - ) - existing = data / "models/stable.gguf" - existing.parent.mkdir(parents=True) - existing.write_bytes(b"existing model\n") - inode = existing.stat().st_ino - env = fixture_env(root / "bad-env", manifest, data) - result = run_fetch(env, "stable", expected=1) - if "was not installed" not in result.stderr: - raise AssertionError("checksum failure was not explicit") - if existing.read_bytes() != b"existing model\n" or existing.stat().st_ino != inode: - raise AssertionError("checksum failure replaced the existing model") - if stable == existing.read_bytes(): - raise AssertionError("checksum fixture is invalid") - assert_no_partial_downloads(data) - - -def test_interrupted_download_cleanup(root, server): - manifest = root / "interrupted-components.json" - data = root / "interrupted-data" - stable, _, _ = write_manifest( - manifest, - server.origin, - server.artifacts["/llama.tar.gz"], - stable_url="/interrupted.gguf", - ) - server.artifacts["/interrupted.gguf"] = stable - env = fixture_env(root / "interrupted-env", manifest, data) - result = run_fetch(env, "stable", expected=1) - if "Download failed" not in result.stderr: - raise AssertionError("interrupted download did not fail clearly") - if (data / "models/stable.gguf").exists(): - raise AssertionError("interrupted download published a model") - assert_no_partial_downloads(data) - - -def main(): - artifacts = { - "/stable.gguf": b"stable model\n", - "/experimental.gguf": b"experimental model\n", - "/llama.tar.gz": engine_archive(), - } - with tempfile.TemporaryDirectory(prefix="elastos-model-bootstrap-") as temp: - root = Path(temp) - try: - with ArtifactServer(artifacts) as server: - test_platform_selection_and_verified_reuse(root / "reuse", server) - test_changed_dylib_reuse_is_rejected(root / "changed-dylib", server) - test_missing_bundle_file_reuse_is_rejected(root / "missing-file", server) - test_added_bundle_file_reuse_is_rejected(root / "added-file", server) - test_unsafe_bundle_symlink_reuse_is_rejected( - root / "unsafe-link", server - ) - test_redirected_bundle_symlink_reuse_is_rejected( - root / "redirected-link", server - ) - test_bundle_protection_reuse_is_rejected(root / "protection", server) - test_experimental_selection(root / "experimental", server) - test_linux_file_install_shape(root / "linux", server) - test_checksum_rejection_preserves_existing(root / "checksum", server) - test_interrupted_download_cleanup(root / "interrupted", server) - finally: - make_tree_removable(root) - print("PASS local model bootstrap smoke") - - -if __name__ == "__main__": - main() diff --git a/scripts/fetch/fetch-model.sh b/scripts/fetch/fetch-model.sh deleted file mode 100755 index 133c1ea3c..000000000 --- a/scripts/fetch/fetch-model.sh +++ /dev/null @@ -1,574 +0,0 @@ -#!/usr/bin/env bash -# -# Fetch one pinned local-model candidate and its llama.cpp engine. -# -# Artifact URLs, versions, checksums, and install paths come from components.json. -# Model bytes stay below the operator-owned Runtime data root and outside Git. -# -# Usage: -# ./scripts/fetch/fetch-model.sh # Stable candidate -# ./scripts/fetch/fetch-model.sh experimental # Low-memory comparison -# ./scripts/fetch/fetch-model.sh --list - -set -euo pipefail -umask 077 - -BOLD='\033[1m' -GREEN='\033[0;32m' -CYAN='\033[0;36m' -YELLOW='\033[0;33m' -RED='\033[0;31m' -NC='\033[0m' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -MANIFEST="${ELASTOS_COMPONENTS_MANIFEST:-${PROJECT_ROOT}/components.json}" -INSTALL_DIR="${ELASTOS_DATA_DIR:-${HOME}/.local/share/elastos}" -BIN_DIR="${INSTALL_DIR}/bin" -LLAMA_COMPONENT="llama-server" - -cleanup_paths=("") - -die() { echo -e "${RED}Error:${NC} $*" >&2; exit 1; } -info() { echo -e " ${CYAN}▶${NC} $*"; } -ok() { echo -e " ${GREEN}✓${NC} $*"; } -warn() { echo -e " ${YELLOW}!${NC} $*"; } - -cleanup() { - local path - for path in "${cleanup_paths[@]}"; do - [[ -n "$path" ]] && rm -rf -- "$path" - done - return 0 -} -trap cleanup EXIT HUP INT TERM - -require_cmd() { - command -v "$1" >/dev/null 2>&1 || die "'$1' is required." -} - -sha256_file() { - local path="$1" - if command -v shasum >/dev/null 2>&1; then - shasum -a 256 "$path" | awk '{print $1}' - elif command -v sha256sum >/dev/null 2>&1; then - sha256sum "$path" | awk '{print $1}' - else - die "shasum or sha256sum is required." - fi -} - -verify_sha256() { - local path="$1" expected="$2" actual - if [[ ! "$expected" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "Invalid SHA-256 declaration for $(basename "$path")." >&2 - return 1 - fi - actual="$(sha256_file "$path")" - if [[ "sha256:${actual}" != "$expected" ]]; then - echo "SHA-256 verification failed for $(basename "$path")." >&2 - return 1 - fi -} - -detect_platform() { - local os arch - os="$(uname -s)" - arch="$(uname -m)" - case "${os}:${arch}" in - Darwin:arm64) printf '%s\n' darwin-arm64 ;; - Linux:x86_64) printf '%s\n' linux-amd64 ;; - Linux:aarch64|Linux:arm64) printf '%s\n' linux-arm64 ;; - *) die "Unsupported model bootstrap platform: ${os} ${arch}." ;; - esac -} - -component_value() { - local component="$1" platform="$2" field="$3" - python3 - "$MANIFEST" "$component" "$platform" "$field" <<'PY' -import json -import pathlib -import sys - -manifest_path, component_name, platform, field = sys.argv[1:] -try: - manifest = json.loads(pathlib.Path(manifest_path).read_text(encoding="utf-8")) - component = manifest["external"][component_name] - platform_info = component.get("platforms", {}).get(platform) - if platform_info is None: - platform_info = component.get("platforms", {}).get("*", {}) - value = platform_info.get(field, component.get(field)) - if not isinstance(value, (str, int)): - raise ValueError(f"{component_name}.{platform}.{field} is missing or invalid") -except (KeyError, OSError, ValueError, json.JSONDecodeError) as error: - raise SystemExit(f"invalid component manifest: {error}") -print(value) -PY -} - -validate_relative_path() { - python3 - "$1" <<'PY' -import pathlib -import sys - -path = pathlib.PurePosixPath(sys.argv[1]) -if path.is_absolute() or not path.parts or ".." in path.parts: - raise SystemExit("component install path must be a non-empty relative path") -PY -} - -validate_download_url() { - local url="$1" - case "$url" in - https://*) ;; - http://127.0.0.1:*|http://localhost:*) - [[ "${ELASTOS_MODEL_ALLOW_LOCAL_HTTP_FIXTURE:-}" == "1" ]] || die "Model artifacts require HTTPS." - ;; - *) die "Model artifacts require an HTTPS URL." ;; - esac -} - -download_verified() { - local label="$1" url="$2" checksum="$3" destination="$4" - local parent temporary - parent="$(dirname "$destination")" - mkdir -p "$parent" - - if [[ -f "$destination" && ! -L "$destination" ]]; then - if verify_sha256 "$destination" "$checksum" 2>/dev/null; then - ok "${label} already present and verified" - return 0 - fi - warn "${label} checksum changed; downloading a verified replacement" - elif [[ -e "$destination" || -L "$destination" ]]; then - die "${label} destination is not a regular file." - fi - - validate_download_url "$url" - temporary="$(mktemp "${parent}/.$(basename "$destination").download.XXXXXX")" - cleanup_paths+=("$temporary") - info "Downloading ${label}..." - curl --fail --location --progress-bar --output "$temporary" "$url" || die "Download failed for ${label}." - verify_sha256 "$temporary" "$checksum" || die "${label} was not installed." - chmod 0600 "$temporary" - mv -f -- "$temporary" "$destination" - ok "${label} downloaded and verified" -} - -engine_receipt_matches() { - local receipt="$1" binary="$2" version="$3" platform="$4" archive_checksum="$5" - [[ -f "$receipt" && -f "$binary" && ! -L "$binary" ]] || return 1 - python3 - "$receipt" "$version" "$platform" "$archive_checksum" "sha256:$(sha256_file "$binary")" <<'PY' -import json -import pathlib -import sys - -receipt_path, version, platform, archive_checksum, binary_checksum = sys.argv[1:] -try: - receipt = json.loads(pathlib.Path(receipt_path).read_text(encoding="utf-8")) -except (OSError, json.JSONDecodeError): - raise SystemExit(1) -expected = { - "schema": "elastos.local-model-engine/v1", - "version": version, - "platform": platform, - "archive_sha256": archive_checksum, - "llama_server_sha256": binary_checksum, -} -raise SystemExit(0 if receipt == expected else 1) -PY -} - -write_engine_receipt() { - local receipt="$1" binary="$2" version="$3" platform="$4" archive_checksum="$5" - python3 - "$receipt" "$version" "$platform" "$archive_checksum" "sha256:$(sha256_file "$binary")" <<'PY' -import json -import os -import pathlib -import sys - -receipt_path, version, platform, archive_checksum, binary_checksum = sys.argv[1:] -path = pathlib.Path(receipt_path) -payload = { - "schema": "elastos.local-model-engine/v1", - "version": version, - "platform": platform, - "archive_sha256": archive_checksum, - "llama_server_sha256": binary_checksum, -} -path.write_text(json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8") -os.chmod(path, 0o600) -PY -} - -bundle_engine_receipt() { - local mode="$1" bundle="$2" version="$3" platform="$4" archive_checksum="$5" - python3 - "$mode" "$bundle" "$version" "$platform" "$archive_checksum" <<'PY' -import hashlib -import json -import os -import pathlib -import posixpath -import stat -import sys - -mode, bundle_arg, version, platform, archive_checksum = sys.argv[1:] -bundle = pathlib.Path(bundle_arg) -receipt_path = bundle / ".elastos-engine.json" -entries = [] -directories = [] -regular_files = [] - - -def reject(message): - raise ValueError(message) - - -def file_sha256(path): - digest = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(block) - return "sha256:" + digest.hexdigest() - - -def inventory(directory, prefix=""): - directories.append(directory) - for child in os.scandir(directory): - relative = posixpath.join(prefix, child.name) - if relative == ".elastos-engine.json": - continue - metadata = child.stat(follow_symlinks=False) - path = pathlib.Path(child.path) - if stat.S_ISDIR(metadata.st_mode): - inventory(path, relative) - elif stat.S_ISREG(metadata.st_mode): - entries.append( - {"path": relative, "sha256": file_sha256(path), "type": "file"} - ) - regular_files.append((path, metadata.st_mode)) - elif stat.S_ISLNK(metadata.st_mode): - target = os.readlink(path) - if not target: - reject(f"symlink {relative} has an empty target") - target_path = pathlib.PurePosixPath(target) - if target_path.is_absolute(): - reject(f"symlink {relative} has an absolute target") - if ".." in target_path.parts: - reject(f"symlink {relative} has a '..' target") - normalized = posixpath.normpath( - posixpath.join(posixpath.dirname(relative), target) - ) - if ( - normalized == ".." - or normalized.startswith("../") - or posixpath.isabs(normalized) - ): - reject(f"symlink {relative} leaves the bundle") - entries.append( - {"path": relative, "target": target, "type": "symlink"} - ) - else: - reject(f"entry {relative} is a special file") - - -def validate_protection(): - owner = os.getuid() - for path in directories: - metadata = os.lstat(path) - if metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o500: - reject(f"directory {path} has unsafe ownership or mode") - for path, _ in regular_files: - metadata = os.lstat(path) - file_mode = stat.S_IMODE(metadata.st_mode) - if ( - metadata.st_uid != owner - or metadata.st_nlink != 1 - or file_mode not in (0o400, 0o500) - ): - reject(f"file {path} has unsafe ownership, links, or mode") - metadata = os.lstat(receipt_path) - if ( - metadata.st_uid != owner - or metadata.st_nlink != 1 - or stat.S_IMODE(metadata.st_mode) != 0o400 - ): - reject("receipt has unsafe ownership, links, or mode") - - -try: - bundle_metadata = os.lstat(bundle) - if not stat.S_ISDIR(bundle_metadata.st_mode): - reject("bundle is not a directory") - if mode == "write" and os.path.lexists(receipt_path): - reject("archive contains the reserved receipt path") - if mode == "verify": - receipt_metadata = os.lstat(receipt_path) - if not stat.S_ISREG(receipt_metadata.st_mode): - reject("receipt is not a regular file") - elif mode != "write": - reject(f"unsupported receipt mode: {mode}") - - inventory(bundle) - entries.sort(key=lambda entry: entry["path"]) - payload = { - "schema": "elastos.local-model-engine/v2", - "version": version, - "platform": platform, - "archive_sha256": archive_checksum, - "entries": entries, - } - - if mode == "verify": - validate_protection() - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - if receipt != payload: - reject("receipt does not match the installed inventory") - else: - owner = os.getuid() - if any(os.lstat(path).st_uid != owner for path in directories): - reject("bundle directory owner differs from the current user") - if any( - os.lstat(path).st_uid != owner or os.lstat(path).st_nlink != 1 - for path, _ in regular_files - ): - reject("bundle file owner or link count is unsafe") - receipt_path.write_text( - json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - os.chmod(receipt_path, 0o400) - for path, file_mode in regular_files: - os.chmod(path, 0o500 if file_mode & 0o111 else 0o400) - for path in sorted(directories, key=lambda value: len(value.parts), reverse=True): - os.chmod(path, 0o500) - validate_protection() -except (OSError, ValueError) as error: - print(f"invalid llama-server bundle: {error}", file=sys.stderr) - raise SystemExit(1) -PY -} - -protect_engine_install_parents() { - python3 - "$1" "$2" <<'PY' -import os -import stat -import sys -from pathlib import Path - -root = Path(sys.argv[1]) -if not root.is_absolute(): - root = Path.cwd() / root -relative = Path(sys.argv[2]) -if relative.is_absolute() or any(part in (os.pardir, os.curdir) for part in relative.parts): - raise SystemExit("llama-server bundle install path is invalid") -if not root.is_dir(): - raise SystemExit(f"model engine parent is not protected: {root}") -root = root.resolve() -uid = os.getuid() -flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW -if hasattr(os, "O_CLOEXEC"): - flags |= os.O_CLOEXEC -path = root -for part in relative.parts: - path = path / part - try: - path.relative_to(root) - except ValueError as exc: - raise SystemExit(f"model engine parent is not protected: {path}") from exc - try: - fd = os.open(path, flags) - except OSError as exc: - raise SystemExit(f"model engine parent is not protected: {path}") from exc - try: - metadata = os.fstat(fd) - if metadata.st_uid != uid: - raise SystemExit(f"model engine parent is not protected: {path}") - mode = stat.S_IMODE(metadata.st_mode) - if mode & 0o022: - os.fchmod(fd, mode & ~0o022) - after = os.fstat(fd) - if after.st_uid != uid or stat.S_IMODE(after.st_mode) & 0o022: - raise SystemExit(f"model engine parent is not protected: {path}") - finally: - os.close(fd) -PY -} - -install_prebuilt_engine() { - local platform="$1" version url checksum extract_path install_path binary_path - local archive engine_dir engine_binary receipt stage staged_binary install_temp link_temp - local engine_ready=0 - version="$(component_value "$LLAMA_COMPONENT" "$platform" version)" - url="$(component_value "$LLAMA_COMPONENT" "$platform" url)" - checksum="$(component_value "$LLAMA_COMPONENT" "$platform" checksum)" - extract_path="$(component_value "$LLAMA_COMPONENT" "$platform" extract_path)" - install_path="$(component_value "$LLAMA_COMPONENT" "$platform" install_path)" - binary_path="$(python3 - "$MANIFEST" "$platform" <<'PY' -import json -import pathlib -import sys - -manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) -info = manifest["external"]["llama-server"].get("platforms", {}).get(sys.argv[2], {}) -value = info.get("binary_path", "") -if not isinstance(value, str): - raise SystemExit("llama-server binary_path must be a string") -print(value) -PY -)" - validate_relative_path "$extract_path" - validate_relative_path "$install_path" - - archive="${INSTALL_DIR}/artifacts/llama-${version}-${platform}.tar.gz" - if [[ -n "$binary_path" ]]; then - validate_relative_path "$binary_path" - [[ "$extract_path" == "llama-${version}" ]] \ - || die "llama-server bundle path does not match its version." - [[ "$install_path" == "libexec/llama.cpp/${version}/${platform}" ]] \ - || die "llama-server bundle install path does not match its version and platform." - engine_dir="${INSTALL_DIR}/${install_path}" - engine_binary="${engine_dir}/${binary_path}" - receipt="${engine_dir}/.elastos-engine.json" - else - engine_dir="" - engine_binary="${INSTALL_DIR}/${install_path}" - receipt="${BIN_DIR}/.llama-server-${version}-${platform}.json" - fi - mkdir -p "$BIN_DIR" - - if [[ -n "$binary_path" ]]; then - if [[ -e "$engine_dir" || -L "$engine_dir" ]]; then - if [[ -d "$engine_dir" && ! -L "$engine_dir" \ - && -f "$engine_binary" && -x "$engine_binary" && ! -L "$engine_binary" ]] \ - && bundle_engine_receipt verify "$engine_dir" "$version" "$platform" "$checksum"; then - engine_ready=1 - else - die "Existing llama-server bundle failed verification; keep it for operator review." - fi - fi - elif engine_receipt_matches "$receipt" "$engine_binary" "$version" "$platform" "$checksum"; then - engine_ready=1 - fi - - if [[ "$engine_ready" == "1" ]]; then - ok "llama-server ${version} already present and verified" - else - download_verified "llama-server ${version}" "$url" "$checksum" "$archive" - stage="$(mktemp -d "${INSTALL_DIR}/.llama-engine.XXXXXX")" - cleanup_paths+=("$stage") - tar -xzf "$archive" -C "$stage" - staged_binary="${stage}/${extract_path}" - [[ -n "$binary_path" ]] && staged_binary="${staged_binary}/${binary_path}" - [[ -f "$staged_binary" && ! -L "$staged_binary" ]] \ - || die "Verified llama-server archive has an invalid binary layout." - chmod 0700 "$staged_binary" - if [[ -n "$binary_path" ]]; then - bundle_engine_receipt write "${stage}/${extract_path}" \ - "$version" "$platform" "$checksum" - chmod 0700 "${stage}/${extract_path}" - mkdir -p "$(dirname "$engine_dir")" - [[ ! -e "$engine_dir" && ! -L "$engine_dir" ]] \ - || die "Existing llama-server bundle failed verification; keep it for operator review." - mv -- "${stage}/${extract_path}" "$engine_dir" - chmod 0500 "$engine_dir" - else - mkdir -p "$(dirname "$engine_binary")" - install_temp="$(mktemp "$(dirname "$engine_binary")/.llama-server.install.XXXXXX")" - cleanup_paths+=("$install_temp") - install -m 0700 "$staged_binary" "$install_temp" - mv -f -- "$install_temp" "$engine_binary" - write_engine_receipt "$receipt" "$engine_binary" "$version" "$platform" "$checksum" - fi - ok "llama-server ${version} installed and verified" - fi - if [[ -n "$binary_path" ]]; then - protect_engine_install_parents "$INSTALL_DIR" "$install_path" - fi - - if [[ -n "$binary_path" ]]; then - if [[ (-e "${BIN_DIR}/llama-server" || -L "${BIN_DIR}/llama-server") \ - && ! -L "${BIN_DIR}/llama-server" ]]; then - die "llama-server bundle cannot replace an existing non-link executable." - fi - link_temp="${BIN_DIR}/.llama-server.link.$$" - cleanup_paths+=("$link_temp") - ln -s "$engine_binary" "$link_temp" - mv -f -- "$link_temp" "${BIN_DIR}/llama-server" - fi -} - -case "${1:-stable}" in - --help|-h) - echo "Usage: ./scripts/fetch/fetch-model.sh [stable|experimental|--list]" - exit 0 - ;; - --list) - echo -e "${BOLD}Local model evaluation candidates:${NC}" - echo " stable Qwen3.5-9B Q4_K_M" - echo " experimental PrismML Bonsai 8B Q1_0" - exit 0 - ;; - stable) MODEL_COMPONENT="model-qwen3.5-9b" ;; - experimental) MODEL_COMPONENT="model-bonsai-8b-q1" ;; - *) die "Unknown candidate '$1'. Use stable, experimental, or --list." ;; -esac - -[[ -f "$MANIFEST" ]] || die "components.json is unavailable." -require_cmd python3 -require_cmd curl -require_cmd tar - -PLATFORM="$(detect_platform)" -MODEL_URL="$(component_value "$MODEL_COMPONENT" "$PLATFORM" url)" -MODEL_CHECKSUM="$(component_value "$MODEL_COMPONENT" "$PLATFORM" checksum)" -MODEL_INSTALL_PATH="$(component_value "$MODEL_COMPONENT" "$PLATFORM" install_path)" -MODEL_DESCRIPTION="$(component_value "$MODEL_COMPONENT" "$PLATFORM" description)" -validate_relative_path "$MODEL_INSTALL_PATH" -[[ "$MODEL_INSTALL_PATH" == models/*.gguf ]] || die "Model install path must be a GGUF below models/." - -echo -echo -e "${BOLD}ElastOS local model preparation${NC}" -echo " Candidate: ${MODEL_DESCRIPTION}" -echo " Platform: ${PLATFORM}" -echo - -download_verified "$MODEL_COMPONENT" "$MODEL_URL" "$MODEL_CHECKSUM" "${INSTALL_DIR}/${MODEL_INSTALL_PATH}" - -LLAMA_STRATEGY="$(python3 - "$MANIFEST" "$PLATFORM" <<'PY' -import json -import pathlib -import sys - -manifest = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) -info = manifest["external"]["llama-server"].get("platforms", {}).get(sys.argv[2], {}) -print(info.get("strategy", "prebuilt")) -PY -)" -if [[ "$LLAMA_STRATEGY" == "source-build" ]]; then - if [[ -x "${BIN_DIR}/llama-server" ]]; then - ok "Existing source-built llama-server is available" - else - die "This platform requires scripts/build/build-llama-server.sh before model evaluation." - fi -elif [[ "$LLAMA_STRATEGY" == "prebuilt" ]]; then - install_prebuilt_engine "$PLATFORM" -else - die "Unsupported llama-server strategy '${LLAMA_STRATEGY}'." -fi - -LLAMA_ENGINE_PATH="$(python3 - "${BIN_DIR}/llama-server" <<'PY' -import pathlib -import sys - -path = pathlib.Path(sys.argv[1]).resolve(strict=True) -if not path.is_file(): - raise SystemExit("llama-server does not resolve to a regular file") -print(path) -PY -)" - -echo -ok "Local model prerequisites are ready" -echo " Model: ${INSTALL_DIR}/${MODEL_INSTALL_PATH}" -echo " llama-server: ${LLAMA_ENGINE_PATH}" diff --git a/scripts/local-carrier-setup-smoke.sh b/scripts/local-carrier-setup-smoke.sh index dfa8c4112..fab233e75 100755 --- a/scripts/local-carrier-setup-smoke.sh +++ b/scripts/local-carrier-setup-smoke.sh @@ -400,9 +400,6 @@ for name, capsule_dir in browser_capsules.items(): components_dest.parent.mkdir(parents=True, exist_ok=True) components_dest.write_text(json.dumps(manifest, indent=2) + "\n") -catalog_dest = data_dir / "model-catalog.json" -catalog_dest.write_bytes(components_src.with_name("model-catalog.json").read_bytes()) -catalog_dest.chmod(0o600) PY echo "[local-carrier-setup] staged local artifacts into ${ARTIFACTS_DIR}" @@ -522,7 +519,6 @@ DATA_DIR="${XDG_DATA_HOME}/elastos" [[ ! -e "$DATA_DIR" && ! -L "$DATA_DIR" ]] mkdir -p "${DATA_DIR}/bin" cp "${PUBLISHER_DATA_DIR}/components.json" "${DATA_DIR}/components.json" -cp "${PUBLISHER_DATA_DIR}/model-catalog.json" "${DATA_DIR}/model-catalog.json" install -m 700 "${REPO_ROOT}/elastos/target/release/localhost-provider" \ "${DATA_DIR}/bin/localhost-provider" [[ ! -e "${DATA_DIR}/bin/kubo" && ! -e "${DATA_DIR}/capsules/kubo" ]] diff --git a/scripts/model-package-handoff-test.py b/scripts/model-package-handoff-test.py deleted file mode 100755 index 3784b1fa1..000000000 --- a/scripts/model-package-handoff-test.py +++ /dev/null @@ -1,496 +0,0 @@ -#!/usr/bin/env python3 -"""Behavior tests for scripts/model-package-handoff.py against pinned Kubo 0.40.1; stdlib only. - -ELASTOS_TEST_KUBO_PATH=/path/to/kubo python3 scripts/model-package-handoff-test.py -""" - -import contextlib -import hashlib -import http.client -import http.server -import importlib.util -import io -import json -import os -from pathlib import Path -import socket -import stat -import subprocess -import tempfile -import threading -import time -import unittest -from unittest import mock - -HERE = Path(__file__).resolve().parent - - -def load_sibling(name): - spec = importlib.util.spec_from_file_location(name.replace("-", "_"), HERE / f"{name}.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -helper = load_sibling("model-package-handoff") -bootstrap = load_sibling("install-bootstrap-test") - -KUBO = os.environ.get("ELASTOS_TEST_KUBO_PATH") -KUBO_SKIP = "requires explicit pinned ELASTOS_TEST_KUBO_PATH" -OTHER_DID = bootstrap.PUBLISHER_DID -CACHE_BUDGET = 7 * 2**30 -MEMORY_BUDGET = 5 * 2**30 - - -def run_helper(*argv): - out, err = io.StringIO(), io.StringIO() - with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): - try: - code = helper.main([str(arg) for arg in argv]) - except SystemExit as exit: - code = exit.code - return code, out.getvalue(), err.getvalue() - - -def sha256_file(path): - digest = hashlib.sha256() - with open(path, "rb") as source: - for chunk in iter(lambda: source.read(65536), b""): - digest.update(chunk) - return digest.hexdigest() - - -def unbound_port(): - with socket.socket() as probe: - probe.bind(("127.0.0.1", 0)) - return probe.getsockname()[1] - - -def synthetic_closure(root): - files = { - "capsule.json": b'{"name":"synthetic-model","role":"content","type":"data"}\n', - "LICENSE": b"Apache-2.0 (synthetic quantized model)\n", - "LICENSE.base": b"Apache-2.0 (synthetic base model)\n", - "PROVENANCE.md": b"# Provenance\n\nSynthetic bytes for the handoff test.\n", - "weights.gguf": b"GGUF\x03\x00\x00\x00" + bytes((index * 7919 + 13) % 256 for index in range(600 * 1024)), - } - for name, data in files.items(): - (root / name).write_bytes(data) - return sum(map(len, files.values())) - - -def signed_catalog(package_cid, published_at=1, expires_at=None): - payload = { - "schema": "elastos.model.catalog/v1", - "published_at": published_at, - "entries": [{ - "cid": package_cid, - "capsule_manifest": {"name": "synthetic-model", "role": "content", "type": "data"}, - "object_manifest": {"files": ["weights.gguf"], "publisher_did": None}, - }], - } - if expires_at is not None: - payload["expires_at"] = expires_at - signed = bootstrap.sign_envelope(payload, "elastos.model.catalog.v1") - return bootstrap.encode_envelope(signed), signed["signer_did"] - - -def fake_routes(**overrides): - routes = { - "/api/v0/version": (200, b'{"Version":"0.40.1"}\n'), - "/api/v0/config?arg=Import.UnixFSHAMTDirectorySizeThreshold": - (200, b'{"Key":"Import.UnixFSHAMTDirectorySizeThreshold","Value":null}\n'), - "/api/v0/config?arg=Import.UnixFSHAMTDirectorySizeEstimation": - (200, b'{"Key":"Import.UnixFSHAMTDirectorySizeEstimation","Value":null}\n'), - } - routes.update(overrides) - return routes - - -class FakeKuboHandler(http.server.BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self): - answer = self.server.routes.get(self.path) - if callable(answer): - answer(self) - return - status, body = answer or (404, b'{"Message":"no such route","Type":"error"}\n') - self.send_response(status) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, *_): - pass - - -@contextlib.contextmanager -def fake_kubo(routes): - server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), FakeKuboHandler) - server.daemon_threads = True - server.routes = routes - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield f"http://127.0.0.1:{server.server_port}" - finally: - server.shutdown() - server.server_close() - thread.join() - - -class KuboDaemon: - def __init__(self, repo): - self.repo = Path(repo) - self.env = {**os.environ, "IPFS_PATH": str(self.repo)} - self.process = None - self.port = None - self.api_url = None - - def run(self, *args): - completed = subprocess.run([KUBO, *map(str, args)], env=self.env, check=True, capture_output=True, text=True) - return completed.stdout.strip() - - def start(self): - self.repo.mkdir() - self.run("init", "--empty-repo", "--profile=test") - self.run("config", "Addresses.API", "/ip4/127.0.0.1/tcp/0") - self.run("config", "Addresses.Gateway", "/ip4/127.0.0.1/tcp/0") - self.run("config", "--json", "Addresses.Swarm", "[]") - log_path = self.repo.parent / f"{self.repo.name}.log" - with open(log_path, "wb") as log: - self.process = subprocess.Popen([KUBO, "daemon", "--offline"], env=self.env, stdout=log, stderr=subprocess.STDOUT) - deadline = time.monotonic() + 30 - while time.monotonic() < deadline: - if self.process.poll() is not None: - raise RuntimeError(f"kubo daemon exited early:\n{log_path.read_text()}") - api_file = self.repo / "api" - if api_file.exists() and self._answers(int(api_file.read_text().strip().rsplit("/", 1)[1])): - return self - time.sleep(0.05) - raise RuntimeError(f"kubo daemon did not become ready:\n{log_path.read_text()}") - - def _answers(self, port): - connection = http.client.HTTPConnection("127.0.0.1", port, timeout=2) - try: - connection.request("POST", "/api/v0/version") - if connection.getresponse().status != 200: - return False - except OSError: - return False - finally: - connection.close() - self.port, self.api_url = port, f"http://127.0.0.1:{port}" - return True - - def stop(self): - if self.process is None or self.process.poll() is not None: - return - self.process.terminate() - try: - self.process.wait(timeout=15) - except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait() - - -class OfflineTest(unittest.TestCase): - def test_head_cid_matches_recorded_kubo_vector(self): - # Recorded from `kubo add --cid-version=1 --raw-leaves -Q -n` on Kubo 0.40.1 for these exact bytes. - self.assertEqual(helper.raw_cid(b'{"a":1}\n'), "bafkreihdizbsainqif4vddmwctzvmdgnoe2uutxbaho4xcj5nfm2tvrqdq") - - -@unittest.skipUnless(KUBO and os.access(KUBO, os.X_OK), KUBO_SKIP) -class KuboHandoffTest(unittest.TestCase): - def setUp(self): - scratch = tempfile.TemporaryDirectory(prefix="model-package-handoff-") - self.addCleanup(scratch.cleanup) - self.root = Path(scratch.name) - self.sender = KuboDaemon(self.root / "sender").start() - self.addCleanup(self.sender.stop) - self.receiver = KuboDaemon(self.root / "receiver").start() - self.addCleanup(self.receiver.stop) - self.closure = self.root / "closure" - self.closure.mkdir() - self.payload_bytes = synthetic_closure(self.closure) - self.cid = self.sender.run("add", "-r", "--cid-version=1", "--raw-leaves", "--chunker=size-262144", "-Q", self.closure) - self.assertTrue(self.cid.startswith("bafybei"), self.cid) - self.other_cid = self.cid[:-1] + ("a" if self.cid[-1] != "a" else "b") - - def export_package(self): - car = self.root / "package.car" - code, out, err = run_helper("export", "--cid", self.cid, "--output", car, "--kubo-api", self.sender.api_url) - self.assertEqual((code, err), (0, "")) - return car, self.root / "package.car.receipt.json", json.loads(out) - - def dag_stat(self, node): - return json.loads(node.run("dag", "stat", "--enc=json", "--progress=false", self.cid))["DagStats"][0] - - def recursive_pins(self, node): - return sorted(node.run("pin", "ls", "--type=recursive").splitlines()) - - def receiver_data_dir(self): - data_dir = self.root / "receiver-data" - data_dir.mkdir(mode=0o700) - components = { - "schema": "elastos.components/v1", - "capsules": {"shell": {"cid": "", "sha256": "", "size": 0, "platforms": ["x86_64-linux"]}}, - "external": { - "kubo": {"version": "0.40.1", "platforms": {"x86_64-linux": { - "url": "https://example.invalid/kubo.tar.gz", "checksum": "sha256:" + "a" * 64, "extract_path": "kubo/ipfs"}}}, - "llama-server": {"version": "b1", "platforms": {}}, - "carrier": {"version": "1", "platforms": {}}, - }, - "profiles": {"default": {"external": ["kubo"], "capsules": ["shell"]}}, - "model_catalog": {"head_cid": "bafkreiold", "publisher_dids": ["did:key:z6MkOld"]}, - } - (data_dir / "components.json").write_bytes(json.dumps(components, indent=2).encode() + b"\n") - return data_dir, components - - def pin_catalog(self, data_dir, catalog_path, head, did, package_cid=None): - return run_helper( - "pin-catalog", "--data-dir", data_dir, "--catalog", catalog_path, "--head-cid", head, - "--publisher-did", did, "--publisher-did", OTHER_DID, "--package-cid", package_cid or self.cid, - "--max-cache-bytes", CACHE_BUDGET, "--max-model-memory-bytes", MEMORY_BUDGET) - - @staticmethod - def snapshot(data_dir): - return {path.name: path.read_bytes() for path in data_dir.iterdir()} - - def test_verify_kubo_accepts_pinned_version_and_canonical_profile(self): - code, out, err = run_helper("verify-kubo", "--kubo-api", self.sender.api_url) - self.assertEqual((code, err), (0, "")) - self.assertEqual(json.loads(out), {"kubo_version": "0.40.1", "import_profile": "canonical"}) - - data_dir = self.root / "sender-data" - data_dir.mkdir() - coords = {"kubo_pid": self.sender.process.pid, "api_port": self.sender.port, "gateway_port": 0, "started_at": 0, "last_used": 0} - (data_dir / "ipfs-coords.json").write_text(json.dumps(coords)) - code, out, err = run_helper("verify-kubo", "--data-dir", data_dir) - self.assertEqual((code, err, json.loads(out)["kubo_version"]), (0, "", "0.40.1")) - - with fake_kubo(fake_routes(**{"/api/v0/version": (200, b'{"Version":"0.39.0"}\n')})) as url: - code, out, err = run_helper("verify-kubo", "--kubo-api", url) - self.assertEqual((code, out, err), (1, "", "error: kubo version '0.39.0' is not 0.40.1\n")) - - threshold = "/api/v0/config?arg=Import.UnixFSHAMTDirectorySizeThreshold" - with fake_kubo(fake_routes(**{threshold: (200, b'{"Key":"Import.UnixFSHAMTDirectorySizeThreshold","Value":"1MiB"}\n')})) as url: - code, out, err = run_helper("verify-kubo", "--kubo-api", url) - self.assertEqual((code, out), (1, "")) - self.assertIn("Import.UnixFSHAMTDirectorySizeThreshold='1MiB' is not canonical", err) - - def test_export_streams_car_with_receipt_and_verify_car_refuses_corruption(self): - car = self.root / "package.car" - receipt_path = self.root / "package.car.receipt.json" - reads = [] - original_read = helper.TrailerResponse.read - - def counting_read(response, amt=None): - reads.append(amt) - return original_read(response, amt) - - with mock.patch.object(helper, "CHUNK_BYTES", 4096), mock.patch.object(helper.TrailerResponse, "read", counting_read): - code, out, err = run_helper("export", "--cid", self.cid, "--output", car, "--kubo-api", self.sender.api_url) - self.assertEqual((code, err), (0, "")) - receipt = json.loads(out) - size = car.stat().st_size - self.assertEqual(receipt, { - "schema": "elastos.model.package-car/v1", "package_cid": self.cid, "car_sha256": sha256_file(car), - "car_bytes": size, "kubo_version": "0.40.1", "exported_at": receipt["exported_at"]}) - self.assertGreater(size, self.payload_bytes) - self.assertGreaterEqual(reads.count(4096), size // 4096, "the CAR body was not read in 4 KiB chunks") - self.assertEqual(json.loads(receipt_path.read_bytes()), receipt) - self.assertFalse((self.root / "package.car.partial").exists()) - code, out, err = run_helper("verify-car", "--car", car, "--receipt", receipt_path, "--cid", self.cid) - self.assertEqual((code, err, json.loads(out)), (0, "", receipt)) - - truncated = self.root / "truncated.car" - truncated.write_bytes(car.read_bytes()[:-1000]) - flipped_bytes = bytearray(car.read_bytes()) - flipped_bytes[size // 2] ^= 0xFF - flipped = self.root / "flipped.car" - flipped.write_bytes(flipped_bytes) - other_receipt = self.root / "other.receipt.json" - other_receipt.write_text(json.dumps({**receipt, "package_cid": self.other_cid})) - dead_kubo = f"http://127.0.0.1:{unbound_port()}" - cases = [ - ("truncated", truncated, receipt_path, f"CAR size {size - 1000} does not match receipt car_bytes {size}"), - ("flipped byte", flipped, receipt_path, "does not match receipt car_sha256"), - ("other receipt", car, other_receipt, f"receipt package_cid {self.other_cid} does not match expected CID {self.cid}"), - ] - for label, car_path, receipt_file, expected in cases: - with self.subTest(label): - code, out, err = run_helper("verify-car", "--car", car_path, "--receipt", receipt_file, "--cid", self.cid) - self.assertEqual((code, out), (1, "")) - self.assertIn(expected, err) - code, out, err = run_helper( - "import", "--car", car_path, "--receipt", receipt_file, "--cid", self.cid, - "--free-space-floor-bytes", 0, "--kubo-api", dead_kubo) - self.assertEqual((code, out), (1, "")) - self.assertIn(expected, err) - self.assertNotIn("unreachable", err, "import contacted Kubo before the CAR gate") - - def failing_export(handler): - handler.send_response(200) - handler.send_header("Transfer-Encoding", "chunked") - handler.send_header("Trailer", "X-Stream-Error") - handler.end_headers() - handler.wfile.write(b"5\r\nCARv1\r\n0\r\nX-Stream-Error: block bafkreimissing not found\r\n\r\n") - - with fake_kubo(fake_routes(**{f"/api/v0/dag/export?arg={self.cid}": failing_export})) as url: - code, out, err = run_helper("export", "--cid", self.cid, "--output", self.root / "failed.car", "--kubo-api", url) - self.assertEqual((code, out), (1, "")) - self.assertIn("failed mid-stream: block bafkreimissing not found", err) - self.assertEqual(sorted(path.name for path in self.root.glob("failed.car*")), []) - - def test_import_pins_complete_root_on_receiver_and_is_idempotent(self): - car, receipt_path, _ = self.export_package() - code, out, err = run_helper("verify-kubo", "--kubo-api", self.receiver.api_url) - self.assertEqual((code, err), (0, "")) - base = ["import", "--car", car, "--receipt", receipt_path, "--cid", self.cid, "--kubo-api", self.receiver.api_url] - - code, out, err = run_helper(*base, "--free-space-floor-bytes", 2**62) - self.assertEqual((code, out), (1, "")) - self.assertIn(f"leave less than the {2**62}-byte --free-space-floor-bytes floor", err) - self.assertNotIn(f"{self.cid} recursive", self.recursive_pins(self.receiver)) - - code, out, err = run_helper(*base, "--free-space-floor-bytes", 0) - self.assertEqual((code, err), (0, "")) - first = json.loads(out) - sender_stat = self.dag_stat(self.sender) - self.assertEqual(first, { - "schema": "elastos.model.package-import/v1", "package_cid": self.cid, "pinned": True, - "blocks": sender_stat["NumBlocks"], "block_bytes": sender_stat["Size"], "kubo_version": "0.40.1"}) - pins = self.recursive_pins(self.receiver) - self.assertIn(f"{self.cid} recursive", pins) - self.assertEqual(self.dag_stat(self.receiver), sender_stat) - fetched = self.root / "fetched" - self.receiver.run("get", "-o", fetched, self.cid) - self.assertEqual(sorted(path.name for path in fetched.iterdir()), sorted(path.name for path in self.closure.iterdir())) - for path in self.closure.iterdir(): - self.assertEqual((fetched / path.name).read_bytes(), path.read_bytes(), path.name) - - code, out, err = run_helper(*base, "--free-space-floor-bytes", 0) - self.assertEqual((code, err, json.loads(out)), (0, "", first)) - self.assertEqual(self.recursive_pins(self.receiver), pins) - - def test_pin_catalog_patches_only_model_catalog_and_refuses_timed_snapshots(self): - data_dir, components = self.receiver_data_dir() - catalog, did = signed_catalog(self.cid) - catalog_path = self.root / "model-catalog.json" - catalog_path.write_bytes(catalog) - head = helper.raw_cid(catalog) - self.assertEqual(head, self.sender.run("add", "--cid-version=1", "--raw-leaves", "-Q", "-n", catalog_path)) - code, out, err = run_helper("head-cid", "--catalog", catalog_path) - self.assertEqual((code, err, json.loads(out)), (0, "", {"head_cid": head, "catalog_bytes": len(catalog)})) - - code, out, err = self.pin_catalog(data_dir, catalog_path, head, did) - self.assertEqual((code, err), (0, "")) - expected_pin = {"head_cid": head, "publisher_dids": [did, OTHER_DID], - "local_use": {"max_cache_bytes": CACHE_BUDGET, "max_model_memory_bytes": MEMORY_BUDGET}} - self.assertEqual(json.loads(out), { - "schema": "elastos.model.catalog-pin/v1", "head_cid": head, "package_cid": self.cid, - **expected_pin, "signature_verification": "pending_runtime_read"}) - patched = json.loads((data_dir / "components.json").read_bytes()) - self.assertEqual(list(patched), list(components)) - self.assertEqual({key: value for key, value in patched.items() if key != "model_catalog"}, - {key: value for key, value in components.items() if key != "model_catalog"}) - self.assertEqual(patched["model_catalog"], expected_pin) - self.assertEqual((data_dir / "model-catalog.json").read_bytes(), catalog) - for name in ("model-catalog.json", "components.json"): - info = (data_dir / name).stat() - self.assertEqual((stat.S_IMODE(info.st_mode), info.st_nlink), (0o600, 1), name) - snapshot = self.snapshot(data_dir) - self.assertEqual(sorted(snapshot), ["components.json", "model-catalog.json"]) - - timed, _ = signed_catalog(self.cid, expires_at=4000000000) - timed_path = self.root / "timed-catalog.json" - timed_path.write_bytes(timed) - refusals = [ - ("timed snapshot", (timed_path, helper.raw_cid(timed), did, None), - "expires_at=4000000000 is a timed snapshot; this helper pins permanent snapshots only"), - ("wrong head", (catalog_path, helper.raw_cid(b"other bytes"), did, None), f"catalog head is {head}, not --head-cid"), - ("entry cid", (catalog_path, head, did, self.other_cid), f"does not match --package-cid {self.other_cid}"), - ("unknown signer", (catalog_path, head, "did:key:z6MkUnknownSigner", None), "is not among the given --publisher-did values"), - ] - for label, arguments, expected in refusals: - with self.subTest(label): - code, out, err = self.pin_catalog(data_dir, *arguments) - self.assertEqual((code, out), (1, "")) - self.assertIn(expected, err) - self.assertEqual(self.snapshot(data_dir), snapshot) - data_dir.chmod(0o770) - try: - code, out, err = self.pin_catalog(data_dir, catalog_path, head, did) - finally: - data_dir.chmod(0o700) - self.assertEqual((code, out), (1, "")) - self.assertIn("not writable by group or others", err) - self.assertEqual(self.snapshot(data_dir), snapshot) - - def test_two_pins_stay_separate(self): - car, receipt_path, _ = self.export_package() - code, out, err = run_helper( - "import", "--car", car, "--receipt", receipt_path, "--cid", self.cid, - "--free-space-floor-bytes", 0, "--kubo-api", self.receiver.api_url) - self.assertEqual((code, err), (0, "")) - data_dir, _ = self.receiver_data_dir() - components_path = data_dir / "components.json" - catalog, did = signed_catalog(self.cid) - catalog_path = self.root / "model-catalog.json" - catalog_path.write_bytes(catalog) - code, out, err = self.pin_catalog(data_dir, catalog_path, helper.raw_cid(catalog), did) - self.assertEqual((code, err), (0, "")) - pinned = json.loads(components_path.read_bytes())["model_catalog"] - - self.receiver.run("pin", "rm", self.cid) - pins = self.recursive_pins(self.receiver) - self.assertNotIn(f"{self.cid} recursive", pins) - self.assertEqual(json.loads(components_path.read_bytes())["model_catalog"], pinned) - - second, _ = signed_catalog(self.cid, published_at=2) - second_path = self.root / "second-catalog.json" - second_path.write_bytes(second) - code, out, err = self.pin_catalog(data_dir, second_path, helper.raw_cid(second), did) - self.assertEqual((code, err), (0, "")) - self.assertEqual(json.loads(components_path.read_bytes())["model_catalog"]["head_cid"], helper.raw_cid(second)) - self.assertNotEqual(helper.raw_cid(second), pinned["head_cid"]) - self.assertEqual(self.recursive_pins(self.receiver), pins) - - -class RepoCatalogPinTests(unittest.TestCase): - def test_repo_catalog_matches_components_pin(self): - root = HERE.parent - catalog = (root / "model-catalog.json").read_bytes() - components = json.loads((root / "components.json").read_bytes()) - pin = components["model_catalog"] - self.assertEqual(helper.raw_cid(catalog), pin["head_cid"]) - signed = json.loads(catalog) - self.assertNotIn("expires_at", signed["payload"]) - self.assertEqual(len(signed["payload"]["entries"]), 2) - self.assertEqual( - signed["payload"]["entries"][0]["cid"], - "bafybeid5l7gfgsqy2wozia2q7mtyux2wrbnlfehzz4at3ic3cngvyku6hi", - ) - self.assertEqual( - signed["payload"]["entries"][1]["cid"], - "bafybeidy5kfvqwg6g6pfgdfwslmhijosbeskt5b2duqdqxnc7e6fwmr72y", - ) - self.assertIn(signed["signer_did"], pin["publisher_dids"]) - helper.check_catalog( - catalog, - signed["payload"]["entries"][0]["cid"], - pin["publisher_dids"], - ) - helper.check_catalog( - catalog, - signed["payload"]["entries"][1]["cid"], - pin["publisher_dids"], - ) - self.assertGreaterEqual(pin["local_use"]["max_cache_bytes"], 18516684377) - self.assertGreaterEqual(pin["local_use"]["max_model_memory_bytes"], 8589934592) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/scripts/model-package-handoff.py b/scripts/model-package-handoff.py deleted file mode 100755 index 7ff7712e6..000000000 --- a/scripts/model-package-handoff.py +++ /dev/null @@ -1,555 +0,0 @@ -#!/usr/bin/env python3 -"""Hand a signed model content package from one Runtime-managed Kubo to another. - -`export` streams the package closure out of the sender's Kubo as a CAR with a -receipt. `verify-car` and `import` replay that closure into the receiver's Kubo -and confirm the recursive retention pin. `pin-catalog` installs the sender's -signed permanent catalog snapshot and points `components.json.model_catalog` -at it without verifying the signature; the Runtime verifies the catalog -signature when it reads the snapshot, so the receipt reports that as pending. -The retention pin and the catalog trust pin are separate and stay separate. -Stdlib only. Every command prints one JSON receipt on success and one -`error: ...` line on refusal. -""" - -import argparse -import base64 -import contextlib -import dataclasses -import hashlib -import http.client -import json -import os -import re -import shutil -import stat -import sys -import tempfile -import time -import urllib.parse - -KUBO_VERSION = "0.40.1" -CHUNK_BYTES = 1024 * 1024 -MAX_CONTROL_BYTES = 64 * 1024 -MAX_CATALOG_BYTES = 128 * 1024 -MAX_COMPONENTS_BYTES = 4 * 1024 * 1024 -MAX_BUDGET = 2**63 - 1 -CATALOG_SCHEMA = "elastos.model.catalog/v1" -RECEIPT_SCHEMA = "elastos.model.package-car/v1" -IMPORT_SCHEMA = "elastos.model.package-import/v1" -CATALOG_PIN_SCHEMA = "elastos.model.catalog-pin/v1" -BASE32_ALPHABET = frozenset("abcdefghijklmnopqrstuvwxyz234567") - -# Kubo 0.40.1 config/import.go treats null as exactly these defaults; the -# Runtime's directory_hash.rs accepts the same spellings and nothing else. -IMPORT_PROFILE = { - "Import.UnixFSHAMTDirectorySizeThreshold": lambda value: value is None - or (isinstance(value, int) and not isinstance(value, bool) and value == 262144) - or value in ("256KiB", "262144"), - "Import.UnixFSHAMTDirectorySizeEstimation": lambda value: value is None or value == "links", -} - - -class Refusal(Exception): - pass - - -@dataclasses.dataclass(frozen=True) -class CarDigest: - sha256: str - size: int - - -@dataclasses.dataclass(frozen=True) -class ImportStats: - blocks: int - block_bytes: int - - -@dataclasses.dataclass(frozen=True) -class Receipt: - package_cid: str - car_sha256: str - car_bytes: int - kubo_version: str - exported_at: int - - def to_json(self): - return {"schema": RECEIPT_SCHEMA, **dataclasses.asdict(self)} - - @classmethod - def load(cls, path): - try: - record = json.loads(read_bounded(path, MAX_CONTROL_BYTES)) - except ValueError: - raise Refusal(f"receipt {path} is not JSON") - fields = {field.name: field.type for field in dataclasses.fields(cls)} - if ( - not isinstance(record, dict) - or record.get("schema") != RECEIPT_SCHEMA - or set(record) != {"schema", *fields} - or any(not isinstance(record[name], kind) or isinstance(record[name], bool) for name, kind in fields.items()) - or record["car_bytes"] <= 0 - or len(record["car_sha256"]) != 64 - ): - raise Refusal(f"receipt {path} is not a {RECEIPT_SCHEMA} record") - return cls(**{name: record[name] for name in fields}) - - -@dataclasses.dataclass(frozen=True) -class CatalogPin: - head_cid: str - publisher_dids: tuple - max_cache_bytes: int - max_model_memory_bytes: int - - def to_json(self): - return { - "head_cid": self.head_cid, - "publisher_dids": list(self.publisher_dids), - "local_use": { - "max_cache_bytes": self.max_cache_bytes, - "max_model_memory_bytes": self.max_model_memory_bytes, - }, - } - - -class TrailerResponse(http.client.HTTPResponse): - """http.client discards chunked trailers; Kubo reports a mid-stream failure only in X-Stream-Error.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.trailers = {} - - def _read_and_discard_trailer(self): - while True: - line = self.fp.readline(65537) - if len(line) > 65536: - raise http.client.LineTooLong("trailer line") - if not line or line in (b"\r\n", b"\n"): - return - name, _, value = line.decode("latin-1").partition(":") - self.trailers[name.strip()] = value.strip() - - -class KuboApi: - def __init__(self, base_url, timeout): - match = re.fullmatch(r"http://([^/:]+):([1-9][0-9]{0,4})/?", base_url) - if not match: - raise Refusal(f"--kubo-api must look like http://127.0.0.1:PORT, got {base_url!r}") - self.host, self.port, self.timeout = match.group(1), int(match.group(2)), timeout - self.base_url = f"http://{self.host}:{self.port}" - - @classmethod - def from_data_dir(cls, data_dir, timeout): - coords_path = os.path.join(data_dir, "ipfs-coords.json") - try: - coords = json.loads(read_bounded(coords_path, MAX_CONTROL_BYTES)) - port = coords["api_port"] - except (OSError, ValueError, KeyError, TypeError): - raise Refusal(f"{coords_path} does not name the Runtime Kubo api_port; is the Runtime's Kubo running?") - return cls(f"http://127.0.0.1:{port}", timeout) - - def _connect(self): - connection = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) - connection.response_class = TrailerResponse - return connection - - def _open(self, operation, query=None): - target = "/api/v0/" + operation + ("?" + urllib.parse.urlencode(query) if query else "") - connection = self._connect() - try: - connection.request("POST", target) - return connection, connection.getresponse() - except OSError as error: - connection.close() - raise Refusal(f"kubo api {self.base_url} unreachable: {error}") - - def _json(self, operation, query=None): - connection, response = self._open(operation, query) - with contextlib.closing(connection): - body = response.read(MAX_CONTROL_BYTES + 1) - text = body[:300].decode("utf-8", "replace").strip() - if response.status != 200 or len(body) > MAX_CONTROL_BYTES: - raise Refusal(f"kubo {operation} answered {response.status}: {text}") - try: - return json.loads(body) - except ValueError: - raise Refusal(f"kubo {operation} returned invalid JSON: {text}") - - def verify_profile(self): - version = self._json("version").get("Version") - if version != KUBO_VERSION: - raise Refusal(f"kubo version {version!r} is not {KUBO_VERSION}") - for key, accepts in IMPORT_PROFILE.items(): - answer = self._json("config", {"arg": key}) - if answer.get("Key") != key or "Value" not in answer or not accepts(answer["Value"]): - raise Refusal(f"kubo import profile {key}={answer.get('Value')!r} is not canonical") - return version - - def export_car(self, cid, destination): - connection, response = self._open("dag/export", {"arg": cid}) - with contextlib.closing(connection): - if response.status != 200: - text = response.read(MAX_CONTROL_BYTES)[:300].decode("utf-8", "replace").strip() - raise Refusal(f"kubo dag/export {cid} answered {response.status}: {text}") - digest = hashlib.sha256() - size = 0 - with open(destination, "wb") as sink: - for chunk in iter(lambda: response.read(CHUNK_BYTES), b""): - sink.write(chunk) - digest.update(chunk) - size += len(chunk) - sink.flush() - os.fsync(sink.fileno()) - if response.trailers.get("X-Stream-Error"): - raise Refusal(f"kubo dag/export {cid} failed mid-stream: {response.trailers['X-Stream-Error']}") - if size == 0: - raise Refusal(f"kubo dag/export {cid} returned an empty CAR") - return CarDigest(digest.hexdigest(), size) - - def repo_path(self): - path = self._json("repo/stat").get("RepoPath") - if not isinstance(path, str) or not path: - raise Refusal("kubo repo/stat did not report RepoPath") - return path - - def import_car(self, car_path, cid): - boundary = "elastos-model-package-" + os.urandom(12).hex() - head = ( - f"--{boundary}\r\n" - 'Content-Disposition: form-data; name="file"; filename="package.car"\r\n' - "Content-Type: application/vnd.ipld.car\r\n\r\n" - ).encode("ascii") - foot = f"\r\n--{boundary}--\r\n".encode("ascii") - size = os.path.getsize(car_path) - connection = self._connect() - try: - connection.putrequest("POST", "/api/v0/dag/import?pin-roots=true&stats=true") - connection.putheader("Content-Type", f"multipart/form-data; boundary={boundary}") - connection.putheader("Content-Length", str(len(head) + size + len(foot))) - connection.endheaders() - connection.send(head) - sent = 0 - with open(car_path, "rb") as source: - for chunk in iter(lambda: source.read(CHUNK_BYTES), b""): - connection.send(chunk) - sent += len(chunk) - if sent != size: - raise Refusal(f"{car_path} changed size during upload") - connection.send(foot) - response = connection.getresponse() - body = response.read(MAX_CONTROL_BYTES + 1) - except OSError as error: - raise Refusal(f"kubo dag/import upload to {self.base_url} failed: {error}") - finally: - connection.close() - return parse_import_events(response.status, body, response.trailers, cid) - - def is_recursively_pinned(self, cid): - connection, response = self._open("pin/ls", {"arg": cid, "type": "recursive"}) - with contextlib.closing(connection): - body = response.read(MAX_CONTROL_BYTES + 1) - if response.status != 200 or len(body) > MAX_CONTROL_BYTES: - return False - try: - keys = json.loads(body).get("Keys", {}) - except (ValueError, AttributeError): - return False - return isinstance(keys, dict) and keys.get(cid, {}).get("Type") == "recursive" - - -def parse_import_events(status, body, trailers, cid): - text = body.decode("utf-8", "replace") - if status != 200 or len(body) > MAX_CONTROL_BYTES: - raise Refusal(f"kubo dag/import answered {status}: {text.strip()[:300]}") - if trailers.get("X-Stream-Error"): - raise Refusal(f"kubo dag/import failed mid-stream: {trailers['X-Stream-Error']}") - root = stats = None - for line in filter(None, text.splitlines()): - try: - event = json.loads(line) - except ValueError: - raise Refusal(f"kubo dag/import returned a non-JSON line: {line[:300]}") - if not isinstance(event, dict) or "Message" in event or "Type" in event: - raise Refusal(f"kubo dag/import reported an error: {line[:300]}") - if root is None and isinstance(event.get("Root"), dict): - root = event["Root"] - elif stats is None and isinstance(event.get("Stats"), dict): - stats = event["Stats"] - else: - raise Refusal(f"kubo dag/import returned an unexpected event: {line[:300]}") - if root is None or stats is None: - raise Refusal("kubo dag/import did not report exactly one root and its stats") - imported = root.get("Cid", {}).get("/") if isinstance(root.get("Cid"), dict) else None - if imported != cid: - raise Refusal(f"kubo dag/import root {imported!r} is not the expected package {cid}") - if root.get("PinErrorMsg"): - raise Refusal(f"kubo could not pin {cid}: {root['PinErrorMsg']}") - try: - return ImportStats(int(stats["BlockCount"]), int(stats["BlockBytesCount"])) - except (KeyError, TypeError, ValueError): - raise Refusal("kubo dag/import stats lack BlockCount and BlockBytesCount") - - -def raw_cid(data): - multihash = b"\x12\x20" + hashlib.sha256(data).digest() - return "b" + base64.b32encode(b"\x01\x55" + multihash).decode("ascii").lower().rstrip("=") - - -def digest_file(path): - digest = hashlib.sha256() - size = 0 - with open(path, "rb") as source: - for chunk in iter(lambda: source.read(CHUNK_BYTES), b""): - digest.update(chunk) - size += len(chunk) - return CarDigest(digest.hexdigest(), size) - - -def read_bounded(path, limit): - with open(path, "rb") as source: - data = source.read(limit + 1) - if len(data) > limit: - raise Refusal(f"{path} exceeds its {limit}-byte bound") - return data - - -def check_private_dir(path): - info = os.lstat(path) - if not stat.S_ISDIR(info.st_mode): - raise Refusal(f"{path} must be a real directory, not a symlink or file") - if info.st_uid != os.geteuid() or info.st_mode & 0o022: - raise Refusal(f"{path} must be owned by the current user and not writable by group or others") - - -def write_private(path, data): - handle, temp_path = tempfile.mkstemp(prefix=".handoff-", dir=os.path.dirname(path) or ".") - try: - with os.fdopen(handle, "wb") as sink: - os.fchmod(handle, 0o600) - sink.write(data) - sink.flush() - os.fsync(sink.fileno()) - os.replace(temp_path, path) - except BaseException: - if os.path.exists(temp_path): - os.unlink(temp_path) - raise - - -def check_car(car_path, receipt_path, cid): - receipt = Receipt.load(receipt_path) - if receipt.package_cid != cid: - raise Refusal(f"receipt package_cid {receipt.package_cid} does not match expected CID {cid}") - size = os.path.getsize(car_path) - if size != receipt.car_bytes: - raise Refusal(f"CAR size {size} does not match receipt car_bytes {receipt.car_bytes}") - digest = digest_file(car_path) - if digest.sha256 != receipt.car_sha256: - raise Refusal(f"CAR sha256 {digest.sha256} does not match receipt car_sha256 {receipt.car_sha256}") - return receipt - - -def check_catalog(catalog, package_cid, publisher_dids): - try: - signed = json.loads(catalog) - except ValueError: - raise Refusal("catalog is not JSON") - if not isinstance(signed, dict) or set(signed) != {"payload", "signature", "signer_did"}: - raise Refusal("catalog must be a {payload, signature, signer_did} envelope") - payload = signed["payload"] - if not isinstance(payload, dict) or payload.get("schema") != CATALOG_SCHEMA: - raise Refusal(f"catalog payload.schema must be {CATALOG_SCHEMA}") - if payload.get("expires_at") is not None: - raise Refusal( - f"catalog expires_at={payload['expires_at']} is a timed snapshot; this helper pins permanent" - " snapshots only (sign a payload that omits expires_at)" - ) - entries = payload.get("entries") - if not isinstance(entries, list) or not (1 <= len(entries) <= 8): - raise Refusal("catalog must carry 1 to 8 entries") - if any(not isinstance(entry, dict) for entry in entries): - raise Refusal("catalog entries must be objects") - cids = [entry.get("cid") for entry in entries] - if len(set(cids)) != len(cids): - raise Refusal("catalog entries must use unique package CIDs") - if package_cid not in cids: - raise Refusal(f"catalog entries do not include --package-cid {package_cid}") - if signed["signer_did"] not in publisher_dids: - raise Refusal(f"catalog signer_did {signed['signer_did']!r} is not among the given --publisher-did values") - - -def package_cid_argument(value): - if not (value.startswith("bafy") and 8 <= len(value) <= 128 and set(value) <= BASE32_ALPHABET): - raise argparse.ArgumentTypeError(f"{value!r} is not a base32 CIDv1 starting with bafy") - return value - - -def bounded_int(minimum): - def parse(value): - number = int(value) - if not minimum <= number <= MAX_BUDGET: - raise argparse.ArgumentTypeError(f"{value} is outside {minimum}..{MAX_BUDGET}") - return number - - return parse - - -def kubo_from_args(args): - if args.kubo_api: - return KuboApi(args.kubo_api, args.timeout_seconds) - return KuboApi.from_data_dir(args.data_dir, args.timeout_seconds) - - -def cmd_verify_kubo(args): - return {"kubo_version": kubo_from_args(args).verify_profile(), "import_profile": "canonical"} - - -def cmd_head_cid(args): - catalog = read_bounded(args.catalog, MAX_CATALOG_BYTES) - return {"head_cid": raw_cid(catalog), "catalog_bytes": len(catalog)} - - -def cmd_export(args): - kubo = kubo_from_args(args) - version = kubo.verify_profile() - partial = args.output + ".partial" - try: - digest = kubo.export_car(args.cid, partial) - except BaseException: - if os.path.exists(partial): - os.unlink(partial) - raise - os.replace(partial, args.output) - receipt = Receipt(args.cid, digest.sha256, digest.size, version, int(time.time())) - with open(args.output + ".receipt.json", "wb") as sink: - sink.write(json.dumps(receipt.to_json(), indent=2).encode("utf-8") + b"\n") - return receipt.to_json() - - -def cmd_verify_car(args): - return check_car(args.car, args.receipt, args.cid).to_json() - - -def cmd_import(args): - receipt = check_car(args.car, args.receipt, args.cid) - kubo = kubo_from_args(args) - version = kubo.verify_profile() - repo_path = kubo.repo_path() - try: - free = shutil.disk_usage(repo_path).free - except OSError as error: - raise Refusal(f"cannot measure free space at kubo repo {repo_path}: {error}") - if free - receipt.car_bytes < args.free_space_floor_bytes: - raise Refusal( - f"kubo repo {repo_path} has {free} free bytes; importing {receipt.car_bytes} bytes would" - f" leave less than the {args.free_space_floor_bytes}-byte --free-space-floor-bytes floor" - ) - stats = kubo.import_car(args.car, args.cid) - if not kubo.is_recursively_pinned(args.cid): - raise Refusal(f"kubo imported {args.cid} but does not list it as a recursive pin") - return { - "schema": IMPORT_SCHEMA, - "package_cid": args.cid, - "pinned": True, - "blocks": stats.blocks, - "block_bytes": stats.block_bytes, - "kubo_version": version, - } - - -def cmd_pin_catalog(args): - dids = args.publisher_dids - if len(set(dids)) != len(dids) or len(dids) > 8 or any(not did.startswith("did:key:z") or len(did) > 128 for did in dids): - raise Refusal("--publisher-did values must be unique did:key:z... identifiers, at most eight") - check_private_dir(args.data_dir) - catalog = read_bounded(args.catalog, MAX_CATALOG_BYTES) - head = raw_cid(catalog) - if head != args.head_cid: - raise Refusal(f"catalog head is {head}, not --head-cid {args.head_cid}") - check_catalog(catalog, args.package_cid, args.publisher_dids) - pin = CatalogPin(head, tuple(args.publisher_dids), args.max_cache_bytes, args.max_model_memory_bytes) - components_path = os.path.join(args.data_dir, "components.json") - try: - components = json.loads(read_bounded(components_path, MAX_COMPONENTS_BYTES)) - except ValueError: - raise Refusal(f"{components_path} is not JSON") - if not isinstance(components, dict): - raise Refusal(f"{components_path} must be a JSON object") - components["model_catalog"] = pin.to_json() - write_private(os.path.join(args.data_dir, "model-catalog.json"), catalog) - write_private(components_path, json.dumps(components, indent=2).encode("utf-8") + b"\n") - return { - "schema": CATALOG_PIN_SCHEMA, - "head_cid": head, - "package_cid": args.package_cid, - **pin.to_json(), - "signature_verification": "pending_runtime_read", - } - - -def add_kubo_arguments(parser): - target = parser.add_mutually_exclusive_group(required=True) - target.add_argument("--kubo-api", metavar="URL", help="Kubo HTTP API, e.g. http://127.0.0.1:5001") - target.add_argument("--data-dir", metavar="DIR", help="Runtime data dir holding ipfs-coords.json") - parser.add_argument("--timeout-seconds", type=float, default=60.0, help="per-request and per-chunk idle timeout") - - -def build_parser(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - commands = parser.add_subparsers(dest="command", required=True) - - verify_kubo = commands.add_parser("verify-kubo", help="check the pinned Kubo version and canonical import profile") - add_kubo_arguments(verify_kubo) - verify_kubo.set_defaults(run=cmd_verify_kubo) - - head_cid = commands.add_parser("head-cid", help="print the raw CIDv1 SHA-256 of a catalog file") - head_cid.add_argument("--catalog", required=True) - head_cid.set_defaults(run=cmd_head_cid) - - export = commands.add_parser("export", help="stream a package closure to FILE.car with a receipt") - export.add_argument("--cid", required=True, type=package_cid_argument) - export.add_argument("--output", required=True, metavar="FILE.car") - add_kubo_arguments(export) - export.set_defaults(run=cmd_export) - - verify_car = commands.add_parser("verify-car", help="check a CAR against its receipt and expected CID") - verify_car.add_argument("--car", required=True) - verify_car.add_argument("--receipt", required=True) - verify_car.add_argument("--cid", required=True, type=package_cid_argument) - verify_car.set_defaults(run=cmd_verify_car) - - import_ = commands.add_parser("import", help="verify, import and pin a CAR in the receiver's Kubo") - import_.add_argument("--car", required=True) - import_.add_argument("--receipt", required=True) - import_.add_argument("--cid", required=True, type=package_cid_argument) - import_.add_argument("--free-space-floor-bytes", required=True, type=bounded_int(0)) - add_kubo_arguments(import_) - import_.set_defaults(run=cmd_import) - - pin_catalog = commands.add_parser("pin-catalog", help="install a permanent catalog snapshot and pin its head") - pin_catalog.add_argument("--data-dir", required=True) - pin_catalog.add_argument("--catalog", required=True, metavar="model-catalog.json") - pin_catalog.add_argument("--head-cid", required=True) - pin_catalog.add_argument("--publisher-did", dest="publisher_dids", action="append", required=True, metavar="DID") - pin_catalog.add_argument("--package-cid", required=True, type=package_cid_argument) - pin_catalog.add_argument("--max-cache-bytes", required=True, type=bounded_int(1)) - pin_catalog.add_argument("--max-model-memory-bytes", required=True, type=bounded_int(1)) - pin_catalog.set_defaults(run=cmd_pin_catalog) - return parser - - -def main(argv=None): - args = build_parser().parse_args(argv) - try: - receipt = args.run(args) - except (Refusal, OSError) as error: - print(f"error: {error}", file=sys.stderr) - return 1 - print(json.dumps(receipt)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/prepare-release-platform-test.py b/scripts/prepare-release-platform-test.py index 839c5f39c..3ad78fbc9 100644 --- a/scripts/prepare-release-platform-test.py +++ b/scripts/prepare-release-platform-test.py @@ -131,7 +131,7 @@ def setUp(self): for name in ( "publish-release.sh", "prepare-release-platform.sh", "release-platform-input.py", "components-release-integrity-check.py", "check-versioning.sh", "build-media-tools.sh", - "release-upstream-assets.py", "release-upstream-input.py", "model-package-handoff.py", + "release-upstream-assets.py", "release-upstream-input.py", ): (scripts / name).write_bytes((SOURCE / "scripts" / name).read_bytes()) (scripts / name).chmod(0o755) @@ -467,118 +467,6 @@ def test_arm64_rejects_unpinned_engine_before_build(self): self.assertFalse(output.exists()) self.assertFalse((self.root / "cargo.log").exists()) - def model_preparation_env(self): - return {**self.env, 'ELASTOS_RELEASE_MODEL_KUBO_BIN': str(self.root / 'fixture-kubo'), - 'ELASTOS_RELEASE_MODEL_KUBO_REPO': str(self.root / 'fixture-kubo-repo'), - 'ELASTOS_RELEASE_MODEL_PUBLISHED_AT': '1800000000', - 'ELASTOS_RELEASE_MODEL_PUBLISHER_DID': 'did:key:z6MkFixture', - 'ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT': str(self.root / 'model-handoff')} - - def test_model_preparation_refuses_partial_env_and_bad_handoff_before_build(self): - complete = self.model_preparation_env() - for name in [key for key in complete if key.startswith('ELASTOS_RELEASE_MODEL_')]: - with self.subTest(missing=name): - partial = dict(complete) - partial.pop(name) - output, result = self.prepare(env=partial) - self.assertNotEqual(result.returncode, 0) - self.assertIn('Model preparation requires', result.stderr) - self.assertFalse(output.exists()) - self.assertFalse((self.root / 'cargo.log').exists()) - for handoff in (self.root / 'prepared', self.root / 'prepared/nested'): - with self.subTest(handoff=handoff): - output, result = self.prepare(env={**complete, 'ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT': str(handoff)}) - self.assertNotEqual(result.returncode, 0) - self.assertIn('new sibling', result.stderr) - self.assertFalse(output.exists()) - self.assertFalse((self.root / 'cargo.log').exists()) - (self.root / 'model-handoff').mkdir() - _, result = self.prepare(env=complete) - self.assertNotEqual(result.returncode, 0) - self.assertIn('new sibling', result.stderr) - _, result = self.prepare(env=complete, reuse_support=self.root / 'unused-support') - self.assertNotEqual(result.returncode, 0) - self.assertIn('fresh support', result.stderr) - - def test_model_handoff_reuse_selects_one_input_and_refuses_mixed_export_mode(self): - env = {**self.env, 'ELASTOS_RELEASE_MODEL_PUBLISHED_AT': '1800000000', - 'ELASTOS_RELEASE_MODEL_PUBLISHER_DID': 'did:key:z6MkFixture', - 'ELASTOS_RELEASE_MODEL_HANDOFF_INPUT': str(self.root / 'shared-models'), - 'ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT': str(self.root / 'native-models')} - command = ('/bin/bash', '-c', 'source scripts/publish-release.sh; release_model_preparation_enabled') - enabled = self.command(*command, env=env) - self.assertEqual(enabled.returncode, 0, enabled.stderr) - self.assertEqual(enabled.stdout.strip(), 'true') - for variable in ('KUBO_BIN', 'KUBO_REPO'): - with self.subTest(variable=variable): - refused = self.command(*command, env={**env, 'ELASTOS_RELEASE_MODEL_' + variable: 'fixture'}) - self.assertNotEqual(refused.returncode, 0) - self.assertIn('without KUBO_BIN or KUBO_REPO', refused.stderr) - - def test_optional_model_wiring_retains_unsigned_handoff_outside_native_artifacts(self): - # Exercise the native worker and real upstream capsule packaging. Only - # the expensive Kubo/model retention operation is a local inert stub. - driver = self.repo / 'scripts/release-upstream-assets.py' - driver.rename(driver.with_name('release-upstream-assets-real.py')) - driver.write_text(r'''#!/usr/bin/env python3 -import hashlib, json, os, pathlib, subprocess, sys -args = sys.argv[1:] -flags = ('--model-kubo-bin', '--model-kubo-repo', '--published-at', '--model-publisher-did', '--model-handoff-output') -assert all(args.count(flag) == 1 for flag in flags) -values = {flag: args[args.index(flag) + 1] for flag in flags} -for flag, variable in zip(flags, ('KUBO_BIN', 'KUBO_REPO', 'PUBLISHED_AT', 'PUBLISHER_DID', 'HANDOFF_OUTPUT')): - assert values[flag] == os.environ['ELASTOS_RELEASE_MODEL_' + variable] -baseline = [value for index, value in enumerate(args) if value not in flags and (index == 0 or args[index - 1] not in flags)] -reply = subprocess.check_output([sys.executable, str(pathlib.Path(__file__).with_name('release-upstream-assets-real.py')), *baseline], text=True) -result = json.loads(reply) -output = pathlib.Path(args[args.index('--output') + 1]) -handoff = pathlib.Path(values['--model-handoff-output']) -handoff.mkdir(mode=0o700) -def write(name, content): - path = handoff / name - path.write_bytes(content) - path.chmod(0o600) - return {'release_path': name, 'checksum': 'sha256:' + hashlib.sha256(content).hexdigest(), 'size': len(content)} -retention = {} -for index, name in enumerate(('model-qwen3.5-0.8b', 'model-qwen3.5-4b', 'model-qwen3.5-9b', 'model-bonsai-8b-q1')): - retention[name] = {'package_cid': 'bfixture' + str(index), - 'car': write(name + '.car', ('inert CAR ' + name).encode()), - 'receipt': write(name + '.car.receipt.json', b'{"fixture":true}\n')} -catalogue = write('model-catalog.unsigned.json', json.dumps({'schema': 'elastos.model.catalog/v1', - 'published_at': int(values['--published-at']), 'entries': []}).encode()) -catalogue['publisher_did'] = values['--model-publisher-did'] -result.update(model_retention=retention, model_catalog_unsigned=catalogue) -receipt_path = output / 'upstream-input.json' -receipt = json.loads(receipt_path.read_bytes()) -receipt.update(model_retention=retention, model_catalog_unsigned=catalogue) -receipt_path.write_text(json.dumps(receipt) + '\n') -print(json.dumps(result)) -''') - driver.chmod(0o755) - self.commit('inert model retention wiring fixture') - output, result = self.prepare(env=self.model_preparation_env()) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - handoff = self.root / 'model-handoff' - self.assertEqual(len(list(handoff.iterdir())), 9) - provenance = json.loads((output / 'model-handoff.json').read_bytes()) - self.assertEqual(provenance['schema'], 'elastos.release-model-handoff/v1') - self.assertEqual(provenance['handoff_directory'], '../model-handoff') - receipt = json.loads((output / 'platform-input.json').read_bytes()) - self.assertEqual(provenance['source'], {key: receipt['source'][key] for key in ('commit', 'tree', 'clean')}) - self.assertEqual(receipt['files']['model-handoff.json']['sha256'], hashlib.sha256((output / 'model-handoff.json').read_bytes()).hexdigest()) - self.assertEqual(provenance['upstream_input_sha256'], hashlib.sha256((output / 'upstream-input.json').read_bytes()).hexdigest()) - for name, info in provenance['files'].items(): - self.assertEqual(info['checksum'], 'sha256:' + hashlib.sha256((handoff / name).read_bytes()).hexdigest()) - self.assertEqual(info['size'], (handoff / name).stat().st_size) - self.assertEqual((handoff / name).stat().st_mode & 0o077, 0) - self.assertFalse((output / 'artifacts' / name).exists()) - manifest = json.loads((output / 'components.json').read_bytes()) - self.assertNotIn('model_retention', manifest) - self.assertNotIn('model_catalog_unsigned', manifest) - self.assertFalse(any('model_retention' in component for component in manifest['external'].values())) - checked = self.command('python3', 'scripts/release-platform-input.py', 'verify', str(output)) - self.assertEqual(checked.returncode, 0, checked.stderr) - def test_success_uses_real_packaging_and_receipt(self): output, result = self.prepare() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) diff --git a/scripts/prepare-release-platform.sh b/scripts/prepare-release-platform.sh index 1faa2da2d..6fb7e6bf7 100755 --- a/scripts/prepare-release-platform.sh +++ b/scripts/prepare-release-platform.sh @@ -15,13 +15,6 @@ checkout. DIR must be absent. Use a directory outside the checkout, or one whose temporary sibling is ignored by Git. CARGO_TARGET_DIR selects the native build cache; otherwise Cargo resolves it. CARGO_BUILD_JOBS defaults to 4 (max 4). The output contains artifacts/, draft components.json and platform-input.json. -Optional key-free model preparation uses ELASTOS_RELEASE_MODEL_* inputs: -Fresh export uses KUBO_BIN, KUBO_REPO, PUBLISHED_AT, PUBLISHER_DID and HANDOFF_OUTPUT. -Other native workers use HANDOFF_INPUT with PUBLISHED_AT, PUBLISHER_DID and -HANDOFF_OUTPUT, reusing all nine public bytes from that first export. HANDOFF_OUTPUT -must name a new protected sibling of DIR. Its CARs, receipts and unsigned catalogue -remain outside artifacts/ and temporary cleanup; model-handoff.json binds them to -this native input. The custodian finalizes and signs the catalogue separately. Generic provider microVM rootfs and Browser substrate acceptance are separate. Fresh Linux ARM64 support requires ELASTOS_LLAMA_ARM64_BUNDLE to name the reviewed b10516 archive. The archive is verified against its build recipe before any build. @@ -51,9 +44,6 @@ while [[ $# -gt 0 ]]; do esac done [[ -n "$VERSION" && -n "$OUTPUT" ]] || { usage >&2; exit 2; } -MODEL_PREPARATION=$(release_model_preparation_enabled) -[[ "$MODEL_PREPARATION" != true || -z "$REUSE_SUPPORT" ]] \ - || die "Model preparation requires fresh support; --reuse-support preserves the qualified input" for tool in git python3 jq cargo rustc rustup tar; do command -v "$tool" >/dev/null 2>&1 || die "Required tool not found: $tool" done @@ -67,26 +57,6 @@ path = Path(os.path.abspath(os.path.join(sys.argv[1], sys.argv[2]))) print(path.parent.resolve() / path.name) PY ) -if [[ "$MODEL_PREPARATION" == true ]]; then - ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT=$(python3 - "$CALLER_DIR" "$ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT" "$OUTPUT" <<'PY' -import os, sys -from pathlib import Path -path = Path(os.path.abspath(os.path.join(sys.argv[1], sys.argv[2]))) -output = Path(sys.argv[3]) -if path.parent != output.parent or path == output or path.exists() or path.is_symlink(): - raise SystemExit('Model handoff must be a new sibling of the native output') -if any(parent.is_symlink() for parent in path.parents): - raise SystemExit('Model handoff path must use its canonical parent') -if not path.parent.is_dir(): - raise SystemExit('Model handoff parent must already exist') -metadata = path.parent.stat() -if metadata.st_uid != os.geteuid() or metadata.st_mode & 0o022: - raise SystemExit('Model handoff parent must be owned and protected') -print(path) -PY -) - export ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT -fi if [[ -n "$REUSE_SUPPORT" ]]; then REUSE_SUPPORT=$(python3 - "$CALLER_DIR" "$REUSE_SUPPORT" <<'PY' import os, sys @@ -278,58 +248,11 @@ for asset in "$TMPDIR/supported-assets-$PLATFORM"/* \ [[ ! -L "$asset" ]] || die "Prepared artifact must be a regular owned file: $asset" destination="$STAGING/artifacts/$(basename "$asset")" [[ ! -e "$destination" && ! -L "$destination" ]] || die "Prepared artifact names must be distinct" - # Build output and stage share the owned WORK_DIR volume. Rename the files - # so staging does not allocate a second copy of the full model payloads. + # Build output and stage share a volume; avoid copying the payloads. mv "$asset" "$destination" done cp "$TMPDIR/supported-upstream-assets-$PLATFORM/upstream-input.json" "$STAGING/upstream-input.json" cp scripts/release-upstream-recipes.json "$STAGING/upstream-recipes.json" -if [[ "$MODEL_PREPARATION" == true ]]; then - # Bind the retained proposal to clean source inputs. Consumer components - # remain unchanged until the separate signed catalogue finalization phase. - python3 - "$STAGING" "$ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT" "$SOURCE_COMMIT" "$SOURCE_TREE" "$PLATFORM" "$VERSION" <<'PY' -import hashlib, json, os -from pathlib import Path -import stat, sys - -stage, handoff = Path(sys.argv[1]), Path(sys.argv[2]) -metadata = handoff.lstat() -if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077: - raise SystemExit('Model handoff directory must be protected and owned') -document = json.loads((stage / 'upstream-input.json').read_bytes()) -catalogue, retention = document['model_catalog_unsigned'], document['model_retention'] -names = [catalogue['release_path'], *(record[kind]['release_path'] - for record in retention.values() for kind in ('car', 'receipt'))] -files = {} -for name in names: - if not isinstance(name, str) or Path(name).name != name or name in files: - raise SystemExit('Model handoff file names must be distinct basenames') - path = handoff / name - metadata = path.lstat() - if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077: - raise SystemExit('Model handoff files must be protected owned regular files') - with path.open('rb') as stream: - value = hashlib.sha256() - for chunk in iter(lambda: stream.read(1024 * 1024), b''): - value.update(chunk) - checksum = 'sha256:' + value.hexdigest() - files[name] = {'checksum': checksum, 'size': metadata.st_size} -expected = {catalogue['release_path']: catalogue, **{record[kind]['release_path']: record[kind] - for record in retention.values() for kind in ('car', 'receipt')}} -if any(files[name] != {key: expected[name][key] for key in ('checksum', 'size')} for name in files): - raise SystemExit('Model handoff bytes differ from upstream preparation receipts') -if {path.name for path in handoff.iterdir()} != set(files): - raise SystemExit('Model handoff inventory differs from preparation receipts') -record = {'schema': 'elastos.release-model-handoff/v1', - 'scope': 'key-free model preparation; signed catalogue finalization follows', - 'source': {'commit': sys.argv[3], 'tree': sys.argv[4], 'clean': True}, - 'platform': sys.argv[5], 'version': sys.argv[6], - 'handoff_directory': '../' + handoff.name, - 'upstream_input_sha256': hashlib.sha256((stage / 'upstream-input.json').read_bytes()).hexdigest(), - 'model_catalog_unsigned': catalogue, 'model_retention': retention, 'files': files} -(stage / 'model-handoff.json').write_text(json.dumps(record, indent=2, sort_keys=True) + '\n') -PY -fi if [[ -f "$SOURCE_ROOT/model-catalog.json" ]]; then cp "$SOURCE_ROOT/model-catalog.json" "$STAGING/artifacts/model-catalog.json" fi diff --git a/scripts/publish-platform-artifacts-test.py b/scripts/publish-platform-artifacts-test.py index 4ab535b40..f6a629eed 100644 --- a/scripts/publish-platform-artifacts-test.py +++ b/scripts/publish-platform-artifacts-test.py @@ -575,45 +575,6 @@ def test_publication_export_promotes_exact_advertised_set_head_last(self): artifacts = sorted(name for name in expected if name.startswith("artifacts/")) self.assertEqual(order, artifacts + ["install.sh", "release.json", "publish-state.json", "release-head.json"]) - def test_publication_export_retains_model_car_files_and_refuses_tampering(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - prepared, publisher = root / 'prepared', root / 'publisher' - publication_fixture(prepared) - retention = {} - for model in ('model-qwen3.5-0.8b', 'model-qwen3.5-4b', 'model-qwen3.5-9b', 'model-bonsai-8b-q1'): - entry = {'package_cid': 'fixture-model-root'} - for kind, suffix in (('car', '.car'), ('receipt', '.car.receipt.json')): - name = model + suffix - data = ('inert retention fixture ' + name).encode() - (prepared / 'artifacts' / name).write_bytes(data) - entry[kind] = {'release_path': name, 'cid': 'fixture-file-cid', - 'checksum': 'sha256:' + sha256(data), 'size': len(data)} - retention[model] = entry - for platform in RELEASE_PLATFORMS: - path = prepared / 'artifacts' / f'components-{platform}.json' - manifest = json.loads(path.read_bytes()) - manifest['model_retention'] = retention - data = json.dumps(manifest).encode() - path.write_bytes(data) - rewrite_release(prepared, lambda release, p=platform, b=data: - release['payload']['platforms'][p]['components'].update(sha256=sha256(b), size=len(b))) - result = self.run_publication_export(prepared, publisher) - self.assertEqual(result.returncode, 0, result.stderr) - before = snapshot(publisher) - for model in retention.values(): - for kind in ('car', 'receipt'): - name = model[kind]['release_path'] - self.assertEqual((publisher / 'artifacts' / name).read_bytes(), - (prepared / 'artifacts' / name).read_bytes()) - first = prepared / 'artifacts' / retention['model-qwen3.5-0.8b']['car']['release_path'] - original = first.read_bytes() - first.write_bytes(b'!' * len(original)) - result = self.run_publication_export(prepared, publisher) - self.assertNotEqual(result.returncode, 0) - self.assertIn('prepared model retention bytes differ', result.stderr) - self.assertEqual(snapshot(publisher), before) - def test_publication_export_rejects_incomplete_or_mismatched_sets_before_mutation(self): def replace(name, data): return lambda prepared: (prepared / name).write_bytes(data) diff --git a/scripts/publish-release.sh b/scripts/publish-release.sh index 3b941c272..5c9b40da7 100755 --- a/scripts/publish-release.sh +++ b/scripts/publish-release.sh @@ -968,62 +968,14 @@ build_platform_independent_provider_capsule_metadata_assets() { # Builders above return unsigned descriptors with no placeholder CIDs. # Upstream software is fetched only by this build worker. Installed clients use # the same signed release-path/CID contract as every first-party component. -release_model_preparation_enabled() { - local name count=0 - for name in ELASTOS_RELEASE_MODEL_KUBO_BIN ELASTOS_RELEASE_MODEL_KUBO_REPO \ - ELASTOS_RELEASE_MODEL_PUBLISHED_AT ELASTOS_RELEASE_MODEL_PUBLISHER_DID \ - ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT ELASTOS_RELEASE_MODEL_HANDOFF_INPUT; do - [[ -z "${!name:-}" ]] || count=$((count + 1)) - done - if [[ "$count" == 0 ]]; then echo false; return; fi - if [[ -z "${ELASTOS_RELEASE_MODEL_PUBLISHED_AT:-}" || \ - -z "${ELASTOS_RELEASE_MODEL_PUBLISHER_DID:-}" || \ - -z "${ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT:-}" ]]; then - die "Model preparation requires PUBLISHED_AT, PUBLISHER_DID and HANDOFF_OUTPUT in ELASTOS_RELEASE_MODEL_*" - fi - if [[ -n "${ELASTOS_RELEASE_MODEL_HANDOFF_INPUT:-}" ]]; then - [[ "$count" == 4 ]] || die "Model preparation requires shared HANDOFF_INPUT without KUBO_BIN or KUBO_REPO" - else - [[ "$count" == 5 ]] || die "Model preparation requires KUBO_BIN and KUBO_REPO for the first authoritative handoff" - fi - echo true -} - build_upstream_direct_assets() { local platform="$1" setup_platform="$2" local output="$TMPDIR/supported-upstream-assets-$platform" local cache="${ELASTOS_RELEASE_UPSTREAM_CACHE:-$TMPDIR/upstream-cache}" - local models local args=() - models=$(release_model_preparation_enabled) || return if [[ -n "${ELASTOS_LLAMA_ARM64_BUNDLE:-}" ]]; then args+=(--llama-arm64-bundle "$ELASTOS_LLAMA_ARM64_BUNDLE") fi - if [[ "$models" == true ]]; then - # The explicit operator handoff survives temporary build cleanup. This - # phase signs nothing; public catalogue finalization follows separately. - python3 - "$ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT" "$TMPDIR" <<'PY' || return -from pathlib import Path -import os, sys -path, temporary = Path(sys.argv[1]), Path(sys.argv[2]).resolve() -if not path.is_absolute() or any(parent.is_symlink() for parent in (path, *path.parents)): - raise SystemExit('Model handoff requires an absolute path without symlinks') -if path.exists() or not path.parent.is_dir() or temporary == path or temporary in path.parents: - raise SystemExit('Model handoff requires a new directory outside temporary build output') -metadata = path.parent.stat() -if metadata.st_uid != os.geteuid() or metadata.st_mode & 0o022: - raise SystemExit('Model handoff parent must be owned and protected') -PY - args+=(--published-at "$ELASTOS_RELEASE_MODEL_PUBLISHED_AT" - --model-publisher-did "$ELASTOS_RELEASE_MODEL_PUBLISHER_DID" - --model-handoff-output "$ELASTOS_RELEASE_MODEL_HANDOFF_OUTPUT") - if [[ -n "${ELASTOS_RELEASE_MODEL_HANDOFF_INPUT:-}" ]]; then - args+=(--model-handoff-input "$ELASTOS_RELEASE_MODEL_HANDOFF_INPUT") - else - args+=(--model-kubo-bin "$ELASTOS_RELEASE_MODEL_KUBO_BIN" - --model-kubo-repo "$ELASTOS_RELEASE_MODEL_KUBO_REPO") - fi - fi python3 scripts/release-upstream-assets.py --platform "$setup_platform" \ --cache "$cache" --output "$output" ${args[@]+"${args[@]}"} } @@ -1314,29 +1266,6 @@ try: for name, entry in (manifest.get("capsules") or {}).items(): if isinstance(entry, dict) and platform in (entry.get("platforms") or []): referenced.add(f"{name}-{platform}.capsule.tar.gz") - retention = manifest.get("model_retention") - if retention is not None: - if not isinstance(retention, dict) or len(retention) != 4: - raise ValueError("prepared model_retention requires four models") - for model in retention.values(): - if not isinstance(model, dict): - raise ValueError("prepared model retention must be an object") - for kind in ("car", "receipt"): - info = model.get(kind) - if not isinstance(info, dict): - raise ValueError(f"prepared model retention {kind} descriptor is required") - name = info.get("release_path") - if name not in present or not isinstance(info.get("cid"), str) or not info["cid"]: - raise ValueError(f"prepared model retention {kind} artifact/CID is required") - path = present[name] - sha = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - sha.update(chunk) - if (type(info.get("size")) is not int or info["size"] != path.stat().st_size - or info.get("checksum") != "sha256:" + sha.hexdigest()): - raise ValueError(f"prepared model retention bytes differ: {name}") - referenced.add(name) if errors: raise ValueError("; ".join(errors)) extra = sorted(set(present) - referenced) diff --git a/scripts/release-platform-input-test.py b/scripts/release-platform-input-test.py index dd7fcf4aa..4401e7f1e 100644 --- a/scripts/release-platform-input-test.py +++ b/scripts/release-platform-input-test.py @@ -1538,307 +1538,6 @@ def test_release_refuses_url_only_dependency_even_with_source_pin(self): self.assertTrue(any("URL-only" in error for error in inputs.integrity.audit_release_artifacts(manifest, ["darwin-arm64"], root / "artifacts"))) -class ModelFinalizationTest(unittest.TestCase): - write_json = PlatformInputTest.write_json - refresh = PlatformInputTest.refresh - make_bundle = PlatformInputTest.make_bundle - - def setUp(self): - PlatformInputTest.setUp(self) - self.root = self.root.resolve() - self.bundles = {name: path.resolve() for name, path in self.bundles.items()} - candidates = [shutil.which("openssl"), "/opt/homebrew/opt/openssl@3/bin/openssl"] - self.openssl = None - for candidate in candidates: - if candidate and Path(candidate).exists(): - path = Path(candidate).resolve() - result = subprocess.run([str(path), "version"], capture_output=True) - if result.returncode == 0 and result.stdout.startswith(b"OpenSSL 3."): - self.openssl = path - break - if self.openssl is None: - self.skipTest("public Ed25519 fixture requires OpenSSL 3") - self.native, recipe, previous = UpstreamCapsuleAdmissionTest.upstream_fixture(self, model=True) - self.native.chmod(0o700) - (self.native / "artifacts" / previous["release_path"]).unlink() - script = Path(__file__).with_name("release-upstream-input.py") - spec = importlib.util.spec_from_file_location("model_finalization_upstream_fixture", script) - helper = importlib.util.module_from_spec(spec) - spec.loader.exec_module(helper) - source = self.root / "source" - recipes, capsules = [], [] - template = json.loads((self.native / "components-template.json").read_bytes()) - manifest = json.loads((self.native / "components.json").read_bytes()) - del template["external"]["model-fixture"] - del manifest["external"]["model-fixture"] - for name in sorted(inputs.MODEL_COMPONENTS): - item = copy.deepcopy(recipe) - item.update(component=name, root=name, extract_path=name + "/fixture.gguf") - resolved = copy.deepcopy(item) - for record in [resolved["source"], *(notice["source"] for notice in resolved["license"]["files"]), - *(notice["source"] for notice in resolved.get("notices", []))]: - record["path"] = str(source / record["path"]) - receipt = helper.package(resolved, self.root / "upstream-cache", self.native / "artifacts") - receipt["recipe_sha256"] = hashlib.sha256(inputs.canonical(item)).hexdigest() - recipes.append(item) - capsules.append(receipt) - template["external"][name] = {"platforms": {"darwin-arm64": {"strategy": "source-build", - "release_path": receipt["release_path"], "install_path": item["install_path"], "extract_path": item["extract_path"]}}} - manifest["external"][name] = {"role": "content", "platforms": {"darwin-arm64": { - key: receipt[key] for key in ("release_path", "checksum", "size", "extract_path", "install_path")}}, - "capsule_metadata": {"role": "content", "type": "data", "install_path": "capsules/" + name, - "platforms": {"darwin-arm64": receipt["capsule_metadata"]}}} - template["external"][name]["role"] = "content" - inventory = {"schema": "elastos.release-upstream-recipes/v1", "recipes": recipes} - self.write_json(source / "scripts/release-upstream-recipes.json", inventory) - self.write_json(self.native / "upstream-recipes.json", inventory) - self.write_json(source / "components.json", template) - self.write_json(self.native / "components-template.json", template) - self.write_json(self.native / "components.json", manifest) - self.upstream = {"schema": "elastos.release-upstream-assets/v1", "platform": "darwin-arm64", - "recipes_sha256": inputs.digest(self.native / "upstream-recipes.json"), "capsules": capsules} - self.model_root = self.native.parent / "model-handoff" - self.model_root.mkdir(mode=0o700) - # Disposable signing material belongs only to this test's private, - # automatically cleaned directory. Tool output never enters test logs. - self.key = self.root / "fixture-key.pem" - self.crypto(["genpkey", "-algorithm", "ED25519", "-out", str(self.key)]) - self.key.chmod(0o600) - public = self.crypto(["pkey", "-in", str(self.key), "-pubout", "-outform", "DER"]) - self.did = signer.public_did(public[12:]) - self.payload = {"schema": "elastos.model.catalog/v1", "published_at": 1, "expires_at": None, "entries": []} - retention = {} - _, handoff = inputs.model_tools() - for capsule in capsules: - name = capsule["component"] - cid = "b" + base64.b32encode(b"\x01\x70\x12\x20" + hashlib.sha256(name.encode()).digest()).decode().lower().rstrip("=") - self.payload["entries"].append({"cid": cid, "capsule_manifest": copy.deepcopy(capsule["capsule_manifest"]), - "object_manifest": copy.deepcopy(capsule["object_manifest"])}) - car = self.model_root / (name + ".car") - car.write_bytes(b"inert CAR byte-binding fixture " + name.encode()) - receipt = self.model_root / (name + ".car.receipt.json") - receipt.write_bytes(signer.json_bytes({"schema": handoff.RECEIPT_SCHEMA, "package_cid": cid, - "car_sha256": inputs.digest(car), "car_bytes": car.stat().st_size, "kubo_version": handoff.KUBO_VERSION, - "exported_at": 1})) - retention[name] = {"package_cid": cid, **{kind: self.descriptor(path) for kind, path in (("car", car), ("receipt", receipt))}} - draft = self.model_root / "model-catalog.unsigned.json" - draft.write_bytes(signer.json_bytes(self.payload)) - self.record = {"schema": "elastos.release-model-handoff/v1", - "scope": "key-free model preparation; signed catalogue finalization follows", - "source": {"commit": "a" * 40, "tree": "b" * 40, "clean": True}, "platform": "aarch64-darwin", "version": "0.7.1", - "handoff_directory": "../model-handoff", "model_catalog_unsigned": {**self.descriptor(draft), "publisher_did": self.did}, - "model_retention": retention} - self.bind_handoff() - self.catalogue = self.root / "public-catalogue.json" - self.sign() - self.args = SimpleNamespace(input=self.native, handoff=self.model_root, catalogue=self.catalogue, - publisher_did=self.did, openssl=self.openssl, output=self.root / "finalized") - - def crypto(self, arguments): - result = subprocess.run([str(self.openssl), *arguments], capture_output=True, - env={"OPENSSL_CONF": "/dev/null", "LANG": "C"}, timeout=20) - if result.returncode: - self.fail("disposable public signature fixture operation failed") - return result.stdout - - def descriptor(self, path): - return {"release_path": path.name, "checksum": "sha256:" + inputs.digest(path), "size": path.stat().st_size} - - def bind_handoff(self): - self.upstream.update({key: self.record[key] for key in ("model_catalog_unsigned", "model_retention")}) - self.write_json(self.native / "upstream-input.json", self.upstream) - self.record["upstream_input_sha256"] = inputs.digest(self.native / "upstream-input.json") - descriptors = [self.record["model_catalog_unsigned"], *(item[kind] for item in self.record["model_retention"].values() - for kind in ("car", "receipt"))] - self.record["files"] = {info["release_path"]: {key: info[key] for key in ("checksum", "size")} for info in descriptors} - self.write_json(self.native / "model-handoff.json", self.record) - for path in self.model_root.iterdir(): - path.chmod(0o600) - self.refresh(self.native) - - def sign(self, domain=b"elastos.model.catalog.v1"): - message = self.root / "fixture-digest" - signature = self.root / "fixture-signature" - message.write_bytes(hashlib.sha256(domain + b"\0" + signer.json_bytes(self.payload)).digest()) - self.crypto(["pkeyutl", "-sign", "-rawin", "-inkey", str(self.key), "-in", str(message), "-out", str(signature)]) - self.catalogue.write_bytes(signer.json_bytes({"payload": self.payload, "signature": signature.read_bytes().hex(), "signer_did": self.did})) - self.catalogue.chmod(0o600) - - def refused(self, pattern): - with self.assertRaisesRegex((ValueError, OSError), pattern): - inputs.finalize_models(self.args) - self.assertFalse(self.args.output.exists()) - self.assertFalse(list(self.root.glob(".finalize-models-*"))) - self.assertFalse(list(self.root.glob(".model-public-verify-*"))) - - def test_public_signature_and_retention_finalize_preserves_native_source(self): - before = (self.native / "platform-input.json").read_bytes() - with patch.object(inputs, "source_identity", side_effect=AssertionError("finalization must not rebuild")): - receipt = inputs.finalize_models(self.args) - self.assertEqual((self.native / "platform-input.json").read_bytes(), before) - self.assertEqual((self.args.output / "model-native-input.json").read_bytes(), before) - self.assertEqual(inputs.verify(self.args.output), receipt) - manifest = json.loads((self.args.output / "components.json").read_bytes()) - self.assertEqual(manifest["model_catalog"], {"head_cid": inputs.catalog_head_cid(self.catalogue.read_bytes()), "publisher_dids": [self.did]}) - self.assertEqual(manifest["model_retention"], self.record["model_retention"]) - self.assertTrue(all("cid" not in item[kind] for item in manifest["model_retention"].values() for kind in ("car", "receipt"))) - self.assertEqual(len(list((self.args.output / "artifacts").glob("*.car*"))), 8) - - def test_domain_and_signature_failure_clean_public_scratch(self): - self.sign(b"elastos.release.v1") - self.refused("verification failed") - - def test_exact_public_publisher_required(self): - self.args.publisher_did = signer.public_did(bytes(32)) - self.refused("public publisher differs") - - def test_signed_payload_must_equal_worker_proposal(self): - self.payload["published_at"] = 2 - self.sign() - self.refused("payload differs") - - def test_payload_identity_preserves_json_types(self): - unsigned = copy.deepcopy(self.payload) - unsigned["published_at"] = True - draft = self.model_root / "model-catalog.unsigned.json" - draft.write_bytes(signer.json_bytes(unsigned)) - self.record["model_catalog_unsigned"].update(self.descriptor(draft)) - self.bind_handoff() - self.refused("payload differs") - - def test_public_catalogue_closure_refuses_execution_and_noncanonical_roots(self): - self.payload["entries"][0]["capsule_manifest"]["network"] = True - self.sign() - self.refused("passive catalogue") - - def test_distinct_four_directory_roots_required(self): - self.payload["entries"][1]["cid"] = self.payload["entries"][0]["cid"] - self.sign() - self.refused("distinct canonical directory") - - def test_signed_native_closure_cannot_change_under_a_valid_signature(self): - # The two closures are each valid passive objects; the reviewed native - # receipt owns which object belongs in this public catalogue. - self.payload["entries"][0]["capsule_manifest"]["model_content"]["minimum_memory_mb"] += 1 - capsule = self.payload["entries"][0]["capsule_manifest"] - index = self.payload["entries"][0]["object_manifest"] - encoded = signer.json_bytes(capsule) - for item in index["files"]: - if item["path"] == "capsule.json": - item.update(sha256=hashlib.sha256(encoded).hexdigest(), size=len(encoded)) - content = hashlib.sha256() - for item in index["files"]: - for field in (item["path"], item["sha256"], str(item["size"])): - content.update(field.encode() + b"\0") - index["content_digest"] = "sha256:" + content.hexdigest() - draft = self.model_root / "model-catalog.unsigned.json" - draft.write_bytes(signer.json_bytes(self.payload)) - self.record["model_catalog_unsigned"].update(self.descriptor(draft)) - self.bind_handoff() - self.sign() - self.refused("closure differs from native") - - def test_car_corruption_is_refused_before_copy(self): - path = next(self.model_root.glob("*.car")) - path.write_bytes(path.read_bytes() + b"changed") - self.refused("protection or bytes differ") - - def test_car_receipt_root_must_equal_catalogue(self): - item = next(iter(self.record["model_retention"].values())) - path = self.model_root / item["receipt"]["release_path"] - value = json.loads(path.read_bytes()) - value["package_cid"] = self.payload["entries"][1]["cid"] - path.write_bytes(signer.json_bytes(value)) - item["receipt"] = self.descriptor(path) - self.bind_handoff() - self.refused("does not match") - - def test_protected_handoff_and_exact_inventory_required(self): - self.model_root.chmod(0o755) - self.refused("protected owned directory") - - def test_public_native_directory_keeps_private_handoff_and_scratch(self): - self.native.chmod(0o755) - self.root.chmod(0o755) - inputs.finalize_models(self.args) - self.assertEqual(self.args.output.stat().st_mode & 0o777, 0o700) - inputs.verify(self.args.output) - - def test_extra_handoff_file_is_refused(self): - (self.model_root / "unapproved.bin").write_bytes(b"extra") - self.refused("file inventory differs") - - def test_output_collision_preserves_existing_directory(self): - self.args.output.mkdir() - with self.assertRaisesRegex(ValueError, "new output outside"): - inputs.finalize_models(self.args) - self.assertTrue(self.args.output.is_dir()) - self.assertEqual(list(self.args.output.iterdir()), []) - - def test_copy_failure_cleans_owned_output_scratch(self): - with patch.object(inputs.shutil, "copyfile", side_effect=OSError("fixture copy failure")): - self.refused("fixture copy failure") - inputs.verify(self.native) - - def test_repeated_finalization_refuses_to_rewrite_build_provenance(self): - inputs.finalize_models(self.args) - self.args.input = self.args.output - self.args.output = self.root / "second-finalization" - self.refused("original native build input") - - def test_handoff_source_binding_and_original_build_required(self): - self.record["source"]["tree"] = "d" * 40 - self.bind_handoff() - self.refused("source binding differs") - - def test_whole_copy_budget_precedes_any_scratch(self): - with patch.object(inputs.shutil, "disk_usage", return_value=SimpleNamespace(total=100, free=16)), \ - patch.object(inputs.tempfile, "TemporaryDirectory", side_effect=AssertionError("scratch before disk gate")): - self.refused("15 percent") - - def test_approved_budget_checks_largest_package_charge(self): - budget = {"max_cache_bytes": 1, "max_model_memory_bytes": 8 * 1024**3} - for name in ("components.json", "components-template.json"): - value = json.loads((self.native / name).read_bytes()) - value["model_catalog"] = {"head_cid": inputs.catalog_head_cid(b"old catalog"), "publisher_dids": [self.did], "local_use": budget} - self.write_json(self.native / name, value) - (self.native / "artifacts/model-catalog.json").write_bytes(b"old catalog") - self.refresh(self.native) - self.refused("approved local-use budget") - - def test_source_budget_and_previous_catalogue_bytes_are_preserved(self): - budget = {"max_cache_bytes": 64 * 1024**2, "max_model_memory_bytes": 8 * 1024**3} - for name in ("components.json", "components-template.json"): - value = json.loads((self.native / name).read_bytes()) - value["model_catalog"] = {"head_cid": inputs.catalog_head_cid(b"old catalog"), "publisher_dids": [self.did], "local_use": budget} - self.write_json(self.native / name, value) - (self.native / "artifacts/model-catalog.json").write_bytes(b"old catalog") - self.refresh(self.native) - inputs.finalize_models(self.args) - self.assertEqual((self.args.output / "model-native-catalog.json").read_bytes(), b"old catalog") - manifest = json.loads((self.args.output / "components.json").read_bytes()) - self.assertEqual(manifest["model_catalog"]["local_use"], budget) - inputs.verify(self.args.output) - - def test_valid_reused_support_input_requires_original_build(self): - reused = self.root / "reused-native" - reused.mkdir(mode=0o700) - args = SimpleNamespace(input=self.native, root=reused, platform="aarch64-darwin", version="0.7.2") - inputs.copy_support(args) - PlatformInputTest.record_reuse(self, args) - self.args.input = reused - self.refused("original native build input") - - def test_finalized_receipt_preserves_prior_native_provenance(self): - inputs.finalize_models(self.args) - path = self.args.output / "platform-input.json" - value = json.loads(path.read_bytes()) - value["source"]["tree"] = "d" * 40 - self.write_json(path, value) - with self.assertRaisesRegex(ValueError, "rewrote native build provenance"): - inputs.verify(self.args.output) - - class ReleaseSignerInputTest(unittest.TestCase): def test_custodian_refuses_hostile_inputs_without_running_candidates(self): # CI's existing release-input gate runs these public-data/fake-backend @@ -1868,7 +1567,7 @@ def load_tests(loader, tests, pattern): # Load each inert wrapper case into CI's existing entry point. The same # loader keeps unittest -k filters; explicit class/method selections retain # unittest's normal behavior and do not enter this module hook. - for name in ("release-upstream-input-test.py", "release-upstream-assets-test.py", "release-source-upstream-test.py"): + for name in ("release-upstream-input-test.py", "release-source-upstream-test.py"): sibling_spec = importlib.util.spec_from_file_location( name.replace("-", "_").removesuffix(".py"), Path(__file__).with_name(name)) sibling = importlib.util.module_from_spec(sibling_spec) diff --git a/scripts/release-platform-input.py b/scripts/release-platform-input.py index b0f26cf1d..b9937bad3 100644 --- a/scripts/release-platform-input.py +++ b/scripts/release-platform-input.py @@ -25,7 +25,6 @@ SCRIPT_ROOT = Path(__file__).resolve().parent SOURCE_ROOT = SCRIPT_ROOT.parent SCHEMA = "elastos.release-platform-input/v1" -MODEL_COMPONENTS = {"model-qwen3.5-0.8b", "model-qwen3.5-4b", "model-qwen3.5-9b", "model-bonsai-8b-q1"} PLATFORMS = { "x86_64-linux": ("linux-amd64", "x86_64-unknown-linux-musl", 62), "aarch64-linux": ("linux-arm64", "aarch64-unknown-linux-musl", 183), @@ -81,339 +80,6 @@ def admit_model_catalog_artifact(manifest, artifact_root, referenced): if actual != pin["head_cid"]: raise ValueError(f"model-catalog.json head {actual} does not match pin {pin['head_cid']}") referenced.add("model-catalog.json") - retention = manifest.get("model_retention") - if retention is not None: - signer, handoff = model_tools() - envelope = signer.parse_json(data) - if envelope.get("signer_did") not in pin.get("publisher_dids", []): - raise ValueError("catalogue publisher differs from trust pin") - roots = admit_catalog_payload(envelope.get("payload"), signer) - admit_model_budget(pin.get("local_use"), envelope["payload"]) - admit_retention(retention, roots, artifact_root, referenced, handoff) - - -def model_tools(): - """Load reviewed source helpers, independent of candidate artifact paths.""" - modules = [] - for name in ("release-signer", "model-package-handoff"): - module_name = "release_input_" + name.replace("-", "_") - if module_name not in sys.modules: - definition = importlib.util.spec_from_file_location(module_name, SCRIPT_ROOT / (name + ".py")) - module = importlib.util.module_from_spec(definition) - sys.modules[module_name] = module - definition.loader.exec_module(module) - modules.append(sys.modules[module_name]) - return modules - - -def admit_catalog_payload(payload, signer): - signer.require(type(payload) is dict and set(payload) == {"schema", "published_at", "expires_at", "entries"} - and payload["schema"] == "elastos.model.catalog/v1", "catalogue payload fields refused") - now = int(datetime.now(timezone.utc).timestamp()) - published, expiry = payload["published_at"], payload["expires_at"] - signer.require(type(published) is int and 0 <= published <= now - and (expiry is None or type(expiry) is int and expiry > now and expiry > published), - "catalogue publication or expiry refused") - signer.require(type(payload["entries"]) is list and len(payload["entries"]) == 4, - "four catalogue entries required") - roots = {} - for entry in payload["entries"]: - signer.require(type(entry) is dict and set(entry) == {"cid", "capsule_manifest", "object_manifest"}, - "catalogue entry fields refused") - cid, capsule, index = entry["cid"], entry["capsule_manifest"], entry["object_manifest"] - codec, _ = signer.cid_info(cid) - signer.require(cid.startswith("b") and codec == 0x70 and cid not in roots.values(), - "distinct canonical directory CIDs required") - signer.require(type(capsule) is dict and type(index) is dict - and capsule.get("name") in MODEL_COMPONENTS and capsule["name"] not in roots, - "catalogue component inventory differs") - signer.catalogue_closure(capsule, index) - roots[capsule["name"]] = cid - signer.require(set(roots) == MODEL_COMPONENTS, "four canonical models required") - return roots - - -def retention_descriptor(value): - if (type(value) is not dict or set(value) != {"release_path", "checksum", "size"} - or type(value["release_path"]) is not str - or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,239}", value["release_path"]) - or value["release_path"].endswith(".") - or type(value["checksum"]) is not str or not re.fullmatch(r"sha256:[0-9a-f]{64}", value["checksum"]) - or type(value["size"]) is not int or not 0 < value["size"] <= 16 * 1024**3): - raise ValueError("model retention descriptor refused") - return value - - -def admit_model_budget(budget, payload): - if budget is None: - return - if (type(budget) is not dict or set(budget) != {"max_cache_bytes", "max_model_memory_bytes"} - or any(type(value) is not int or not 0 < value < 2**63 for value in budget.values())): - raise ValueError("approved model local-use budget refused") - for entry in payload["entries"]: - total = sum(item["size"] for item in entry["object_manifest"]["files"]) - # Runtime also stages the bounded object index, which is outside its own - # closure list. Charge its actual canonical bytes with serializer slack. - total += len(json.dumps(entry["object_manifest"], sort_keys=True, separators=(",", ":")).encode()) + 1 - charge = 3 * total + 8 * 1024**2 + 3 * 65536 - memory = entry["capsule_manifest"]["model_content"]["minimum_memory_mb"] * 1024**2 - if charge > budget["max_cache_bytes"] or memory > budget["max_model_memory_bytes"]: - raise ValueError("model closure exceeds approved local-use budget") - - -def admit_retention(retention, roots, root, referenced, handoff): - if type(retention) is not dict or set(retention) != MODEL_COMPONENTS: - raise ValueError("four model retention records required") - names = set(referenced) - for name, record in retention.items(): - if (type(record) is not dict or set(record) != {"package_cid", "car", "receipt"} - or record["package_cid"] != roots[name]): - raise ValueError("model retention catalogue root differs") - paths = {} - for kind in ("car", "receipt"): - info = retention_descriptor(record[kind]) - relative = info["release_path"] - if relative in names: - raise ValueError("model retention artifact alias refused") - names.add(relative) - path = regular_file(root, relative) - if path.stat().st_size != info["size"] or "sha256:" + digest(path) != info["checksum"]: - raise ValueError("model retention artifact bytes differ") - paths[kind] = path - referenced.add(relative) - try: - receipt = handoff.check_car(str(paths["car"]), str(paths["receipt"]), record["package_cid"]) - except handoff.Refusal as exc: - raise ValueError(str(exc)) from exc - if receipt.kubo_version != handoff.KUBO_VERSION: - raise ValueError("model retention Kubo version differs") - - -def public_catalog_verification(data, publisher_did, openssl, parent): - """Verify public bytes with an explicit OpenSSL 3 executable; key access is absent.""" - signer, _ = model_tools() - envelope = signer.parse_json(data) - signer.check_did(publisher_did) - signer.require(set(envelope) == {"payload", "signature", "signer_did"} - and envelope["signer_did"] == publisher_did, "catalogue public publisher differs") - roots = admit_catalog_payload(envelope["payload"], signer) - signature = envelope["signature"] - signer.require(type(signature) is str and re.fullmatch(r"[0-9a-f]{128}", signature), - "Ed25519 catalogue signature required") - executable = Path(openssl) - signer.require(executable.is_absolute() and executable == executable.resolve(strict=True) - and os.access(executable, os.X_OK), "explicit qualified OpenSSL path required") - for path in (*executable.parents, executable): - metadata = path.lstat() - signer.require(metadata.st_uid in (0, os.geteuid()) and not metadata.st_mode & 0o022, - "OpenSSL path protection refused") - tool_sha = digest(executable) - def execute(arguments, cwd): - result = subprocess.run([str(executable), *arguments], cwd=cwd, - env={"OPENSSL_CONF": "/dev/null", "LANG": "C"}, - stdin=subprocess.DEVNULL, capture_output=True, timeout=20) - signer.require(result.returncode == 0 and len(result.stdout) <= 65536 and len(result.stderr) <= 65536, - "OpenSSL public verification failed") - return result.stdout - signer.require(execute(["version"], parent).startswith(b"OpenSSL 3."), "OpenSSL 3 required") - number = 0 - for char in publisher_did[len("did:key:z"):]: - number = number * 58 + signer.BASE58.index(char) - raw = number.to_bytes(34, "big")[2:] - with tempfile.TemporaryDirectory(prefix=".model-public-verify-", dir=parent) as temporary: - scratch = Path(temporary) - (scratch / "public.der").write_bytes(bytes.fromhex("302a300506032b6570032100") + raw) - (scratch / "digest").write_bytes(hashlib.sha256(b"elastos.model.catalog.v1\0" + signer.json_bytes(envelope["payload"])).digest()) - (scratch / "signature").write_bytes(bytes.fromhex(signature)) - execute(["pkeyutl", "-provider", "default", "-verify", "-rawin", "-pubin", "-keyform", "DER", - "-inkey", "public.der", "-in", "digest", "-sigfile", "signature"], scratch) - signer.require(digest(executable) == tool_sha, "OpenSSL executable changed") - return envelope, roots, {"path": str(executable), "sha256": tool_sha} - - -def protected_model_directory(path, private=True): - held = directory_descriptor(path) - try: - metadata = os.fstat(held) - if metadata.st_uid != os.geteuid() or metadata.st_mode & (0o077 if private else 0o022): - raise ValueError("model handoff requires a protected owned directory") - finally: - os.close(held) - - -def finalize_models(args): - signer, handoff = model_tools() - source = Path(os.path.abspath(args.input)) - protected_model_directory(source, private=False) - original = verify(source) - if "support_origin" in original or "model_finalization" in original: - raise ValueError("model finalization requires an original native build input") - record_bytes = signer.regular_bytes(regular_file(source, "model-handoff.json"), 128 * 1024) - record = signer.parse_json(record_bytes) - fields = {"schema", "scope", "source", "platform", "version", "handoff_directory", "upstream_input_sha256", - "model_catalog_unsigned", "model_retention", "files"} - if (set(record) != fields or record["schema"] != "elastos.release-model-handoff/v1" - or record["scope"] != "key-free model preparation; signed catalogue finalization follows" - or type(record["source"]) is not dict or record["source"].get("clean") is not True - or record["source"] != {key: original["source"][key] for key in ("commit", "tree", "clean")} - or record["platform"] != original["platform"] or record["version"] != original["version"]): - raise ValueError("model handoff native source binding differs") - model_root = Path(os.path.abspath(args.handoff)) - if model_root.parent != source.parent or record["handoff_directory"] != "../" + model_root.name: - raise ValueError("model handoff must be the recorded sibling directory") - protected_model_directory(model_root) - upstream_bytes = signer.regular_bytes(regular_file(source, "upstream-input.json"), signer.MAX_JSON) - upstream = signer.parse_json(upstream_bytes) - if (hashlib.sha256(upstream_bytes).hexdigest() != record["upstream_input_sha256"] - or any(upstream.get(key) != record[key] for key in ("model_catalog_unsigned", "model_retention"))): - raise ValueError("model handoff upstream binding differs") - unsigned = record["model_catalog_unsigned"] - if type(unsigned) is not dict or set(unsigned) != {"release_path", "checksum", "size", "publisher_did"}: - raise ValueError("unsigned catalogue descriptor refused") - signer.check_did(args.publisher_did) - if unsigned["publisher_did"] != args.publisher_did or unsigned["release_path"] != "model-catalog.unsigned.json": - raise ValueError("model handoff public publisher differs") - expected = {unsigned["release_path"]: retention_descriptor({key: unsigned[key] for key in ("release_path", "checksum", "size")})} - if type(record["model_retention"]) is not dict or set(record["model_retention"]) != MODEL_COMPONENTS: - raise ValueError("four model handoff retention records required") - for item in record["model_retention"].values(): - if type(item) is not dict or set(item) != {"package_cid", "car", "receipt"}: - raise ValueError("model retention record fields refused") - for kind in ("car", "receipt"): - descriptor = retention_descriptor(item[kind]) - if descriptor["release_path"] in expected: - raise ValueError("model handoff artifact alias refused") - expected[descriptor["release_path"]] = descriptor - file_pins = {name: {key: info[key] for key in ("checksum", "size")} for name, info in expected.items()} - if record["files"] != file_pins or {path.name for path in model_root.iterdir()} != set(expected): - raise ValueError("model handoff file inventory differs") - for name, pin in file_pins.items(): - path = regular_file(model_root, name) - metadata = path.stat() - if (metadata.st_nlink != 1 or metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077 - or metadata.st_size != pin["size"] or "sha256:" + digest(path) != pin["checksum"]): - raise ValueError("model handoff file protection or bytes differ") - payload = signer.parse_json(signer.regular_bytes(model_root / unsigned["release_path"], 128 * 1024)) - signed_bytes = signer.regular_bytes(args.catalogue, 128 * 1024) - destination = Path(os.path.abspath(args.output)) - parent = destination.parent - protected_model_directory(parent, private=False) - if (destination.exists() or destination.is_symlink() or destination.is_relative_to(source) - or destination.is_relative_to(model_root)): - raise ValueError("model finalization requires a new output outside its inputs") - tool_path = Path(args.openssl) - if any(tool_path.is_relative_to(path) for path in (source, model_root, destination)): - raise ValueError("qualified OpenSSL must be outside candidate inputs") - # Reserve the complete copy plus bounded new control documents before even - # creating public-verification scratch. CARs stay streamed throughout. - total = sum(info["size"] for info in original["files"].values()) + sum(info["size"] for info in expected.values()) - total += 8 * 1024**2 + len(signed_bytes) - usage = shutil.disk_usage(parent) - if (usage.free - total) * 100 < usage.total * 15: - raise ValueError("model finalization requires 15 percent free after its complete copy") - envelope, roots, tool = public_catalog_verification(signed_bytes, args.publisher_did, args.openssl, parent) - if signer.json_bytes(payload) != signer.json_bytes(envelope["payload"]): - raise ValueError("signed catalogue payload differs from unsigned preparation") - capsules = {item.get("component"): item for item in upstream.get("capsules", []) - if item.get("component") in MODEL_COMPONENTS} - if set(capsules) != MODEL_COMPONENTS: - raise ValueError("four native model capsule receipts required") - for entry in payload["entries"]: - capsule = capsules[entry["capsule_manifest"]["name"]] - if any(signer.json_bytes(capsule.get(key)) != signer.json_bytes(entry[key]) - for key in ("capsule_manifest", "object_manifest")): - raise ValueError("catalogue closure differs from native capsule receipt") - admitted = {unsigned["release_path"]} - admit_retention(record["model_retention"], roots, model_root, admitted, handoff) - manifest = signer.parse_json(regular_file(source, "components.json").read_bytes()) - if "model_retention" in manifest: - raise ValueError("native input already has model retention") - template = signer.parse_json(regular_file(source, "components-template.json").read_bytes()) - budget = template.get("model_catalog", {}).get("local_use") - admit_model_budget(budget, payload) - manifest["model_catalog"] = {"head_cid": catalog_head_cid(signed_bytes), "publisher_dids": [args.publisher_did]} - if budget is not None: - manifest["model_catalog"]["local_use"] = budget - manifest["model_retention"] = record["model_retention"] - receipt_bytes = signer.regular_bytes(regular_file(source, "platform-input.json"), signer.MAX_JSON) - components_bytes = signer.regular_bytes(regular_file(source, "components.json"), signer.MAX_JSON) - with tempfile.TemporaryDirectory(prefix=".finalize-models-", dir=parent) as temporary: - output = Path(temporary) / "input" - output.mkdir(mode=0o700) - for name, pin in original["files"].items(): - target = output / name - target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - shutil.copyfile(regular_file(source, name), target) - target.chmod(0o700 if pin["executable"] else 0o600) - if file_record(target) != pin: - raise ValueError("native input changed during model finalization") - for name in admitted - {unsigned["release_path"]}: - target = output / "artifacts" / name - if target.exists(): - raise ValueError("model retention collides with a native artifact") - shutil.copyfile(regular_file(model_root, name), target) - target.chmod(0o600) - if file_record(target)["sha256"] != file_pins[name]["checksum"][7:] or target.stat().st_size != file_pins[name]["size"]: - raise ValueError("model handoff changed during finalization") - (output / "artifacts/model-catalog.json").write_bytes(signed_bytes) - (output / "components.json").write_bytes(signer.json_bytes(manifest)) - (output / "model-native-input.json").write_bytes(receipt_bytes) - (output / "model-native-components.json").write_bytes(components_bytes) - if "artifacts/model-catalog.json" in original["files"]: - shutil.copyfile(regular_file(source, "artifacts/model-catalog.json"), output / "model-native-catalog.json") - receipt = copy.deepcopy(original) - receipt["model_finalization"] = {"native_receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), - "publisher_did": args.publisher_did, "catalogue_sha256": hashlib.sha256(signed_bytes).hexdigest(), - "signature_domain": "elastos.model.catalog.v1", "openssl": tool, - "scope": "public signature and CAR receipt bytes verified; retention import and consumer activation require acceptance"} - receipt["files"] = {str(path.relative_to(output)): file_record(path) - for path in output.rglob("*") if path.is_file()} - (output / "platform-input.json").write_bytes(signer.json_bytes(receipt)) - verify(output) - if verify(source) != original or regular_file(source, "platform-input.json").read_bytes() != receipt_bytes: - raise ValueError("native input changed during model finalization") - if destination.exists() or destination.is_symlink(): - raise ValueError("model finalization output already exists") - output.rename(destination) - return receipt - - -def verify_model_finalization(root, receipt): - signer, _ = model_tools() - proof = receipt["model_finalization"] - if (type(proof) is not dict or set(proof) != {"native_receipt_sha256", "publisher_did", "catalogue_sha256", - "signature_domain", "openssl", "scope"} or proof["signature_domain"] != "elastos.model.catalog.v1"): - raise ValueError("model finalization proof fields refused") - original_bytes = signer.regular_bytes(regular_file(root, "model-native-input.json"), signer.MAX_JSON) - if hashlib.sha256(original_bytes).hexdigest() != proof["native_receipt_sha256"]: - raise ValueError("model native provenance receipt differs") - original = verify_receipt_header(signer.parse_json(original_bytes)) - if "support_origin" in original or "model_finalization" in original: - raise ValueError("nested model finalization provenance refused") - for key, value in original.items(): - if key != "files" and receipt.get(key) != value: - raise ValueError("model finalization rewrote native build provenance") - for name, pin in original["files"].items(): - retained = {"components.json": "model-native-components.json", - "artifacts/model-catalog.json": "model-native-catalog.json"}.get(name, name) - if file_record(regular_file(root, retained)) != pin: - raise ValueError("model finalization original artifact differs") - data = signer.regular_bytes(regular_file(root, "artifacts/model-catalog.json"), 128 * 1024) - envelope = signer.parse_json(data) - signer.check_did(proof["publisher_did"]) - if (set(envelope) != {"payload", "signature", "signer_did"} - or envelope["signer_did"] != proof["publisher_did"] - or hashlib.sha256(data).hexdigest() != proof["catalogue_sha256"]): - raise ValueError("model finalization catalogue differs") - original_components = signer.parse_json(regular_file(root, "model-native-components.json").read_bytes()) - handoff_record = signer.parse_json(regular_file(root, "model-handoff.json").read_bytes()) - budget = signer.parse_json(regular_file(root, "components-template.json").read_bytes()).get("model_catalog", {}).get("local_use") - original_components["model_catalog"] = {"head_cid": catalog_head_cid(data), "publisher_dids": [proof["publisher_did"]]} - if budget is not None: - original_components["model_catalog"]["local_use"] = budget - original_components["model_retention"] = handoff_record["model_retention"] - if signer.parse_json(regular_file(root, "components.json").read_bytes()) != original_components: - raise ValueError("model finalization changed unrelated native components") def file_record(path): @@ -876,7 +542,7 @@ def record(args): omissions = json.loads(args.omissions_json.read_text()) check_contents(root, args.platform, omissions) paths = [root / "components.json", template, *sorted((root / "artifacts").rglob("*"))] - for name in ("upstream-input.json", "upstream-recipes.json", "model-handoff.json"): + for name in ("upstream-input.json", "upstream-recipes.json"): if (root / name).exists(): paths.append(regular_file(root, name)) if not reuse and (root / "upstream-recipes.json").exists(): @@ -1006,10 +672,6 @@ def verify(root): raise ValueError("reused platform omissions differ from original receipt") elif "support-input.json" in files: raise ValueError("support input lacks provenance binding") - if "model_finalization" in receipt: - verify_model_finalization(root, receipt) - elif any(name in files for name in ("model-native-input.json", "model-native-components.json", "model-native-catalog.json")): - raise ValueError("model native provenance lacks finalization binding") return receipt @@ -1180,12 +842,6 @@ def validate_inputs(values, version=None, preview_platform=None): if receipt["platform"] != name: raise ValueError("input label differs from receipt platform") manifest = json.loads((Path(path) / "components.json").read_text()) - if "upstream-input.json" in receipt["files"]: - upstream = json.loads(regular_file(Path(path), "upstream-input.json").read_bytes()) - prepared_models = {item["component"] for item in upstream.get("capsules", [])} & MODEL_COMPONENTS - if prepared_models and (prepared_models != MODEL_COMPONENTS - or set(manifest.get("model_retention", {})) != MODEL_COMPONENTS): - raise ValueError("model publication requires the finalized four-model catalogue and retention set") for component_name in manifest["profiles"]["home"]["components"]: component = manifest["external"].get(component_name) if component is None: @@ -1253,9 +909,9 @@ def merged_input_components(values, receipts): if actual != expected: raise ValueError(f"input manifest changed after admission: {platform}") manifest = json.loads(manifest_bytes) - contract = {name: manifest.get(name) for name in ("model_catalog", "model_retention")} + contract = {"model_catalog": manifest.get("model_catalog")} if model_contract is not None and contract != model_contract: - raise ValueError("platform model catalogue/retention contracts differ") + raise ValueError("platform model catalogue pins differ") model_contract = contract for name, value in contract.items(): if value is None: @@ -1374,10 +1030,6 @@ def prepared_components_bytes(stage, record, cids): for info in entry.get("platforms", {}).values(): if info.get("release_path"): info["cid"] = cids[info["release_path"]] - for model in manifest.get("model_retention", {}).values(): - for kind in ("car", "receipt"): - descriptor = model[kind] - descriptor["cid"] = cids[descriptor["release_path"]] # Each selected platform gets the same complete manifest. Descriptors were # replaced as units during staging; each release descriptor names admitted bytes. output_bytes = (json.dumps(manifest, indent=2) + "\n").encode() @@ -1506,10 +1158,6 @@ def main(): reuse.add_argument("--" + name, type=Path, required=True) reuse.add_argument("--platform", choices=PLATFORMS, required=True) reuse.add_argument("--version", required=True) - finalize = commands.add_parser("finalize-models", help="verify public catalogue and retained CAR bytes into a new native input") - for name in ("input", "handoff", "catalogue", "output", "openssl"): - finalize.add_argument("--" + name, type=Path, required=True) - finalize.add_argument("--publisher-did", required=True) combined = commands.add_parser("validate-inputs") combined.add_argument("--input", action="append", required=True) combined.add_argument("--version") @@ -1539,9 +1187,6 @@ def main(): record(args) elif args.command == "copy-support": copy_support(args) - elif args.command == "finalize-models": - finalize_models(args) - print("Verified public catalogue and CAR receipt bytes in a new native input; consumer acceptance remains separate.") elif args.command == "verify": print(json.dumps(verify(args.root), sort_keys=True)) elif args.command == "stage-inputs": diff --git a/scripts/release-signer-test.py b/scripts/release-signer-test.py index 5169076d7..c7f97a86c 100644 --- a/scripts/release-signer-test.py +++ b/scripts/release-signer-test.py @@ -51,142 +51,6 @@ def close(self): self.calls.append("close") -class CatalogSignerTests(unittest.TestCase): - # Reuse the public Git fixture, without inheriting its release test cases. - def setUp(self): - self.catalog_setup() - - def fetch(self, path): - return SignerTests.fetch(self, path) - - def approve_manifest(self): - return SignerTests.approve_manifest(self) - - def catalog_setup(self): - SignerTests.setUp(self) - self.payload = {"schema": "elastos.model.catalog/v1", "published_at": 1, "expires_at": None, "entries": []} - bindings = {} - for name in sorted(S.MODEL_COMPONENTS): - digest = hashlib.sha256(name.encode()).digest() - cid = "b" + base64.b32encode(b"\x01\x70\x12\x20" + digest).decode().lower().rstrip("=") - capsule = {"schema": "elastos.capsule/v1", "name": name, "version": "1", "role": "content", "type": "data", - "projections": ["content"], "entrypoint": "model.gguf", "model_content": { - "format": "gguf", "quantization": "Q4_K_M", "engine": "llama.cpp", "consumer_interface": "elastos.provider.model", - "consumer_interface_version": "0.1.0", "minimum_memory_mb": 2048, - "license": {"spdx_id": "Apache-2.0", "path": "LICENSE"}, - "provenance": {"base_repository": "fixture/base", "base_revision": COMMIT, - "quantized_repository": "fixture/quantized", "quantized_revision": TREE, - "base_license": {"spdx_id": "Apache-2.0", "path": "LICENSE.base"}, "path": "PROVENANCE.md"}}} - files = [] - content = hashlib.sha256() - for path, data in sorted({"capsule.json": S.json_bytes(capsule), "model.gguf": b"GGUF fixture", - "LICENSE": b"fixture licence", "LICENSE.base": b"base licence", "PROVENANCE.md": b"pinned provenance"}.items()): - item = {"path": path, "sha256": S.sha256(data), "size": len(data)} - files.append(item) - for field in (path, item["sha256"], str(item["size"])): - content.update(field.encode() + b"\0") - content_digest = "sha256:" + content.hexdigest() - self.payload["entries"].append({"cid": cid, - "capsule_manifest": capsule, - "object_manifest": {"schema": "elastos.content.object.manifest/v1", "kind": "capsule", "content_digest": content_digest, "files": files}}) - bindings[name] = {"package_cid": cid, "content_digest": content_digest} - self.policy.update(purpose="elastos.model.catalog/v1", published_at=1, catalog_entries=bindings) - self.approve_catalog() - - def approve_catalog(self): - data = S.json_bytes(self.payload) - (self.root / "model-catalog.unsigned.json").write_bytes(data) - self.policy["manifest_sha256"] = S.sha256(data) - - def prepare(self): - return S.prepare_catalog(self.policy, self.root, "model-catalog.unsigned.json", self.fetch, now=2) - - def test_exact_catalogue_uses_only_its_approved_domain(self): - backend = FakeBackend() - publication = dict(S.sign_catalog(self.prepare(), backend)) - self.assertEqual(set(publication), {"model-catalog.json"}) - self.assertEqual(S.parse_json(publication["model-catalog.json"])["payload"], self.payload) - expected = hashlib.sha256(b"elastos.model.catalog.v1\0" + S.json_bytes(self.payload)).digest() - self.assertIn(("sign", expected), backend.calls) - self.assertFalse(self.marker.exists()) - - def test_catalogue_requires_an_exact_hash_and_content_binding(self): - self.payload["entries"][0]["object_manifest"]["content_digest"] = "sha256:" + "0" * 64 - (self.root / "model-catalog.unsigned.json").write_bytes(S.json_bytes(self.payload)) - with self.assertRaisesRegex(ValueError, "hash differs"): - self.prepare() - self.approve_catalog() - with self.assertRaisesRegex(ValueError, "content binding differs"): - self.prepare() - - def test_catalogue_refuses_wrong_purpose_duplicate_missing_raw_and_future_entries(self): - original = copy.deepcopy(self.payload) - for change in ("duplicate", "missing", "raw", "future", "expired", "unknown"): - with self.subTest(change=change): - self.payload = copy.deepcopy(original) - if change == "duplicate": self.payload["entries"][1] = copy.deepcopy(self.payload["entries"][0]) - if change == "missing": self.payload["entries"].pop() - if change == "raw": self.payload["entries"][0]["cid"] = S.raw_cid(b"file") - if change == "future": self.payload["published_at"] = 3 - if change == "expired": self.payload["expires_at"] = 2 - if change == "unknown": self.payload["authority"] = "candidate" - self.approve_catalog() - with self.assertRaises(ValueError): self.prepare() - self.payload = original - self.approve_catalog() - self.policy["purpose"] = "elastos.release/v1" - with self.assertRaisesRegex(ValueError, "purpose required"): self.prepare() - - def test_catalogue_source_admission_precedes_signing(self): - prefix = f"/repos/{S.REPOSITORY}" - self.api[f"{prefix}/compare/{COMMIT}...{MAIN}?per_page=1"]["status"] = "diverged" - with self.assertRaisesRegex(ValueError, "outside main"): self.prepare() - self.assertFalse(self.marker.exists()) - - def test_catalogue_requires_matching_public_key_and_verified_signature(self): - prepared = self.prepare() - backend = FakeBackend() - backend.public = bytes(32) - with self.assertRaisesRegex(ValueError, "public DID differs"): S.sign_catalog(prepared, backend) - self.assertEqual(backend.calls, ["public"]) - backend = FakeBackend() - backend.verified = False - with self.assertRaisesRegex(ValueError, "verification failed"): S.sign_catalog(prepared, backend) - - def test_catalogue_refuses_execution_authority_missing_notice_and_changed_capsule_bytes(self): - original = copy.deepcopy(self.payload) - for change in ("authority", "notice", "capsule", "aliases", "digest"): - with self.subTest(change=change): - self.payload = copy.deepcopy(original) - entry = self.payload["entries"][0] - if change == "authority": entry["capsule_manifest"]["permissions"] = {"network": ["*"]} - if change == "notice": entry["capsule_manifest"]["model_content"]["license"]["path"] = "MISSING" - if change == "capsule": entry["capsule_manifest"]["version"] = "2" - if change == "aliases": entry["object_manifest"]["files"][1]["path"] = entry["object_manifest"]["files"][0]["path"].lower() - if change == "digest": entry["object_manifest"]["files"][0]["size"] += 1 - self.approve_catalog() - with self.assertRaises(ValueError): self.prepare() - - def test_catalogue_refuses_nonportable_model_paths(self): - original = copy.deepcopy(self.payload) - for field in ("entrypoint", "notice", "repository", "closure"): - with self.subTest(field=field): - self.payload = copy.deepcopy(original) - entry = self.payload["entries"][0] - capsule = entry["capsule_manifest"] - if field == "entrypoint": capsule["entrypoint"] = "model./model.gguf" - if field == "notice": capsule["model_content"]["license"]["path"] = "LICENSE." - if field == "repository": capsule["model_content"]["provenance"]["base_repository"] = "owner./repo" - if field == "closure": entry["object_manifest"]["files"][0]["path"] = "LICENSE." - self.approve_catalog() - with self.assertRaisesRegex(ValueError, "canonical model path"): - self.prepare() - for value in ("model./model.gguf", "../model.gguf", "a//b", "a\\b", "é.gguf", "a" * 257): - with self.assertRaisesRegex(ValueError, "canonical model path"): - S.model_path(value) - self.assertEqual(S.model_path("_private/model-Q4_K_M.gguf"), "_private/model-Q4_K_M.gguf") - - class SignerTests(unittest.TestCase): def setUp(self): # These directories are removed by unittest cleanup, including refusals. diff --git a/scripts/release-signer.py b/scripts/release-signer.py index 9f7cab569..f7a16138b 100644 --- a/scripts/release-signer.py +++ b/scripts/release-signer.py @@ -41,7 +41,6 @@ import subprocess import sys import tempfile -import time REPOSITORY = "Elacity/elastos-runtime" @@ -52,8 +51,6 @@ MAX_JSON = 2 * 1024 * 1024 MAX_FILE = 512 * 1024 * 1024 CHANNELS = {"stable", "canary", "jetson-test"} -MODEL_CATALOG_DOMAIN = "elastos.model.catalog.v1" -MODEL_COMPONENTS = {"model-qwen3.5-0.8b", "model-qwen3.5-4b", "model-qwen3.5-9b", "model-bonsai-8b-q1"} STAMPS = {"MAINTAINER_DID", "SOURCE_CONNECT_TICKET", "PUBLISHER_GATEWAY", "PUBLISHER_NODE_ID", "IPNS_NAME"} @@ -563,149 +560,10 @@ def component_refs(value): def signature_digest(domain, payload): - require(domain in ("elastos.release.v1", "elastos.release.head.v1", MODEL_CATALOG_DOMAIN), "signing domain refused") + require(domain in ("elastos.release.v1", "elastos.release.head.v1"), "signing domain refused") return hashlib.sha256(domain.encode() + b"\0" + payload).digest() -@dataclass(frozen=True) -class PreparedCatalog: - publisher_did: str - payload: bytes - - -def model_path(value): - """Match the Runtime's portable model-package path admission.""" - require(type(value) is str and 0 < len(value) <= 256 - and all(re.fullmatch(r"[A-Za-z0-9._-]+", part) and part not in (".", "..") - and not part.endswith(".") for part in value.split("/")), - "canonical model path required") - return value - - -def catalogue_closure(capsule, index): - """Check the passive model closure without executing candidate helpers.""" - require(set(capsule) == {"schema", "name", "version", "role", "type", "projections", "entrypoint", "model_content"} - and capsule["schema"] == "elastos.capsule/v1" and capsule["role"] == "content" - and capsule["type"] == "data" and capsule["projections"] == ["content"], "passive catalogue capsule required") - require(type(capsule["version"]) is str and 0 < len(capsule["version"]) <= 32, "model version refused") - entrypoint = capsule["entrypoint"] - model_path(entrypoint) - require(entrypoint.endswith(".gguf"), "GGUF model entrypoint required") - model = capsule["model_content"] - require(type(model) is dict and set(model) == {"format", "quantization", "engine", "consumer_interface", - "consumer_interface_version", "minimum_memory_mb", "license", "provenance"}, "model content fields refused") - require(model["format"] == "gguf" and model["quantization"] in ("Q4_K_M", "Q8_0", "Q1_0") - and model["engine"] == "llama.cpp" and model["consumer_interface"] == "elastos.provider.model" - and model["consumer_interface_version"] == "0.1.0" and type(model["minimum_memory_mb"]) is int - and 1 <= model["minimum_memory_mb"] <= 1048576, "model consumer contract refused") - provenance = model["provenance"] - require(type(provenance) is dict and set(provenance) == {"base_repository", "base_revision", "base_license", - "quantized_repository", "quantized_revision", "path"}, "model provenance fields refused") - for name in ("base_repository", "quantized_repository"): - model_path(provenance[name]) - require(len(provenance[name].split("/")) == 2, "model owner/repository required") - for name in ("base_revision", "quantized_revision"): - oid(provenance[name]) - notices = [provenance["path"]] - for license in (model["license"], provenance["base_license"]): - require(type(license) is dict and set(license) == {"spdx_id", "path"} - and license["spdx_id"] == "Apache-2.0", "model licence facts refused") - notices.append(license["path"]) - for path in notices: - model_path(path) - require(path not in (entrypoint, "capsule.json"), "model notice requires a distinct closure file") - require(set(index) == {"schema", "kind", "files", "content_digest"} - and index["schema"] == "elastos.content.object.manifest/v1" and index["kind"] == "capsule", - "self-contained model closure required") - files = index["files"] - require(type(files) is list and 1 <= len(files) <= 32, "model closure file count refused") - previous, paths, total, digest = "", {}, 0, hashlib.sha256() - for item in files: - require(type(item) is dict and set(item) == {"path", "sha256", "size"}, "model closure file fields refused") - path = item["path"] - model_path(path) - checked_hash(item["sha256"]) - require(path > previous and path.lower() not in paths and path.lower() != "_elastos_object.json" - and type(item["size"]) is int and 0 < item["size"] <= 16 * 1024**3 - and (path == entrypoint or item["size"] <= 1024**2), "model closure path or size refused") - paths[path.lower()] = item - total += item["size"] - previous = path - for value in (path, item["sha256"], str(item["size"])): - digest.update(value.encode() + b"\0") - require(total <= 16 * 1024**3 and index["content_digest"] == "sha256:" + digest.hexdigest(), - "model closure digest or size differs") - require(all(path.lower() in paths and paths[path.lower()]["path"] == path - for path in (entrypoint, "capsule.json", *notices)), "model closure reference absent") - capsule_bytes = json_bytes(capsule) - record = paths["capsule.json"] - require(record["size"] == len(capsule_bytes) and record["sha256"] == sha256(capsule_bytes), - "model capsule closure bytes differ") - - -def prepare_catalog(policy, root, manifest_name, fetch, now=None): - """Admit an exact operator-approved public catalogue before key access.""" - require(policy.get("purpose") == "elastos.model.catalog/v1", "catalogue signing purpose required") - verify_source(policy, fetch) - check_did(policy.get("publisher_did")) - root = Path(root) - relative_path(manifest_name) - held = directory_fd(root) - try: - data = regular_bytes(manifest_name, 128 * 1024, root_fd=held) - finally: - os.close(held) - require(sha256(data) == checked_hash(policy.get("manifest_sha256")), "approved catalogue hash differs") - payload = parse_json(data) - require(type(payload) is dict and set(payload) == {"schema", "published_at", "expires_at", "entries"} - and payload["schema"] == "elastos.model.catalog/v1", "catalogue payload fields refused") - now = int(time.time()) if now is None else now - published, expiry = payload["published_at"], payload["expires_at"] - require(type(published) is int and type(policy.get("published_at")) is int and 0 <= published <= now - and published == policy.get("published_at"), "approved catalogue publication time differs") - require(expiry is None or type(expiry) is int and expiry > now and expiry > published, - "catalogue expiry refused") - approved = policy.get("catalog_entries") - require(type(approved) is dict and set(approved) == MODEL_COMPONENTS, "four approved catalogue bindings required") - entries = payload["entries"] - require(type(entries) is list and len(entries) == 4, "four catalogue entries required") - seen, roots = set(), set() - for entry in entries: - require(type(entry) is dict and set(entry) == {"cid", "capsule_manifest", "object_manifest"}, - "catalogue entry fields refused") - cid = entry["cid"] - codec, _ = cid_info(cid) - require(cid.startswith("b") and codec == 0x70 and cid not in roots, "distinct catalogue directory CIDs required") - capsule, index = entry["capsule_manifest"], entry["object_manifest"] - require(type(capsule) is dict and type(index) is dict, "catalogue capsule and object manifest required") - name = capsule.get("name") - require(type(name) is str and name in approved and name not in seen, "catalogue component inventory differs") - binding = approved[name] - require(type(binding) is dict and set(binding) == {"package_cid", "content_digest"} - and binding["package_cid"] == cid and binding["content_digest"] == index.get("content_digest") - and type(binding["content_digest"]) is str and binding["content_digest"].startswith("sha256:"), - "approved catalogue content binding differs") - checked_hash(binding["content_digest"][7:]) - require(capsule.get("schema") == "elastos.capsule/v1" and capsule.get("role") == "content" - and capsule.get("type") == "data" and type(capsule.get("model_content")) is dict - and index.get("schema") == "elastos.content.object.manifest/v1" and index.get("kind") == "capsule", - "catalogue model contract refused") - catalogue_closure(capsule, index) - seen.add(name) - roots.add(cid) - return PreparedCatalog(policy["publisher_did"], json_bytes(payload)) - - -def sign_catalog(prepared, backend): - require(public_did(backend.public_key()) == prepared.publisher_did, "custodian public DID differs") - digest = signature_digest(MODEL_CATALOG_DOMAIN, prepared.payload) - signature = backend.sign(digest) - require(type(signature) is bytes and len(signature) == 64, "Ed25519 signature length differs") - require(backend.verify(digest, signature) is True, "signature public verification failed") - return (("model-catalog.json", json_bytes({"payload": parse_json(prepared.payload), - "signature": signature.hex(), "signer_did": prepared.publisher_did})),) - - def sign_publication(prepared, backend): require(len(prepared.release) + 300 <= 256 * 1024, "single-chunk release metadata required") require(public_did(backend.public_key()) == prepared.publisher_did, "custodian public DID differs") @@ -784,8 +642,7 @@ def close(self): def confirmed(prepared, input_stream, output_stream): - subject = "model catalogue" if isinstance(prepared, PreparedCatalog) else "release" - output_stream.write(f"Sign approved {subject} as {prepared.publisher_did}.\nType this complete DID to confirm, or press Enter to cancel: ") + output_stream.write(f"Sign approved release as {prepared.publisher_did}.\nType this complete DID to confirm, or press Enter to cancel: ") output_stream.flush() return input_stream.readline().strip() == prepared.publisher_did @@ -795,7 +652,6 @@ def main(): parser.add_argument("--policy", required=True, type=Path) parser.add_argument("--input-root", required=True, type=Path) parser.add_argument("--manifest", default="signing-input.json") - parser.add_argument("--catalog", action="store_true", help="sign only an exact approved four-model catalogue payload") parser.add_argument("--output-root", required=True, type=Path) args = parser.parse_args() require(sys.flags.isolated == 1 and sys.flags.no_site == 1, "run the pinned interpreter with -I -S") @@ -811,17 +667,13 @@ def main(): require(info.st_uid in (0, os.getuid()) and not info.st_mode & 0o022, "output parent protection refused") with tempfile.TemporaryDirectory(prefix=".elastos-signing-", dir=args.output_root.parent) as scratch: - if args.catalog: - prepared = prepare_catalog(policy, args.input_root, args.manifest, github_json) - else: - require(policy.get("purpose") != "elastos.model.catalog/v1", "catalogue policy requires catalogue mode") - prepared = prepare(policy, args.input_root, args.manifest, github_json, Path(scratch)) + prepared = prepare(policy, args.input_root, args.manifest, github_json, Path(scratch)) require(confirmed(prepared, sys.stdin, sys.stderr), "signing cancelled") # Recheck canonical source authority after confirmation, before backend use. verify_source(policy, github_json) backend = OpenSSLBackend(policy, args.input_root, Path(scratch)) try: - publication = sign_catalog(prepared, backend) if args.catalog else sign_publication(prepared, backend) + publication = sign_publication(prepared, backend) finally: backend.close() args.output_root.mkdir(mode=0o700) @@ -838,7 +690,7 @@ def main(): except Exception: shutil.rmtree(args.output_root) raise - print("Signed approved model catalogue." if args.catalog else "Signed approved publication snapshot.") + print("Signed approved publication snapshot.") if __name__ == "__main__": diff --git a/scripts/release-upstream-assets-test.py b/scripts/release-upstream-assets-test.py deleted file mode 100644 index 71ea3276d..000000000 --- a/scripts/release-upstream-assets-test.py +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/env python3 -"""Small key-free build fixtures. Kubo receipts are mocked, not product proof.""" - -import base64 -import hashlib -import importlib.util -import io -import json -from pathlib import Path -import re -import tarfile -import tempfile -from types import SimpleNamespace -import unittest -from unittest.mock import patch -import urllib.parse - - -WORKSPACE = Path(__file__).resolve().parent.parent -spec = importlib.util.spec_from_file_location("upstream_assets", Path(__file__).with_name("release-upstream-assets.py")) -assets = importlib.util.module_from_spec(spec) -spec.loader.exec_module(assets) - - -def fixture_cid(data, codec=0x70): - # Mock receipt identity only; this does not calculate a UnixFS directory. - return "b" + base64.b32encode(bytes((1, codec, 0x12, 0x20)) + hashlib.sha256(data).digest()).decode().lower().rstrip("=") - - -class Connection: - def __init__(self, kubo): - self.kubo, self.headers, self.chunks = kubo, {}, [] - self.closed = False - - def putrequest(self, method, target): - self.method, self.target = method, target - - def putheader(self, name, value): - self.headers[name] = value - - def endheaders(self): - pass - - def send(self, data): - self.chunks.append(data) - - def getresponse(self): - self.kubo.test.assertEqual(self.method, "POST") - path, query = self.target.split("?", 1) - self.kubo.test.assertEqual(path, "/api/v0/add") - options = dict(urllib.parse.parse_qsl(query)) - only_hash = options.pop("only-hash") == "true" - self.kubo.test.assertEqual(options.pop("pin"), str(not only_hash).lower()) - self.kubo.test.assertEqual(options, assets.MODEL_ADD_OPTIONS) - body = b"".join(self.chunks) - self.kubo.test.assertEqual(len(body), int(self.headers["Content-Length"])) - boundary = self.headers["Content-Type"].split("boundary=", 1)[1].encode() - files = {} - for part in body.split(b"--" + boundary)[1:-1]: - header, data = part.split(b"\r\n\r\n", 1) - name = re.search(rb'filename="([^"]+)"', header)[1].decode() - files[name] = data[:-2] - self.kubo.uploads.append((only_hash, files)) - root = fixture_cid(b"".join(name.encode() + b"\0" + data for name, data in sorted(files.items()))) - if not only_hash: - root = self.kubo.retained_override or root - self.kubo.pins.add(root) - if self.kubo.response_override: - return self.kubo.response_override - response = io.BytesIO(json.dumps({"Name": "", "Hash": root, "Size": "1"}).encode() + b"\n") - response.status, response.trailers = 200, {} - return response - - def close(self): - self.closed = True - - -class Kubo: - def __init__(self, test, repo): - self.test, self.repo = test, repo - self.base_url = "http://127.0.0.1:59101" - self.uploads, self.connections, self.pins, self.exports = [], [], set(), [] - self.retained_override = self.response_override = None - self.profile_error = self.export_error = False - self.pin_verified = True - - def verify_profile(self): - if self.profile_error: - raise assets.handoff.Refusal("fixture noncanonical profile") - return "0.40.1" - - def repo_path(self): - return str(self.repo) - - def _connect(self): - connection = Connection(self) - self.connections.append(connection) - return connection - - def is_recursively_pinned(self, cid): - return self.pin_verified and cid in self.pins - - def export_car(self, cid, destination): - self.exports.append(cid) - data = b"inert CAR fixture for " + cid.encode() - Path(destination).write_bytes(data) - if self.export_error: - raise assets.handoff.Refusal("fixture export stream error") - return assets.handoff.CarDigest(hashlib.sha256(data).hexdigest(), len(data)) - - -class AssetsTest(unittest.TestCase): - def setUp(self): - self.scratch = tempfile.TemporaryDirectory() - self.addCleanup(self.scratch.cleanup) - self.root = Path(self.scratch.name).resolve() - self.output, self.cache, self.repo = self.root / "artifacts", self.root / "cache", self.root / "kubo-repo" - self.repo.mkdir(mode=0o700) - (self.repo / "api").write_text("/ip4/127.0.0.1/tcp/59101\n") - self.bin = self.root / "qualified-kubo" - self.bin.write_bytes(b"inert Kubo binary fixture") - self.bin.chmod(0o700) - self.recipes = [self.recipe("kubo", self.bin.read_bytes(), model=False)] - for index, name in enumerate(sorted(assets.MODEL_COMPONENTS)): - weight = b"GGUF" + bytes((index,)) * (assets.upstream.CHUNK + 7 if index == 0 else 23) - self.recipes.append(self.recipe(name, weight, model=True)) - self.inventory = self.root / "recipes.json" - self.write_inventory() - self.kubo = Kubo(self, self.repo) - self.patch("SOURCE_ROOT", self.root) - self.patch("RECIPE_FILE", self.inventory) - - def patch(self, name, value): - patcher = patch.object(assets, name, value) - patcher.start() - self.addCleanup(patcher.stop) - - def source(self, name, data): - path = self.root / "inputs" / name - path.parent.mkdir(exist_ok=True) - path.write_bytes(data) - return {"path": str(path.relative_to(self.root)), "checksum": "sha256:" + hashlib.sha256(data).hexdigest(), "max_bytes": len(data)} - - def recipe(self, name, data, model): - recipe = {"schema": assets.upstream.SCHEMA, "component": name, "platform": "*" if model else "linux-amd64", - "version": "1.0.0" if model else "0.40.1", "source": self.source(name + "-payload", data), "format": "raw", - "root": name, "entrypoint": "weights.gguf" if model else "ipfs", - "extract_path": name + ("/weights.gguf" if model else "/ipfs"), "install_path": "models/" + name if model else "bin/kubo", - "max_unpacked_bytes": len(data), "license": {"spdx_id": "Apache-2.0", "files": [ - {"name": "LICENSE", "source": self.source(name + "-license", b"inert model license\n")} ]}} - if model: - recipe["license"]["files"].append({"name": "LICENSE.base", "source": self.source(name + "-base-license", b"inert base license\n")}) - recipe["notices"] = [{"name": "PROVENANCE.md", "source": self.source(name + "-provenance", b"inert provenance\n")}] - recipe["model_content"] = {"format": "gguf", "quantization": "Q1_0" if "bonsai" in name else "Q4_K_M", - "engine": "llama.cpp", "consumer_interface": "elastos.provider.model", "consumer_interface_version": "0.1.0", - "minimum_memory_mb": 512, "license": {"spdx_id": "Apache-2.0", "path": "LICENSE"}, - "provenance": {"base_repository": "fixture/base", "base_revision": "a" * 40, - "quantized_repository": "fixture/quantized", "quantized_revision": "b" * 40, - "base_license": {"spdx_id": "Apache-2.0", "path": "LICENSE.base"}, "path": "PROVENANCE.md"}} - return recipe - - def write_inventory(self): - self.inventory.write_text(json.dumps({"schema": "elastos.release-upstream-recipes/v1", "recipes": self.recipes})) - - def prepare(self, **options): - arguments = {"model_kubo_bin": self.bin, "model_kubo_repo": self.repo, "published_at": 123, - "model_publisher_did": "did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c", **options} - with (patch.object(assets.subprocess, "run", return_value=SimpleNamespace(stdout="0.40.1\n")) as command, - patch.object(assets.handoff, "KuboApi", return_value=self.kubo), - patch.object(assets.handoff, "kubo_from_args", return_value=self.kubo)): - result = assets.prepare("linux-amd64", self.cache, self.output, **arguments) - self.assertEqual(command.call_args.args[0], [str(self.bin), "version", "--number"]) - self.assertEqual(command.call_args.kwargs["env"]["IPFS_PATH"], str(self.repo)) - return result - - def model_package(self): - self.output.mkdir(mode=0o700) - recipe = assets.resolved_recipe(self.recipes[1]) - receipt = assets.upstream.package(recipe, self.cache, self.output) - return recipe, receipt - - def test_default_driver_keeps_existing_manifest_and_output_shape(self): - with patch.object(assets.handoff, "KuboApi") as api, patch.object(assets.subprocess, "run") as command: - result = assets.prepare("linux-amd64", self.cache, self.output) - api.assert_not_called() - command.assert_not_called() - self.assertEqual(set(result), {"external"}) - self.assertTrue(all(set(component) == {"platforms", "capsule_metadata"} for component in result["external"].values())) - receipt = json.loads((self.output / "upstream-input.json").read_bytes()) - self.assertEqual(set(receipt), {"schema", "platform", "recipes_sha256", "capsules"}) - self.assertFalse((self.root / "artifacts-model-handoff").exists()) - - def test_four_entry_draft_preserves_exact_index_and_uses_separate_retention_files(self): - result = self.prepare(model_publisher_did="did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c") - handoff_dir = self.root / "artifacts-model-handoff" - draft = json.loads((handoff_dir / "model-catalog.unsigned.json").read_bytes()) - self.assertEqual(set(draft), {"schema", "published_at", "expires_at", "entries"}) - self.assertEqual(draft["schema"], "elastos.model.catalog/v1") - self.assertEqual(draft["published_at"], 123) - self.assertIsNone(draft["expires_at"]) - self.assertEqual(len(draft["entries"]), 4) - self.assertEqual(len({entry["cid"] for entry in draft["entries"]}), 4) - self.assertEqual(len(self.kubo.uploads), 8) - self.assertEqual(len(self.kubo.exports), 4) - self.assertTrue(all(connection.closed for connection in self.kubo.connections)) - self.assertTrue(all(max(map(len, connection.chunks)) <= assets.upstream.CHUNK for connection in self.kubo.connections)) - for index, entry in enumerate(draft["entries"]): - self.assertEqual(set(entry), {"cid", "capsule_manifest", "object_manifest"}) - self.assertEqual(assets.directory_cid(entry["cid"]), entry["cid"]) - name = entry["capsule_manifest"]["name"] - first, second = self.kubo.uploads[index * 2:index * 2 + 2] - self.assertEqual((first[0], second[0]), (True, False)) - self.assertEqual(first[1], second[1]) - self.assertEqual(first[1]["capsule.json"], assets.upstream.canonical(entry["capsule_manifest"])[:-1]) - self.assertEqual(first[1]["_elastos_object.json"], assets.upstream.canonical(entry["object_manifest"])) - for item in entry["object_manifest"]["files"]: - data = first[1][item["path"]] - self.assertEqual((item["size"], item["sha256"]), (len(data), hashlib.sha256(data).hexdigest())) - retention = result["model_retention"][name] - self.assertEqual(set(retention), {"package_cid", "car", "receipt"}) - self.assertEqual(retention["package_cid"], entry["cid"]) - for descriptor in (retention["car"], retention["receipt"]): - path = handoff_dir / descriptor["release_path"] - self.assertEqual(set(descriptor), {"release_path", "checksum", "size"}) - self.assertEqual(descriptor["checksum"], "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()) - self.assertEqual(descriptor["size"], path.stat().st_size) - self.assertFalse((self.output / path.name).exists()) - assets.handoff.check_car(handoff_dir / retention["car"]["release_path"], handoff_dir / retention["receipt"]["release_path"], entry["cid"]) - self.assertEqual(set(result["external"][name]), {"platforms", "capsule_metadata"}) - self.assertFalse(list(self.output.glob(".model-closure-*"))) - self.assertEqual(result["model_catalog_unsigned"]["publisher_did"], "did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c") - build_record = json.loads((self.output / "upstream-input.json").read_bytes()) - self.assertEqual(build_record["model_retention"], result["model_retention"]) - - def test_kubo_options_match_runtime_canonical_directory_options(self): - source = (WORKSPACE / "capsules/ipfs-provider/src/directory_hash.rs").read_text() - block = source.split("const ADD_OPTIONS:", 1)[1].split("];", 1)[0] - expected = dict(re.findall(r'\("([^"]+)", "([^"]+)"\)', block)) - self.assertEqual(expected, {**assets.MODEL_ADD_OPTIONS, "only-hash": "true", "pin": "false"}) - - def test_incomplete_options_invalid_timestamp_and_model_set_refused_before_packaging(self): - with patch.object(assets.upstream, "package") as package: - for options in ({"model_kubo_bin": self.bin}, {"model_publisher_did": "did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c"}, - {"model_kubo_bin": self.bin, "model_kubo_repo": self.repo, "published_at": -1}, - {"model_kubo_bin": self.bin, "model_kubo_repo": self.repo, "published_at": True}): - with self.subTest(options=options), self.assertRaises(ValueError): - assets.prepare("linux-amd64", self.cache, self.output, **options) - self.recipes.pop() - self.write_inventory() - with self.assertRaisesRegex(ValueError, "four pinned"): - assets.prepare("linux-amd64", self.cache, self.output, model_kubo_bin=self.bin, model_kubo_repo=self.repo, published_at=123, - model_publisher_did="did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c") - package.assert_not_called() - - def test_fresh_model_preparation_requires_a_canonical_public_did_before_packaging(self): - with patch.object(assets.upstream, "package") as package: - for publisher in (None, "did:key:z6MkFixture", "did:key:z" + "1" * 48, 123): - with self.subTest(publisher=publisher), self.assertRaises(ValueError): - self.prepare(model_publisher_did=publisher) - package.assert_not_called() - self.assertFalse(self.output.exists()) - - def test_missing_source_pin_refused_before_kubo_or_network(self): - self.recipes[0]["source"].pop("checksum") - self.write_inventory() - with patch.object(assets.handoff, "KuboApi") as api, patch.object(assets.upstream, "response") as network: - with self.assertRaisesRegex(ValueError, "recorded SHA"): - assets.prepare("linux-amd64", self.cache, self.output) - api.assert_not_called() - network.assert_not_called() - - def test_kubo_binary_checksum_mismatch_refused_before_execution(self): - self.bin.write_bytes(b"different Kubo binary") - with patch.object(assets.subprocess, "run") as command: - with self.assertRaisesRegex(ValueError, "pinned capsule entrypoint"): - self.prepare() - command.assert_not_called() - self.assertFalse(self.kubo.uploads) - - def test_wrong_api_repository_and_profile_refused(self): - with patch.object(self.kubo, "repo_path", return_value=str(self.root / "another-repo")): - with self.assertRaisesRegex(ValueError, "repository differs"): - self.prepare() - self.assertFalse(self.kubo.uploads) - self.output = self.root / "second-artifacts" - self.kubo.profile_error = True - with self.assertRaisesRegex(assets.handoff.Refusal, "noncanonical profile"): - self.prepare() - self.assertFalse(self.kubo.uploads) - - def test_hash_add_mismatch_and_missing_recursive_pin_refuse_catalogue(self): - self.kubo.retained_override = fixture_cid(b"wrong retained root") - with self.assertRaisesRegex(ValueError, "differs from Kubo only-hash"): - self.prepare() - self.assertFalse(self.kubo.exports) - self.assertFalse(list(self.output.glob(".model-closure-*"))) - self.output = self.root / "second-artifacts" - self.kubo.retained_override, self.kubo.pin_verified = None, False - with self.assertRaisesRegex(ValueError, "recursive retention pin"): - self.prepare() - self.assertFalse(self.kubo.exports) - - def test_export_stream_error_cleans_partial_car_and_staging(self): - self.kubo.export_error = True - with self.assertRaisesRegex(assets.handoff.Refusal, "export stream error"): - self.prepare() - handoff_dir = self.root / "artifacts-model-handoff" - self.assertFalse(list(handoff_dir.iterdir())) - self.assertFalse(list(self.output.glob(".model-closure-*"))) - - def test_retention_and_export_disk_floors_run_before_their_writes(self): - for index, before_export in enumerate((False, True)): - self.output = self.root / ("floor-artifacts-" + str(index)) - handoff_dir = self.output.with_name(self.output.name + "-model-handoff") - blocked = handoff_dir if before_export else self.repo - self.kubo.uploads.clear() - def gate(path, additional): - if Path(path) == blocked: - raise ValueError("fixture 15% free space refusal") - with patch.object(assets.upstream, "disk_gate", side_effect=gate): - with self.assertRaisesRegex(ValueError, "15% free space"): - self.prepare() - self.assertEqual(len(self.kubo.uploads), 2 if before_export else 1) - self.assertFalse(self.kubo.exports) - self.assertFalse(list(self.output.glob(".model-closure-*"))) - - def test_handoff_inside_release_artifacts_is_refused(self): - with self.assertRaisesRegex(ValueError, "outside release artifacts"): - self.prepare(model_handoff_output=self.output / "nested-handoff") - self.assertFalse(self.kubo.uploads) - self.assertFalse((self.output / "nested-handoff").exists()) - - def authoritative_handoff(self): - result = self.prepare(model_publisher_did="did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c") - source = self.root / "artifacts-model-handoff" - self.output = self.root / "reuse-artifacts" - return source, result - - def reuse(self, source, **options): - arguments = {"model_handoff_input": source, "model_handoff_output": self.root / "reused-handoff", - "published_at": 123, "model_publisher_did": "did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c", **options} - with (patch.object(assets.subprocess, "run") as execute, - patch.object(assets.handoff, "KuboApi") as api, - patch.object(assets, "add_model_directory") as add, - patch.object(assets.handoff, "cmd_export") as export): - result = assets.prepare("linux-amd64", self.cache, self.output, **arguments) - execute.assert_not_called() - api.assert_not_called() - add.assert_not_called() - export.assert_not_called() - return result - - def test_authoritative_handoff_reuse_preserves_all_nine_bytes_without_kubo(self): - source, original = self.authoritative_handoff() - before = {path.name: path.read_bytes() for path in source.iterdir()} - result = self.reuse(source) - destination = self.root / "reused-handoff" - self.assertEqual(len(before), 9) - self.assertEqual(before, {path.name: path.read_bytes() for path in destination.iterdir()}) - self.assertEqual(before, {path.name: path.read_bytes() for path in source.iterdir()}) - self.assertEqual(result["model_retention"], original["model_retention"]) - self.assertEqual(result["model_catalog_unsigned"], original["model_catalog_unsigned"]) - self.assertEqual(json.loads((self.output / "upstream-input.json").read_bytes())["model_retention"], original["model_retention"]) - self.assertTrue(all(path.stat().st_mode & 0o077 == 0 for path in destination.iterdir())) - # Another native platform retains the same models/evidence and writes - # its own platform receipt. No model directory identity is recalculated. - self.recipes[0]["platform"] = "darwin-arm64" - self.write_inventory() - second = self.root / "darwin-artifacts" - with patch.object(assets.subprocess, "run") as execute: - other = assets.prepare("darwin-arm64", self.cache, second, published_at=123, - model_publisher_did="did:key:z6MkeTGwHmLmuCmgg4ABYhzWVh6ZX7hTwWt8gguAretUfc9c", model_handoff_input=source, - model_handoff_output=self.root / "darwin-handoff") - execute.assert_not_called() - self.assertEqual(other["model_retention"], original["model_retention"]) - self.assertEqual(before, {path.name: path.read_bytes() for path in (self.root / "darwin-handoff").iterdir()}) - self.assertEqual(json.loads((second / "upstream-input.json").read_bytes())["platform"], "darwin-arm64") - - def test_reuse_excludes_kubo_and_requires_explicit_public_inputs_before_packaging(self): - source, _ = self.authoritative_handoff() - with patch.object(assets.upstream, "package") as package: - for options in ({"model_kubo_bin": self.bin}, {"model_kubo_repo": self.repo}, - {"published_at": True}, {"model_publisher_did": None}, {"model_handoff_output": None}): - with self.subTest(options=options), self.assertRaises(ValueError): - self.reuse(source, **options) - package.assert_not_called() - - def test_authoritative_handoff_refuses_extra_files_and_unprotected_or_linked_inputs(self): - source, _ = self.authoritative_handoff() - (source / "extra").write_bytes(b"unadvertised") - with self.assertRaisesRegex(ValueError, "exactly nine"): - self.reuse(source) - (source / "extra").unlink() - self.output = self.root / "second-artifacts" - car = source / (sorted(assets.MODEL_COMPONENTS)[0] + ".car") - car.chmod(0o644) - with self.assertRaisesRegex(ValueError, "files must be protected"): - self.reuse(source) - car.chmod(0o600) - self.output = self.root / "third-artifacts" - outside = self.root / "retained-car" - car.rename(outside) - car.symlink_to(outside) - with self.assertRaisesRegex(ValueError, "symlink"): - self.reuse(source) - car.unlink() - outside.rename(car) - assets.os.link(car, outside) - self.output = self.root / "hardlink-artifacts" - with self.assertRaisesRegex(ValueError, "single-link"): - self.reuse(source) - outside.unlink() - source.chmod(0o755) - with patch.object(assets.upstream, "package") as package, self.assertRaisesRegex(ValueError, "directory must be protected"): - self.reuse(source) - package.assert_not_called() - - def test_authoritative_handoff_missing_file_and_existing_output_are_refused(self): - source, _ = self.authoritative_handoff() - draft = source / "model-catalog.unsigned.json" - original = draft.read_bytes() - draft.unlink() - with self.assertRaisesRegex(ValueError, "exactly nine"): - self.reuse(source) - self.assertFalse((self.root / "reused-handoff").exists()) - draft.write_bytes(original) - draft.chmod(0o600) - self.output = self.root / "second-artifacts" - destination = self.root / "reused-handoff" - destination.mkdir(mode=0o700) - (destination / "owned-sentinel").write_bytes(b"preserve existing output") - with self.assertRaisesRegex(ValueError, "output must be new"): - self.reuse(source) - self.assertEqual((destination / "owned-sentinel").read_bytes(), b"preserve existing output") - - def test_authoritative_catalogue_exact_timestamp_and_native_closure_are_required(self): - source, _ = self.authoritative_handoff() - with self.assertRaisesRegex(ValueError, "catalogue facts"): - self.reuse(source, published_at=124) - self.output = self.root / "second-artifacts" - draft = source / "model-catalog.unsigned.json" - payload = json.loads(draft.read_bytes()) - payload["entries"][0]["capsule_manifest"]["model_content"]["minimum_memory_mb"] = 512.0 - draft.write_bytes(assets.upstream.canonical(payload)) - with self.assertRaisesRegex(ValueError, "native model closure"): - self.reuse(source) - - def test_authoritative_car_receipt_profile_and_payload_hash_refused(self): - source, _ = self.authoritative_handoff() - name = sorted(assets.MODEL_COMPONENTS)[0] - receipt_path = source / (name + ".car.receipt.json") - receipt = json.loads(receipt_path.read_bytes()) - receipt["kubo_version"] = "0.39.0" - receipt_path.write_bytes(json.dumps(receipt).encode()) - with self.assertRaisesRegex(ValueError, "profile"): - self.reuse(source) - self.output = self.root / "second-artifacts" - receipt["kubo_version"] = "0.40.1" - receipt_path.write_bytes(json.dumps(receipt).encode()) - car = source / (name + ".car") - data = car.read_bytes() - car.write_bytes(b"X" + data[1:]) - with self.assertRaisesRegex(assets.handoff.Refusal, "CAR sha256"): - self.reuse(source) - - def test_authoritative_reuse_floor_and_copy_failure_preserve_source_and_clean_output(self): - source, _ = self.authoritative_handoff() - before = {path.name: path.read_bytes() for path in source.iterdir()} - destination = self.root / "reused-handoff" - def gate(path, additional): - if Path(path) == destination.parent: - raise ValueError("fixture 15% free space refusal") - with patch.object(assets.upstream, "disk_gate", side_effect=gate), self.assertRaisesRegex(ValueError, "15%"): - self.reuse(source) - self.assertFalse(destination.exists()) - self.assertFalse(list(self.root.glob(".reuse-model-handoff-*"))) - self.output = self.root / "second-artifacts" - real_fsync, copies = assets.os.fsync, [] - def sync(fd): - copies.append(fd) - if len(copies) == 3: - raise OSError("fixture copy fsync failure") - return real_fsync(fd) - with patch.object(assets.os, "fsync", side_effect=sync), self.assertRaisesRegex(OSError, "copy fsync"): - self.reuse(source) - self.assertFalse(destination.exists()) - self.assertFalse(list(self.root.glob(".reuse-model-handoff-*"))) - self.assertEqual(before, {path.name: path.read_bytes() for path in source.iterdir()}) - - def rewrite_archive(self, receipt, transform): - path = self.output / receipt["release_path"] - with tarfile.open(path) as archive: - records = [(member, archive.extractfile(member).read()) for member in archive] - records = transform(records) - with tarfile.open(path, "w:gz") as archive: - for member, data in records: - member.size = len(data) - archive.addfile(member, io.BytesIO(data) if member.isreg() else None) - receipt["size"] = path.stat().st_size - receipt["checksum"] = "sha256:" + assets.upstream.digest(path) - - def test_exact_index_bytes_and_regular_file_closure_are_required(self): - recipe, receipt = self.model_package() - self.rewrite_archive(receipt, lambda records: [(member, data + b" " if member.name.endswith("_elastos_object.json") else data) for member, data in records]) - with tempfile.TemporaryDirectory(dir=self.root) as stage, self.assertRaisesRegex(ValueError, "path, size|hash|metadata bytes"): - assets.extract_model(self.output, recipe, receipt, Path(stage)) - self.output = self.root / "second-artifacts" - recipe, receipt = self.model_package() - def link(records): - member = records[0][0] - member.type, member.linkname = tarfile.SYMTYPE, "../outside" - return records - self.rewrite_archive(receipt, link) - with tempfile.TemporaryDirectory(dir=self.root) as stage, self.assertRaisesRegex(ValueError, "exact regular"): - assets.extract_model(self.output, recipe, receipt, Path(stage)) - - def test_extraction_disk_floor_and_modified_staged_bytes_refused(self): - recipe, receipt = self.model_package() - with tempfile.TemporaryDirectory(dir=self.root) as temporary: - stage = Path(temporary) - with patch.object(assets.upstream.shutil, "disk_usage", return_value=SimpleNamespace(total=1000, free=151)): - with self.assertRaisesRegex(ValueError, "15% free"): - assets.extract_model(self.output, recipe, receipt, stage) - self.assertFalse(list(stage.iterdir())) - files, _ = assets.extract_model(self.output, recipe, receipt, stage) - (stage / recipe["entrypoint"]).write_bytes(b"GGUFtampered") - with self.assertRaisesRegex(ValueError, "staged model bytes changed"): - assets.add_model_directory(self.kubo, stage, files, True) - self.assertTrue(self.kubo.connections[-1].closed) - - def test_directory_response_error_bounds_and_root_identity_refused(self): - recipe, receipt = self.model_package() - with tempfile.TemporaryDirectory(dir=self.root) as temporary: - stage = Path(temporary) - files, _ = assets.extract_model(self.output, recipe, receipt, stage) - cases = [(500, b"error", {}), (200, b"x" * (assets.handoff.MAX_CONTROL_BYTES + 1), {}), - (200, b"", {"X-Stream-Error": "fixture stream failure"}), - (200, json.dumps({"Name": "", "Hash": fixture_cid(b"raw", 0x55)}).encode(), {}), - (200, b'{"Message":"fixture error"}', {}), (200, b"", {})] - for status, data, trailers in cases: - response = io.BytesIO(data) - response.status, response.trailers = status, trailers - self.kubo.response_override = response - with self.subTest(status=status, data=data[:64]), self.assertRaises(ValueError): - assets.add_model_directory(self.kubo, stage, files, True) - self.assertTrue(self.kubo.connections[-1].closed) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/release-upstream-assets.py b/scripts/release-upstream-assets.py index f5a309f16..338694e60 100644 --- a/scripts/release-upstream-assets.py +++ b/scripts/release-upstream-assets.py @@ -1,30 +1,12 @@ #!/usr/bin/env python3 -"""Prepare checksum-pinned upstream capsules for a native release worker. - -Opt-in model preparation uses a prepared build Kubo repository. It writes an -unsigned catalogue and streamed CARs beside the artifact directory, or into ---model-handoff-output. Those files and proposed retention descriptors belong -to the operator handoff; consumer manifests keep their existing shape. ---model-handoff-input reuses one authoritative nine-file export on other native -workers. Copies preserve its exact catalogue and factual export timestamps. -""" +"""Prepare checksum-pinned upstream capsules for a native release worker.""" import argparse -import base64 import copy import hashlib import importlib.util import json -import os from pathlib import Path -import re -import stat -import subprocess -import sys -import tarfile -import tempfile -from types import SimpleNamespace -import urllib.parse SOURCE_ROOT = Path(__file__).resolve().parent.parent @@ -32,20 +14,6 @@ upstream = importlib.util.module_from_spec(spec) spec.loader.exec_module(upstream) RECIPE_FILE = SOURCE_ROOT / "scripts/release-upstream-recipes.json" -handoff_spec = importlib.util.spec_from_file_location("model_package_handoff", SOURCE_ROOT / "scripts/model-package-handoff.py") -handoff = importlib.util.module_from_spec(handoff_spec) -sys.modules[handoff_spec.name] = handoff -handoff_spec.loader.exec_module(handoff) -MODEL_COMPONENTS = {"model-qwen3.5-0.8b", "model-qwen3.5-4b", "model-qwen3.5-9b", "model-bonsai-8b-q1"} -# Match directory_hash.rs. Only-hash and pin change together for retention. -MODEL_ADD_OPTIONS = { - "wrap-with-directory": "true", "cid-version": "1", "hash": "sha2-256", - "raw-leaves": "true", "chunker": "size-262144", "trickle": "false", - "max-file-links": "174", "max-directory-links": "0", "max-hamt-fanout": "256", - "inline": "false", "inline-limit": "32", "nocopy": "false", "fscache": "false", - "preserve-mode": "false", "preserve-mtime": "false", "empty-dirs": "false", - "progress": "false", "fast-provide-root": "false", "fast-provide-wait": "false", -} def selected_recipes(platform): @@ -78,367 +46,10 @@ def resolved_recipe(original, llama_arm64_bundle=None): return recipe -def directory_cid(value): - if not isinstance(value, str) or not re.fullmatch(r"b[a-z2-7]{58}", value): - raise ValueError("model root requires a canonical DAG-PB SHA-256 CIDv1") - decoded = base64.b32decode(value[1:].upper() + "=" * (-len(value[1:]) % 8)) - if (decoded[:4] != b"\x01\x70\x12\x20" or len(decoded) != 36 - or "b" + base64.b32encode(decoded).decode().lower().rstrip("=") != value): - raise ValueError("model root requires a canonical DAG-PB SHA-256 CIDv1") - return value - - -def model_kubo(binary, repo, kubo_recipe, kubo_receipt): - binary = upstream.regular(Path(binary).absolute()) - if not os.access(binary, os.X_OK): - raise ValueError("prepared model Kubo binary must be executable") - repo = Path(repo).absolute() - if not repo.is_dir(): - raise ValueError("prepared model Kubo repository is required") - upstream.directory(repo) - if kubo_recipe["version"] != handoff.KUBO_VERSION: - raise ValueError("model Kubo recipe version differs from the canonical profile") - expected = next(item for item in kubo_receipt["object_manifest"]["files"] - if item["path"] == kubo_recipe["entrypoint"]) - if binary.stat().st_size != expected["size"] or upstream.digest(binary) != expected["sha256"]: - raise ValueError("model Kubo binary differs from its pinned capsule entrypoint") - version = subprocess.run([str(binary), "version", "--number"], - env={**os.environ, "IPFS_PATH": str(repo)}, check=True, capture_output=True, text=True, timeout=30) - if version.stdout.strip() != handoff.KUBO_VERSION: - raise ValueError("model Kubo binary version differs from the canonical profile") - api_file = upstream.regular(repo / "api") - address = handoff.read_bounded(api_file, 256).decode().strip() - match = re.fullmatch(r"/ip4/127\.0\.0\.1/tcp/([1-9][0-9]{0,4})", address) - if not match or int(match[1]) > 65535: - raise ValueError("prepared model Kubo API must use IPv4 loopback") - kubo = handoff.KuboApi("http://127.0.0.1:" + match[1], 300) - kubo.verify_profile() - if Path(kubo.repo_path()).resolve() != repo.resolve(): - raise ValueError("model Kubo API repository differs from the prepared repository") - return kubo - - -def extract_model(output, recipe, receipt, stage): - capsule = receipt["capsule_manifest"] - index = receipt["object_manifest"] - expected_capsule = {"schema": "elastos.capsule/v1", "name": recipe["component"], "version": recipe["version"], - "role": "content", "type": "data", "projections": ["content"], "entrypoint": recipe["entrypoint"], - "model_content": recipe["model_content"]} - if (capsule != expected_capsule - or set(index) != {"schema", "kind", "files", "content_digest"} - or index["schema"] != "elastos.content.object.manifest/v1" or index["kind"] != "capsule"): - raise ValueError("model closure metadata differs from its pinned recipe") - files = index["files"] - if not isinstance(files, list) or not 1 <= len(files) <= 32: - raise ValueError("model closure file bound differs from the Runtime contract") - expected = {} - value = hashlib.sha256() - for item in files: - if (not isinstance(item, dict) or set(item) != {"path", "sha256", "size"} - or type(item["size"]) is not int or not 0 < item["size"] <= upstream.MAX_BYTES - or not re.fullmatch(r"[0-9a-f]{64}", item["sha256"]) - or item["path"] in expected or item["path"] == "_elastos_object.json"): - raise ValueError("invalid model closure file record") - upstream.relative(item["path"]) - expected[item["path"]] = item - for field in (item["path"], item["sha256"], str(item["size"])): - value.update(field.encode() + b"\0") - if list(expected) != sorted(expected) or index["content_digest"] != "sha256:" + value.hexdigest(): - raise ValueError("model closure index digest or ordering differs") - capsule_bytes = upstream.canonical(capsule)[:-1] - index_bytes = upstream.canonical(index) - metadata = {"capsule.json": capsule_bytes, "_elastos_object.json": index_bytes} - expected["_elastos_object.json"] = {"size": len(index_bytes), "sha256": hashlib.sha256(index_bytes).hexdigest()} - total = sum(item["size"] for item in expected.values()) - if total > upstream.MAX_BYTES: - raise ValueError("model closure exceeds its size bound") - upstream.disk_gate(stage, total) - archive_path = upstream.regular(output / upstream.relative(receipt["release_path"])) - if (archive_path.stat().st_size != receipt["size"] - or "sha256:" + upstream.digest(archive_path) != receipt["checksum"]): - raise ValueError("model capsule archive differs from its build receipt") - seen = set() - with tarfile.open(archive_path, "r|gz") as archive: - for member in archive: - prefix = recipe["root"] + "/" - if (member.type not in (tarfile.REGTYPE, tarfile.AREGTYPE) or member.pax_headers - or not member.name.startswith(prefix)): - raise ValueError("model capsule requires exact regular closure files") - name = member.name[len(prefix):] - if name not in expected or name in seen or member.size != expected[name]["size"]: - raise ValueError("model archive path, size or duplicate differs from its closure") - seen.add(name) - destination = stage / name - destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - digest = hashlib.sha256() - with archive.extractfile(member) as source, destination.open("xb") as sink: - os.fchmod(sink.fileno(), 0o600) - for chunk in iter(lambda: source.read(upstream.CHUNK), b""): - digest.update(chunk) - sink.write(chunk) - if digest.hexdigest() != expected[name]["sha256"]: - raise ValueError("model archive file hash differs from its closure") - if name in metadata and destination.read_bytes() != metadata[name]: - raise ValueError("model archive metadata bytes differ from its closure") - if seen != set(expected): - raise ValueError("model archive has an incomplete closure") - return expected, total - - -def add_model_directory(kubo, stage, files, only_hash): - kubo.verify_profile() - boundary = "elastos-model-build-" + os.urandom(12).hex() - headers = {name: (f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="{name}"\r\n' - "Content-Type: application/octet-stream\r\n\r\n").encode() for name in sorted(files)} - tail = f"--{boundary}--\r\n".encode() - length = sum(len(headers[name]) + files[name]["size"] + 2 for name in headers) + len(tail) - query = {**MODEL_ADD_OPTIONS, "only-hash": str(only_hash).lower(), "pin": str(not only_hash).lower()} - connection = kubo._connect() - try: - connection.putrequest("POST", "/api/v0/add?" + urllib.parse.urlencode(query)) - connection.putheader("Content-Type", "multipart/form-data; boundary=" + boundary) - connection.putheader("Content-Length", str(length)) - connection.endheaders() - for name in headers: - connection.send(headers[name]) - value, size = hashlib.sha256(), 0 - with upstream.regular(stage / name).open("rb") as source: - remaining = files[name]["size"] - while remaining: - chunk = source.read(min(upstream.CHUNK, remaining)) - if not chunk: - break - size += len(chunk) - remaining -= len(chunk) - value.update(chunk) - connection.send(chunk) - if source.read(1): - raise ValueError("staged model bytes grew during Kubo add") - if size != files[name]["size"] or value.hexdigest() != files[name]["sha256"]: - raise ValueError("staged model bytes changed during Kubo add") - connection.send(b"\r\n") - connection.send(tail) - response = connection.getresponse() - body = response.read(handoff.MAX_CONTROL_BYTES + 1) - if response.status != 200 or len(body) > handoff.MAX_CONTROL_BYTES or response.trailers.get("X-Stream-Error"): - raise ValueError("model Kubo directory add failed or exceeded its response bound") - finally: - connection.close() - root = None - for line in filter(None, body.splitlines()): - record = json.loads(line) - if (not isinstance(record, dict) or set(record) - {"Name", "Hash", "Size"} - or not isinstance(record.get("Name"), str) or not isinstance(record.get("Hash"), str)): - raise ValueError("model Kubo directory add receipt is invalid") - if record["Name"] == "": - if root is not None: - raise ValueError("model Kubo directory add has duplicate roots") - root = directory_cid(record["Hash"]) - if root is None: - raise ValueError("model Kubo directory add has no root") - kubo.verify_profile() - return root - - -def prepare_model_retention(output, handoff_output, models, kubo, repo, published_at, publisher_did): - entries, retention = [], {} - for recipe, receipt in models: - with tempfile.TemporaryDirectory(prefix=".model-closure-", dir=output) as temporary: - stage = Path(temporary) - files, total = extract_model(output, recipe, receipt, stage) - expected = add_model_directory(kubo, stage, files, True) - upstream.disk_gate(repo, total * 2 + upstream.METADATA_BYTES) - actual = add_model_directory(kubo, stage, files, False) - if actual != expected: - raise ValueError("model retained directory CID differs from Kubo only-hash") - if any(entry["cid"] == actual for entry in entries): - raise ValueError("model catalogue requires distinct directory roots") - if not kubo.is_recursively_pinned(actual): - raise ValueError("model directory requires a recursive retention pin") - car = handoff_output / (recipe["component"] + ".car") - receipt_path = Path(str(car) + ".receipt.json") - if any(path.exists() or path.is_symlink() for path in (car, receipt_path, Path(str(car) + ".partial"))): - raise ValueError("model CAR output already exists") - upstream.disk_gate(handoff_output, total * 2 + upstream.METADATA_BYTES) - try: - handoff.cmd_export(SimpleNamespace(kubo_api=kubo.base_url, data_dir=None, timeout_seconds=300, - cid=actual, output=str(car))) - car.chmod(0o600) - receipt_path.chmod(0o600) - upstream.regular(car) - upstream.regular(receipt_path) - verified = handoff.check_car(car, receipt_path, actual) - retention[recipe["component"]] = {"package_cid": actual, - "car": {"release_path": car.name, "checksum": "sha256:" + verified.car_sha256, "size": verified.car_bytes}, - "receipt": {"release_path": receipt_path.name, "checksum": "sha256:" + upstream.digest(receipt_path), - "size": receipt_path.stat().st_size}} - except BaseException: - car.unlink(missing_ok=True) - receipt_path.unlink(missing_ok=True) - raise - entries.append({"cid": actual, "capsule_manifest": receipt["capsule_manifest"], "object_manifest": receipt["object_manifest"]}) - if len({item["cid"] for item in entries}) != 4: - raise ValueError("model catalogue requires four distinct directory roots") - draft = {"schema": "elastos.model.catalog/v1", "published_at": published_at, "expires_at": None, "entries": entries} - draft_bytes = upstream.canonical(draft) - if len(draft_bytes) > handoff.MAX_CATALOG_BYTES: - raise ValueError("unsigned model catalogue exceeds the Runtime byte bound") - draft_path = handoff_output / "model-catalog.unsigned.json" - with draft_path.open("xb") as sink: - os.fchmod(sink.fileno(), 0o600) - sink.write(draft_bytes) - record = {"release_path": draft_path.name, "checksum": "sha256:" + hashlib.sha256(draft_bytes).hexdigest(), - "size": len(draft_bytes), "publisher_did": publisher_did} - return retention, record - - -def public_model_publisher(value): - alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - if type(value) is not str or not re.fullmatch(r"did:key:z6Mk[1-9A-HJ-NP-Za-km-z]{44}", value): - raise ValueError("model publisher requires a canonical public Ed25519 DID") - number = 0 - for char in value[len("did:key:z"):]: - number = number * 58 + alphabet.index(char) - decoded = number.to_bytes((number.bit_length() + 7) // 8, "big") - if len(decoded) != 34 or decoded[:2] != b"\xed\x01": - raise ValueError("model publisher requires a canonical public Ed25519 DID") - return value - - -def protected_handoff_input(path): - path = Path(path).absolute() - if not path.is_dir(): - raise ValueError("authoritative model handoff directory is required") - for entry in (path, *path.parents): - if entry.is_symlink(): - raise ValueError("authoritative model handoff path contains a symlink") - metadata = path.stat() - if metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077: - raise ValueError("authoritative model handoff directory must be protected and owned") - return path - - -def handoff_file(path): - upstream.regular(path) - metadata = path.stat() - if metadata.st_uid != os.geteuid() or metadata.st_mode & 0o077: - raise ValueError("authoritative model handoff files must be protected and owned") - return metadata - - -def handoff_stamp(metadata): - return (metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mode, - metadata.st_uid, metadata.st_nlink, metadata.st_mtime_ns, metadata.st_ctime_ns) - - -def reuse_model_retention(source, destination, models, published_at, publisher_did): - """Copy one operator-selected Kubo export; preserve all nine original bytes.""" - source = protected_handoff_input(source) - if source.resolve() == destination.resolve(): - raise ValueError("authoritative model handoff input and output must differ") - expected_names = {"model-catalog.unsigned.json"} - expected_names.update(name + suffix for name in MODEL_COMPONENTS for suffix in (".car", ".car.receipt.json")) - if {path.name for path in source.iterdir()} != expected_names: - raise ValueError("authoritative model handoff requires exactly nine files") - if destination.exists() or destination.is_symlink(): - raise ValueError("model handoff reuse output must be new") - parent = destination.parent - if not parent.is_dir() or any(path.is_symlink() for path in (parent, *parent.parents)): - raise ValueError("model handoff output requires an existing canonical parent") - metadata = parent.stat() - if metadata.st_uid != os.geteuid() or metadata.st_mode & 0o022: - raise ValueError("model handoff output parent must be owned and protected") - stamps = {name: handoff_file(source / name) for name in expected_names} - draft_path = source / "model-catalog.unsigned.json" - draft_bytes = handoff.read_bounded(draft_path, handoff.MAX_CATALOG_BYTES) - draft = json.loads(draft_bytes) - if (type(draft) is not dict or set(draft) != {"schema", "published_at", "expires_at", "entries"} - or draft["schema"] != handoff.CATALOG_SCHEMA or type(draft["published_at"]) is not int - or draft["published_at"] != published_at or draft["expires_at"] is not None - or type(draft["entries"]) is not list or len(draft["entries"]) != 4 - or draft_bytes != upstream.canonical(draft)): - raise ValueError("authoritative unsigned catalogue facts or canonical bytes differ") - capsules = {recipe["component"]: receipt for recipe, receipt in models} - seen, roots, retention = set(), set(), {} - for entry in draft["entries"]: - if type(entry) is not dict or set(entry) != {"cid", "capsule_manifest", "object_manifest"}: - raise ValueError("authoritative catalogue entry fields refused") - root = directory_cid(entry["cid"]) - capsule = entry["capsule_manifest"] - name = capsule.get("name") if type(capsule) is dict else None - if name not in capsules or name in seen or root in roots: - raise ValueError("authoritative catalogue requires four distinct pinned models") - for key in ("capsule_manifest", "object_manifest"): - if upstream.canonical(entry[key]) != upstream.canonical(capsules[name][key]): - raise ValueError("authoritative catalogue differs from the native model closure") - seen.add(name) - roots.add(root) - car, receipt_path = source / (name + ".car"), source / (name + ".car.receipt.json") - total = sum(item["size"] for item in entry["object_manifest"]["files"]) - if not 0 < stamps[car.name].st_size <= total * 2 + upstream.METADATA_BYTES: - raise ValueError("authoritative model CAR exceeds its closure bound") - verified = handoff.check_car(car, receipt_path, root) - if verified.kubo_version != handoff.KUBO_VERSION or not 0 <= verified.exported_at <= handoff.MAX_BUDGET: - raise ValueError("authoritative model export profile or timestamp refused") - retention[name] = {"package_cid": root, - "car": {"release_path": car.name, "checksum": "sha256:" + verified.car_sha256, "size": verified.car_bytes}, - "receipt": {"release_path": receipt_path.name, "checksum": "sha256:" + upstream.digest(receipt_path), - "size": stamps[receipt_path.name].st_size}} - pins = {record[kind]["release_path"]: record[kind] for record in retention.values() for kind in ("car", "receipt")} - unsigned = {"release_path": draft_path.name, "checksum": "sha256:" + hashlib.sha256(draft_bytes).hexdigest(), - "size": len(draft_bytes), "publisher_did": publisher_did} - pins[draft_path.name] = unsigned - upstream.disk_gate(parent, sum(pin["size"] for pin in pins.values()) + upstream.METADATA_BYTES) - with tempfile.TemporaryDirectory(prefix=".reuse-model-handoff-", dir=parent) as temporary: - stage = Path(temporary) - for name, pin in sorted(pins.items()): - with (source / name).open("rb") as origin, (stage / name).open("xb") as target: - before = os.fstat(origin.fileno()) - if handoff_stamp(before) != handoff_stamp(stamps[name]): - raise ValueError("authoritative model handoff changed before copy") - os.fchmod(target.fileno(), 0o600) - value, size = hashlib.sha256(), 0 - for chunk in iter(lambda: origin.read(upstream.CHUNK), b""): - size += len(chunk) - if size > pin["size"]: - raise ValueError("authoritative model handoff changed during copy") - target.write(chunk) - value.update(chunk) - target.flush() - os.fsync(target.fileno()) - if (handoff_stamp(os.fstat(origin.fileno())) != handoff_stamp(before) - or handoff_stamp(handoff_file(source / name)) != handoff_stamp(before) - or size != pin["size"] or "sha256:" + value.hexdigest() != pin["checksum"]): - raise ValueError("authoritative model handoff changed during copy") - if ({path.name for path in source.iterdir()} != expected_names - or any(handoff_stamp(handoff_file(source / name)) != handoff_stamp(stamps[name]) for name in expected_names)): - raise ValueError("authoritative model handoff changed after admission") - if destination.exists() or destination.is_symlink(): - raise ValueError("model handoff output appeared during copy") - stage.rename(destination) - return retention, unsigned - - -def prepare(platform, cache, output, llama_arm64_bundle=None, model_kubo_bin=None, model_kubo_repo=None, - published_at=None, model_publisher_did=None, model_handoff_output=None, model_handoff_input=None): - reuse = model_handoff_input is not None - model_options = (model_kubo_bin, model_kubo_repo, published_at) - enabled = any(value is not None for value in (*model_options, model_publisher_did, model_handoff_output, model_handoff_input)) - if (reuse and (model_kubo_bin is not None or model_kubo_repo is not None)): - raise ValueError("model handoff reuse excludes Kubo build inputs") - if enabled and (model_publisher_did is None or type(published_at) is not int or not 0 <= published_at <= 2**63 - 1 - or not reuse and any(value is None for value in model_options) - or reuse and (model_publisher_did is None or model_handoff_output is None)): - raise ValueError("model preparation requires a public timestamp and complete fresh or reuse inputs") - if reuse: - model_handoff_input = protected_handoff_input(model_handoff_input) - if enabled: - public_model_publisher(model_publisher_did) +def prepare(platform, cache, output, llama_arm64_bundle=None): recipes = selected_recipes(platform) - if enabled and {recipe["component"] for recipe in recipes if recipe.get("model_content")} != MODEL_COMPONENTS: - raise ValueError("model preparation requires the four pinned release models") output = upstream.directory(output) - external, receipts, resolved = {}, [], {} + external, receipts = {}, [] for original in recipes: recipe = resolved_recipe(original, llama_arm64_bundle) receipt = upstream.package(recipe, cache, output) @@ -452,34 +63,13 @@ def prepare(platform, cache, output, llama_arm64_bundle=None, model_kubo_bin=Non } receipt["recipe_sha256"] = hashlib.sha256(upstream.canonical(original)).hexdigest() receipts.append(receipt) - resolved[recipe["component"]] = (recipe, receipt) result = {"external": external} - if enabled: - if "kubo" not in resolved: - raise ValueError("model preparation requires the pinned Kubo capsule") - models = [resolved[name] for name in sorted(MODEL_COMPONENTS)] - handoff_output = Path(model_handoff_output).absolute() if model_handoff_output else output.with_name(output.name + "-model-handoff") - if handoff_output.resolve() == output.resolve() or output.resolve() in handoff_output.resolve().parents: - raise ValueError("model handoff directory must be outside release artifacts") - if reuse: - retention, draft = reuse_model_retention(model_handoff_input, handoff_output, models, published_at, model_publisher_did) - else: - upstream.directory(handoff_output) - if handoff_output.stat().st_mode & 0o077: - raise ValueError("model handoff output directory must be private") - kubo = model_kubo(model_kubo_bin, model_kubo_repo, *resolved["kubo"]) - retention, draft = prepare_model_retention(output, handoff_output, models, kubo, Path(model_kubo_repo), published_at, model_publisher_did) - # Key-free proposal for the operator. Consumer component descriptors - # stay unchanged until the publication writer contract is approved. - result["model_retention"] = retention - result["model_catalog_unsigned"] = draft # This receipt belongs to native preparation, not the installed manifest. receipt_path = output / "upstream-input.json" with receipt_path.open("x") as stream: json.dump({"schema": "elastos.release-upstream-assets/v1", "platform": platform, "recipes_sha256": hashlib.sha256(RECIPE_FILE.read_bytes()).hexdigest(), - "capsules": receipts, **({"model_catalog_unsigned": result["model_catalog_unsigned"], - "model_retention": retention} if enabled else {})}, stream, indent=2) + "capsules": receipts}, stream, indent=2) stream.write("\n") return result @@ -490,15 +80,8 @@ def main(): parser.add_argument("--cache", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--llama-arm64-bundle", type=Path) - parser.add_argument("--model-kubo-bin", type=Path, help="opt in with the pinned prepared Kubo binary") - parser.add_argument("--model-kubo-repo", type=Path, help="protected build repo with a running loopback API") - parser.add_argument("--published-at", type=int, help="public timestamp for the unsigned model catalogue") - parser.add_argument("--model-publisher-did", help="public intended catalogue signer; this command signs nothing") - parser.add_argument("--model-handoff-input", type=Path, help="reuse the exact nine protected files from an authoritative model export; excludes Kubo args") - parser.add_argument("--model-handoff-output", type=Path, help="separate CAR/catalogue handoff directory; defaults beside output") args = parser.parse_args() - print(json.dumps(prepare(args.platform, args.cache, args.output, args.llama_arm64_bundle, - args.model_kubo_bin, args.model_kubo_repo, args.published_at, args.model_publisher_did, args.model_handoff_output, args.model_handoff_input))) + print(json.dumps(prepare(args.platform, args.cache, args.output, args.llama_arm64_bundle))) if __name__ == "__main__": diff --git a/scripts/setup-source-home.sh b/scripts/setup-source-home.sh index f4a387d82..95d933015 100755 --- a/scripts/setup-source-home.sh +++ b/scripts/setup-source-home.sh @@ -1233,34 +1233,6 @@ components_dest.write_text(json.dumps(manifest, indent=2) + "\n") PY } -install_signed_model_catalog() { - echo "[setup-source-home] install signed model catalog" - COMPONENTS_SRC="${ROOT}/model-catalog.json" \ - COMPONENTS_DEST="${DATA_DIR}/model-catalog.json" \ - COMPONENTS_MANIFEST="${DATA_DIR}/components.json" \ - python3 - <<'PY' -import base64 -import hashlib -import json -import os -import pathlib -import stat - -src = pathlib.Path(os.environ["COMPONENTS_SRC"]) -dest = pathlib.Path(os.environ["COMPONENTS_DEST"]) -manifest = json.loads(pathlib.Path(os.environ["COMPONENTS_MANIFEST"]).read_bytes()) -pin = manifest.get("model_catalog") -if not isinstance(pin, dict) or not isinstance(pin.get("head_cid"), str) or not pin["head_cid"]: - raise SystemExit("source-home components.json is missing model_catalog.head_cid") -data = src.read_bytes() -head = "b" + base64.b32encode(b"\x01\x55\x12\x20" + hashlib.sha256(data).digest()).decode("ascii").lower().rstrip("=") -if head != pin["head_cid"]: - raise SystemExit(f"model-catalog.json head {head} does not match pin {pin['head_cid']}") -dest.write_bytes(data) -dest.chmod(stat.S_IRUSR | stat.S_IWUSR) -PY -} - install_local_model_engine() { local mode="${SETUP_SOURCE_HOME_INSTALL_LLAMA_SERVER:-0}" @@ -1541,7 +1513,6 @@ source_home_binary_names | while IFS= read -r provider; do install -m 755 "$(source_home_provider_bin "${provider}")" "${DATA_DIR}/bin/${provider}" done stamp_source_home_components_manifest -install_signed_model_catalog echo "[setup-source-home] install content publish backend before final manifest stamp" install_content_publish_backend