From dcc22fe6cc3a5c6233eda3bae7b00c36c226893b Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 00:50:31 +0000 Subject: [PATCH 1/6] feat(cold-store-reaper): sync-manifest to D1 + DB-backup class sync-manifest pushes the local SQLite manifest (objects, bundles, members, transfer_ledger, meta) to the Cloudflare D1 database cold-store-manifest via the D1 REST query API, as batched idempotent upserts keyed by each table's primary key from schema.sql. Token from CLOUDFLARE_API_TOKEN env only; --dry-run prints statement counts. DB backups (design section 5): ensure-folders now also creates supabase/, turso/, d1/ under db-backups/; put-backup uploads one dump unbundled with a dated name and a backup-upload ledger row; list-backups lists newest-first. Retention enforcement still pending. --- pkgs/cold-store-reaper/cold_store.py | 265 ++++++++++++++++++++++++++- 1 file changed, 263 insertions(+), 2 deletions(-) diff --git a/pkgs/cold-store-reaper/cold_store.py b/pkgs/cold-store-reaper/cold_store.py index b9c875906..bf11eb081 100755 --- a/pkgs/cold-store-reaper/cold_store.py +++ b/pkgs/cold-store-reaper/cold_store.py @@ -42,6 +42,19 @@ ROOT_FOLDER_NAME = "Cold Store Reaper" SUBFOLDERS = ("bundles", "manifests", "db-backups") +# DB-backup class (DESIGN.md section 5): nightly dumps live unbundled under +# db-backups//, one subfolder per database. +BACKUP_DBS = ("supabase", "turso", "d1") + +# Manifest sync target (DESIGN.md section 3): the hosted copy of this schema lives +# in Cloudflare D1. The token comes from the environment only; the account +# and database IDs are non-secret and default to the cold-store database. +DEFAULT_D1_BASE = "https://api.cloudflare.com/client/v4" +DEFAULT_CF_ACCOUNT_ID = "234b866b99ce2b142d7963d8d139cbfa" +DEFAULT_CF_D1_DATABASE_ID = "15d5f460-835c-4f24-b229-222908a5b30c" +MANIFEST_TABLES = ("objects", "bundles", "members", "transfer_ledger", "meta") +D1_SYNC_BATCH_ROWS = 50 + # Design constant: bundles are sealed at this size in the full reaper; # `pack` packs exactly what it is given regardless of size. BUNDLE_TARGET_BYTES = 8 * 1024 * 1024 * 1024 # 8 GiB @@ -106,7 +119,13 @@ def http_request(method, url, headers=None, data=None, timeout=120): class ColdStore: def __init__( - self, home: Path, token_url=None, drive_base=None, upload_base=None, zstd=None + self, + home: Path, + token_url=None, + drive_base=None, + upload_base=None, + zstd=None, + d1_base=None, ): self.home = Path(home) self.credentials_path = self.home / "credentials.json" @@ -122,6 +141,9 @@ def __init__( self.upload_base = ( upload_base or os.environ.get("COLD_STORE_UPLOAD_BASE", DEFAULT_UPLOAD_BASE) ).rstrip("/") + self.d1_base = ( + d1_base or os.environ.get("COLD_STORE_D1_BASE", DEFAULT_D1_BASE) + ).rstrip("/") candidate = zstd or os.environ.get("COLD_STORE_ZSTD") or "/usr/local/bin/zstd" if not Path(candidate).exists(): candidate = "zstd" # fall back to PATH lookup @@ -339,6 +361,12 @@ def ensure_folders(self) -> dict: folders[sub] = self._find_folder( sub, folders["root"] ) or self._create_folder(sub, folders["root"]) + for db in BACKUP_DBS: + key = f"db-backups/{db}" + if not folders.get(key): + folders[key] = self._find_folder( + db, folders["db-backups"] + ) or self._create_folder(db, folders["db-backups"]) state["folders"] = folders self.save_state(state) return folders @@ -613,6 +641,193 @@ def fetch(self, object_hash: str, out_path: str) -> dict: "bundle_id": row["bundle_id"], } + # ------------------------------------------------------------- backups + def put_backup(self, db_name: str, file: str) -> dict: + """Upload one DB dump, unbundled, into db-backups// (DESIGN section 5). + + The Drive name is dated (--) so a + folder listing is already a chronological catalogue. Retention + enforcement (30 daily / 12 weekly / 12 monthly, trash-not-delete) is + NOT implemented yet — see README. + """ + if db_name not in BACKUP_DBS: + raise ColdStoreError( + f"unknown backup db {db_name!r} — expected one of {', '.join(BACKUP_DBS)}" + ) + src = Path(file) + if not src.is_file(): + raise ColdStoreError(f"not a file: {src}") + folder_id = self._folder_id(f"db-backups/{db_name}") + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + name = f"{db_name}-{stamp}-{src.name}" + file_id = self._resumable_upload(src, name, folder_id) + size = src.stat().st_size + conn = self.db() + with conn: + self.ledger_add(conn, "upload", size, "backup-upload", name) + conn.close() + return { + "db": db_name, + "name": name, + "drive_file_id": file_id, + "bytes": size, + } + + def list_backups(self, db_name: str | None = None) -> list[dict]: + if db_name is not None and db_name not in BACKUP_DBS: + raise ColdStoreError( + f"unknown backup db {db_name!r} — expected one of {', '.join(BACKUP_DBS)}" + ) + dbs = [db_name] if db_name else list(BACKUP_DBS) + out = [] + for db in dbs: + folder_id = self._folder_id(f"db-backups/{db}") + q = f"'{folder_id}' in parents and trashed = false" + url = ( + self.drive_base + + "/files?" + + urllib.parse.urlencode( + { + "q": q, + "fields": "files(id,name,size,createdTime)", + "orderBy": "createdTime desc", + "pageSize": 1000, + } + ) + ) + status, _h, resp = http_request("GET", url, headers=self._auth_headers()) + if status != 200: + raise ColdStoreError(f"files.list for backups failed (HTTP {status})") + for f in json.loads(resp).get("files", []): + out.append( + { + "db": db, + "id": f["id"], + "name": f["name"], + "size": int(f.get("size") or 0), + "created_time": f.get("createdTime", ""), + } + ) + # Drive was asked for createdTime desc; sort again so the merged + # multi-db listing is deterministic even if a backend ignores it. + out.sort(key=lambda r: r["created_time"], reverse=True) + return out + + # --------------------------------------------------------- manifest sync + @staticmethod + def _table_shape(conn, table: str) -> tuple[list[str], list[str]]: + rows = conn.execute(f"PRAGMA table_info({table})").fetchall() + cols = [r["name"] for r in rows] + pk = [ + r["name"] + for r in sorted((r for r in rows if r["pk"]), key=lambda r: r["pk"]) + ] + return cols, pk + + @staticmethod + def _upsert_sql(table: str, cols: list[str], pk: list[str], nrows: int) -> str: + one_group = "(" + ", ".join("?" for _ in cols) + ")" + groups = ", ".join(one_group for _ in range(nrows)) + updates = [c for c in cols if c not in pk] + if updates: + action = "DO UPDATE SET " + ", ".join(f"{c}=excluded.{c}" for c in updates) + else: + action = "DO NOTHING" + return ( + f"INSERT INTO {table} ({', '.join(cols)}) VALUES {groups} " + f"ON CONFLICT({', '.join(pk)}) {action}" + ) + + def build_sync_statements(self) -> tuple[list[dict], dict]: + """Read the local manifest and build batched D1 upsert statements. + + Returns (statements, per-table {"rows": n, "statements": m}). Each + statement carries at most D1_SYNC_BATCH_ROWS row-groups so a re-sync + is a handful of HTTP calls, and every statement is an idempotent + upsert keyed by the table's primary key from schema.sql. + """ + conn = self.db() + statements: list[dict] = [] + counts: dict[str, dict] = {} + for table in MANIFEST_TABLES: + cols, pk = self._table_shape(conn, table) + order = ", ".join(pk) if pk else cols[0] + rows = conn.execute( + f"SELECT {', '.join(cols)} FROM {table} ORDER BY {order}" + ).fetchall() + table_statements = 0 + for i in range(0, len(rows), D1_SYNC_BATCH_ROWS): + chunk = rows[i : i + D1_SYNC_BATCH_ROWS] + params: list = [] + for row in chunk: + params.extend(row[c] for c in cols) + statements.append( + { + "table": table, + "sql": self._upsert_sql(table, cols, pk, len(chunk)), + "params": params, + "rows": len(chunk), + } + ) + table_statements += 1 + counts[table] = {"rows": len(rows), "statements": table_statements} + conn.close() + return statements, counts + + def sync_manifest(self, dry_run: bool = False) -> dict: + statements, counts = self.build_sync_statements() + total = { + "tables": counts, + "rows": sum(c["rows"] for c in counts.values()), + "statements": len(statements), + "dry_run": dry_run, + } + if dry_run: + return total + token = os.environ.get("CLOUDFLARE_API_TOKEN") + if not token: + raise ColdStoreError( + "CLOUDFLARE_API_TOKEN is not set — materialize it from " + "Bitwarden into the environment first (it is never stored)" + ) + state = self.load_state() + account_id = ( + os.environ.get("COLD_STORE_CF_ACCOUNT_ID") + or state.get("cf_account_id") + or DEFAULT_CF_ACCOUNT_ID + ) + database_id = ( + os.environ.get("COLD_STORE_CF_D1_DATABASE_ID") + or state.get("cf_d1_database_id") + or DEFAULT_CF_D1_DATABASE_ID + ) + url = f"{self.d1_base}/accounts/{account_id}/d1/database/{database_id}/query" + for stmt in statements: + status, _h, resp = http_request( + "POST", + url, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + data=json.dumps( + {"sql": stmt["sql"], "params": stmt["params"]} + ).encode(), + ) + if status != 200: + raise ColdStoreError( + f"D1 sync failed (HTTP {status}) on table {stmt['table']} — " + "statements already sent are idempotent upserts, re-run to resume" + ) + payload = json.loads(resp) if resp else {} + if payload and not payload.get("success", True): + errors = payload.get("errors") or [{}] + raise ColdStoreError( + f"D1 sync rejected a {stmt['table']} statement: " + f"{errors[0].get('message', 'unknown error')}" + ) + return total + # ---------------------------------------------------------------- status def status_data(self) -> dict: conn = self.db() @@ -706,6 +921,24 @@ def main(argv=None) -> int: "status", help="manifest counts, cold bytes, last-24h transfers, hard-stop check", ) + p_put_backup = sub.add_parser( + "put-backup", help="upload one DB dump, unbundled, into db-backups//" + ) + p_put_backup.add_argument("db", choices=BACKUP_DBS) + p_put_backup.add_argument("file") + p_list_backups = sub.add_parser( + "list-backups", help="list DB backups in Drive, newest first" + ) + p_list_backups.add_argument("--db", choices=BACKUP_DBS, default=None) + p_sync = sub.add_parser( + "sync-manifest", + help="push the local manifest to the D1 cold-store-manifest database (idempotent upserts)", + ) + p_sync.add_argument( + "--dry-run", + action="store_true", + help="print statement counts only; contact nothing", + ) args = parser.parse_args(argv) store = build_store(args) try: @@ -728,7 +961,10 @@ def main(argv=None) -> int: ) elif args.cmd == "ensure-folders": folders = store.ensure_folders() - for key in ("root",) + SUBFOLDERS: + keys = ( + ("root",) + SUBFOLDERS + tuple(f"db-backups/{db}" for db in BACKUP_DBS) + ) + for key in keys: print(f"{key}: {folders[key]}") elif args.cmd == "pack": result = store.pack(args.files) @@ -763,6 +999,31 @@ def main(argv=None) -> int: f"daily upload hard stop: {_human(d['daily_upload_hard_stop_bytes'])} — " f"{'OK' if d['hard_stop_ok'] else 'EXCEEDED'}" ) + elif args.cmd == "put-backup": + result = store.put_backup(args.db, args.file) + print( + f"backup {result['name']}: {_human(result['bytes'])}, " + f"uploaded to Drive as {result['drive_file_id']}" + ) + elif args.cmd == "list-backups": + rows = store.list_backups(args.db) + if not rows: + print("no backups found") + for r in rows: + print( + f"{r['created_time']} {r['db']:<8} {_human(r['size']):>24} " + f"{r['name']} ({r['id']})" + ) + elif args.cmd == "sync-manifest": + result = store.sync_manifest(dry_run=args.dry_run) + for table in MANIFEST_TABLES: + c = result["tables"][table] + print(f"{table}: {c['rows']} row(s) in {c['statements']} statement(s)") + verb = "would send" if result["dry_run"] else "sent" + print( + f"total: {verb} {result['statements']} statement(s), " + f"{result['rows']} row(s)" + ) except ColdStoreError as e: print(f"error: {e}", file=sys.stderr) return 1 From 559c3599df679ebbd7370f5616f86b23e93f9cca Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 00:50:31 +0000 Subject: [PATCH 2/6] test(cold-store-reaper): fake-D1 sync tests, backup tests, worker decoder proof Extends the fake Drive (file listings with createdTime) and adds a stubbed D1 REST server: dry-run counts contact nothing, upsert SQL shape + auth + path asserted, HTTP-500 and missing-token paths. Backup tests cover subfolder creation, dated naming, folder routing, ledger rows, newest-first listing. TestWorkerDecoder runs the worker's node proof when node + node_modules are present (skips otherwise). The v1 folder-count assertion moves 4 -> 7 for the new subfolders. --- .../tests/test_cold_store.py | 252 +++++++++++++++++- 1 file changed, 249 insertions(+), 3 deletions(-) diff --git a/pkgs/cold-store-reaper/tests/test_cold_store.py b/pkgs/cold-store-reaper/tests/test_cold_store.py index dbf130bbe..b1c4ccdb4 100644 --- a/pkgs/cold-store-reaper/tests/test_cold_store.py +++ b/pkgs/cold-store-reaper/tests/test_cold_store.py @@ -9,7 +9,9 @@ import json import os import re +import shutil import stat +import subprocess import sys import tempfile import threading @@ -17,6 +19,7 @@ import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import cold_store @@ -30,14 +33,20 @@ class FakeDrive: def __init__(self): self.folders = [] # {id, name, parents} self.files = {} # id -> bytes - self.file_meta = {} # id -> {name, parents} + self.file_meta = {} # id -> {name, parents, createdTime} self.sessions = {} # sid -> {meta, chunks: {start: bytes}, total} self.counter = 0 + self.clock = 0 def next_id(self, prefix): self.counter += 1 return f"{prefix}-{self.counter}" + def stamp(self): + """Monotonic fake createdTime so newest-first ordering is testable.""" + self.clock += 1 + return f"2026-01-01T00:{self.clock // 60:02d}:{self.clock % 60:02d}.000Z" + def make_handler(fake: FakeDrive): class Handler(BaseHTTPRequestHandler): @@ -114,6 +123,18 @@ def do_GET(self): if parent_m and parent_m.group(1) not in folder["parents"]: continue out.append({"id": folder["id"], "name": folder["name"]}) + if parent_m and not name_m: + # parent-only listing (list-backups): include real files + for fid, meta in fake.file_meta.items(): + if parent_m.group(1) in meta.get("parents", []): + out.append( + { + "id": fid, + "name": meta["name"], + "size": str(len(fake.files[fid])), + "createdTime": meta.get("createdTime", ""), + } + ) self._send(200, {"files": out}) return m = re.fullmatch(r"/drive/v3/files/([^/]+)", parsed.path) @@ -165,7 +186,10 @@ def do_PUT(self): data = b"".join(session["chunks"][k] for k in sorted(session["chunks"])) fid = fake.next_id("file") fake.files[fid] = data - fake.file_meta[fid] = session["meta"] + fake.file_meta[fid] = { + **session["meta"], + "createdTime": fake.stamp(), + } self._send(200, {"id": fid, "name": session["meta"]["name"]}) else: self._send(308, b"", headers={"Range": f"bytes=0-{received - 1}"}) @@ -269,7 +293,8 @@ def test_ensure_folders_idempotent(self): first = self.store.ensure_folders() second = self.store.ensure_folders() self.assertEqual(first, second) - self.assertEqual(len(self.fake.folders), 4) # root + 3 subfolders + # root + 3 subfolders + 3 db-backup subfolders (supabase/turso/d1) + self.assertEqual(len(self.fake.folders), 7) def test_upload_verify_and_fetch_roundtrip(self): # force multi-chunk resumable uploads against the fake (308 path) @@ -348,5 +373,226 @@ def test_status_arithmetic_and_rolling_window(self): conn.close() +# ------------------------------------------------------------------ fake D1 +class FakeD1: + """Captures D1 REST query calls; can be told to fail.""" + + def __init__(self): + self.requests = [] # {path, authorization, sql, params} + self.fail = False + + +def make_d1_handler(fake: FakeD1): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *a): # silence + pass + + def _send(self, code, payload): + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + payload = json.loads(body or b"{}") + fake.requests.append( + { + "path": self.path, + "authorization": self.headers.get("Authorization"), + "sql": payload.get("sql", ""), + "params": payload.get("params", []), + } + ) + if fake.fail: + self._send( + 500, {"success": False, "errors": [{"message": "fake failure"}]} + ) + return + self._send( + 200, + { + "success": True, + "errors": [], + "messages": [], + "result": [{"success": True, "results": []}], + }, + ) + + return Handler + + +class TestBackups(ColdStoreTestBase): + def test_ensure_folders_creates_backup_subfolders_once(self): + first = self.store.ensure_folders() + for db in cold_store.BACKUP_DBS: + self.assertIn(f"db-backups/{db}", first) + count = len(self.fake.folders) + second = self.store.ensure_folders() + self.assertEqual(first, second) + self.assertEqual(len(self.fake.folders), count) + + def test_put_backup_naming_routing_ledger(self): + dump = self.write_src("dump.sql", b"-- fake supabase dump\n" * 10) + result = self.store.put_backup("supabase", str(dump)) + self.assertRegex(result["name"], r"^supabase-\d{8}T\d{6}Z-dump\.sql$") + folders = self.store.ensure_folders() + meta = self.fake.file_meta[result["drive_file_id"]] + self.assertEqual(meta["parents"], [folders["db-backups/supabase"]]) + self.assertEqual(meta["name"], result["name"]) + self.assertEqual(self.fake.files[result["drive_file_id"]], dump.read_bytes()) + conn = self.store.db() + rows = conn.execute( + "SELECT * FROM transfer_ledger WHERE kind='backup-upload'" + ).fetchall() + conn.close() + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["direction"], "upload") + self.assertEqual(rows[0]["bytes"], dump.stat().st_size) + self.assertEqual(rows[0]["ref"], result["name"]) + + def test_put_backup_rejects_unknown_db(self): + dump = self.write_src("dump.sql", b"x") + with self.assertRaises(cold_store.ColdStoreError): + self.store.put_backup("oracle", str(dump)) + + def test_list_backups_newest_first_and_filtered(self): + a = self.write_src("a.sql", b"a") + b = self.write_src("b.sql", b"b") + self.store.put_backup("supabase", str(a)) + self.store.put_backup("turso", str(b)) + all_rows = self.store.list_backups() + self.assertEqual(len(all_rows), 2) + self.assertEqual(all_rows[0]["db"], "turso") # later createdTime first + self.assertEqual(all_rows[1]["db"], "supabase") + only = self.store.list_backups("supabase") + self.assertEqual(len(only), 1) + self.assertEqual(only[0]["db"], "supabase") + + +class TestSyncManifest(ColdStoreTestBase): + def setUp(self): + super().setUp() + self.d1 = FakeD1() + self.d1_server = ThreadingHTTPServer(("127.0.0.1", 0), make_d1_handler(self.d1)) + self.d1_thread = threading.Thread( + target=self.d1_server.serve_forever, daemon=True + ) + self.d1_thread.start() + self.store.d1_base = f"http://127.0.0.1:{self.d1_server.server_address[1]}" + a = self.write_src("a.txt", b"sync me " * 10) + packed = self.store.pack([str(a)]) + self.store.upload(packed["bundle_id"]) + + def tearDown(self): + self.d1_server.shutdown() + self.d1_server.server_close() + super().tearDown() + + def _row_counts(self): + conn = self.store.db() + counts = { + t: conn.execute(f"SELECT count(*) n FROM {t}").fetchone()["n"] + for t in cold_store.MANIFEST_TABLES + } + conn.close() + return counts + + def _env_without_token(self): + return {k: v for k, v in os.environ.items() if k != "CLOUDFLARE_API_TOKEN"} + + def test_dry_run_counts_without_contacting_d1(self): + with mock.patch.dict(os.environ, self._env_without_token(), clear=True): + result = self.store.sync_manifest(dry_run=True) + self.assertTrue(result["dry_run"]) + expected = self._row_counts() + for table, n in expected.items(): + self.assertEqual(result["tables"][table]["rows"], n) + self.assertEqual(result["tables"][table]["statements"], 1 if n else 0) + self.assertEqual(result["rows"], sum(expected.values())) + self.assertEqual(self.d1.requests, []) # nothing was contacted + + def test_sync_posts_idempotent_upserts(self): + with mock.patch.dict(os.environ, {"CLOUDFLARE_API_TOKEN": "fake-cf-token"}): + result = self.store.sync_manifest() + again = self.store.sync_manifest() # re-sync must be safe + identical + self.assertFalse(result["dry_run"]) + self.assertEqual(result["rows"], again["rows"]) + self.assertEqual(len(self.d1.requests), 2 * result["statements"]) + expected_path = ( + f"/accounts/{cold_store.DEFAULT_CF_ACCOUNT_ID}" + f"/d1/database/{cold_store.DEFAULT_CF_D1_DATABASE_ID}/query" + ) + for req in self.d1.requests: + self.assertEqual(req["authorization"], "Bearer fake-cf-token") + self.assertIn(expected_path, req["path"]) + self.assertTrue(req["sql"].startswith("INSERT INTO ")) + self.assertIn("ON CONFLICT", req["sql"]) + by_table = {} + for req in self.d1.requests[: result["statements"]]: + m = re.match(r"INSERT INTO (\w+)", req["sql"]) + by_table[m.group(1)] = req + self.assertIn("ON CONFLICT(hash) DO UPDATE SET", by_table["objects"]["sql"]) + self.assertIn( + "ON CONFLICT(bundle_id, object_hash) DO UPDATE SET", + by_table["members"]["sql"], + ) + self.assertIn("offset=excluded.offset", by_table["members"]["sql"]) + self.assertIn( + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + by_table["meta"]["sql"], + ) + col_counts = { + "objects": 8, + "bundles": 10, + "members": 4, + "transfer_ledger": 6, + "meta": 2, + } + for table, req in by_table.items(): + self.assertEqual(len(req["params"]) % col_counts[table], 0) + self.assertGreater(len(req["params"]), 0) + + def test_sync_http_error_is_clear(self): + self.d1.fail = True + with mock.patch.dict(os.environ, {"CLOUDFLARE_API_TOKEN": "fake-cf-token"}): + with self.assertRaises(cold_store.ColdStoreError) as ctx: + self.store.sync_manifest() + self.assertIn("HTTP 500", str(ctx.exception)) + + def test_sync_requires_token(self): + with mock.patch.dict(os.environ, self._env_without_token(), clear=True): + with self.assertRaises(cold_store.ColdStoreError): + self.store.sync_manifest() + + +class TestWorkerDecoder(unittest.TestCase): + """Local proof for the rehydrator Worker: the exact decoder module the + Worker bundles (worker/src/decoder.js, fzstd) decodes one member frame + out of a real bundle produced by `cold-store pack`, sha256-checked by + worker/test/run-local-proof.mjs. Skips when node or the worker's + node_modules are unavailable (e.g. a bare repo checkout in CI).""" + + def test_worker_decoder_decodes_packed_frame(self): + worker_dir = Path(__file__).resolve().parent.parent / "worker" + node = shutil.which("node") + if not node or not (worker_dir / "node_modules" / "fzstd").is_dir(): + self.skipTest("node or worker node_modules (fzstd) unavailable") + proc = subprocess.run( + [node, "test/run-local-proof.mjs"], + cwd=worker_dir, + capture_output=True, + text=True, + timeout=180, + ) + self.assertEqual(proc.returncode, 0, proc.stderr[-500:]) + self.assertIn("PASS", proc.stdout) + + if __name__ == "__main__": unittest.main(verbosity=2) From 0d3314858a9c81275a5b17c617a28f232954bac4 Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 00:51:27 +0000 Subject: [PATCH 3/6] feat(cold-store-reaper): rehydrator Worker (design section 6, v1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /fetch/ (Bearer REHYDRATOR_API_KEY) looks the object up in the D1 MANIFEST binding, then a BundleFlight Durable Object (one per bundle) single-flights the pull: Google refresh-token exchange cached in DO storage, Drive Range GET of exactly the member's zstd frame, fzstd decompression, sha256 verify before serving — 404 unknown sha, 401 unauthenticated, 502 on any Drive/decode/hash failure, bytes never returned unverified. GET /health is unauthenticated. zstd decoding is vendored via fzstd (pure JS) bundled by esbuild into the single self-contained dist/worker.js (force-added past the repo's **/dist ignore, as this artifact is the deployable). src/decoder.js is the one decoder module; test/run-local-proof.mjs packs a real bundle with the Python reaper and decodes a member frame with that module. wrangler.toml.example documents the bindings for the API-driven deploy. --- pkgs/cold-store-reaper/worker/README.md | 79 ++ pkgs/cold-store-reaper/worker/dist/worker.js | 779 ++++++++++++++++++ .../worker/package-lock.json | 508 ++++++++++++ pkgs/cold-store-reaper/worker/package.json | 17 + pkgs/cold-store-reaper/worker/src/decoder.js | 13 + pkgs/cold-store-reaper/worker/src/index.js | 206 +++++ .../worker/test/run-local-proof.mjs | 82 ++ .../worker/wrangler.toml.example | 33 + 8 files changed, 1717 insertions(+) create mode 100644 pkgs/cold-store-reaper/worker/README.md create mode 100644 pkgs/cold-store-reaper/worker/dist/worker.js create mode 100644 pkgs/cold-store-reaper/worker/package-lock.json create mode 100644 pkgs/cold-store-reaper/worker/package.json create mode 100644 pkgs/cold-store-reaper/worker/src/decoder.js create mode 100644 pkgs/cold-store-reaper/worker/src/index.js create mode 100644 pkgs/cold-store-reaper/worker/test/run-local-proof.mjs create mode 100644 pkgs/cold-store-reaper/worker/wrangler.toml.example diff --git a/pkgs/cold-store-reaper/worker/README.md b/pkgs/cold-store-reaper/worker/README.md new file mode 100644 index 000000000..ff3888912 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/README.md @@ -0,0 +1,79 @@ +# cold-store-rehydrator (Worker) + +The read path of the Drive cold store (design section 6): given an object's +sha256, pull its zstd frame out of its Drive bundle by byte range, +decompress it, verify the hash, and return the original bytes. This is the +v1 slice — it serves the object on the same request; filling R2 first and +flipping the manifest state back to `hot` belong to the caller's layer. + +## Endpoints + +| Route | Auth | Result | +| --------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `GET /health` | none | `{"ok": true}` | +| `GET /fetch/` | `Authorization: Bearer ` | `200` + object bytes; `401` without the key; `404` unknown sha; `502` on any Drive/decode/hash failure | + +Successful fetches carry `X-Cold-Store-Sha256`, `X-Cold-Store-Bundle`, and +`Content-Length` headers. Bytes are **never** returned unverified — a sha256 +mismatch after decompression is a 502, not a best-effort serve. + +## How it works + +1. The Worker looks the sha up in the D1 binding **MANIFEST** (database + `cold-store-manifest`): `objects → members → bundles` yields the Drive + file id, the frame's offset/length inside the bundle, and the object's + expected sha256/size. The manifest is kept current by the reaper's + `cold-store sync-manifest`. +2. The job is handed to the Durable Object **BUNDLE_FLIGHT**, keyed by + bundle id (`idFromName(bundle_id)`). One instance per bundle + single-flights concurrent work: identical concurrent member requests + join one in-flight Drive pull; the Google access token is cached in DO + storage until ~60 s before expiry. +3. Inside the DO: refresh-token exchange at + `https://oauth2.googleapis.com/token`, then + `GET https://www.googleapis.com/drive/v3/files/?alt=media` with + `Range: bytes=-`. The single frame is + decompressed with [fzstd](https://www.npmjs.com/package/fzstd) + (pure-JS, decompress-only) and sha256-verified. + +## Layout + +``` +src/index.js Worker entry point + BundleFlight Durable Object +src/decoder.js the one zstd-frame decoder (fzstd) — shared with the proof test +test/run-local-proof.mjs npm test: packs a real bundle with the Python + reaper, decodes a member frame with src/decoder.js, + asserts the sha256 (the decoder that ships is the + decoder that is tested) +dist/worker.js esbuild bundle — this is what gets deployed +wrangler.toml.example binding reference (deploy is API-driven, see below) +``` + +## Build + +```sh +npm ci +npm run build # esbuild -> dist/worker.js (single self-contained file) +npm test # local frame-decode proof (needs python3 + the zstd CLI) +``` + +Commit `package.json`, `package-lock.json`, `src/`, and the built `dist/` +together — the operator deploys `dist/worker.js` as-is. + +## Deploy (operator, via the Cloudflare REST API) + +Wrangler does not deploy this in the current environment; the operator: + +1. Uploads `dist/worker.js` as the Worker script (PUT + `/accounts//workers/scripts/cold-store-rehydrator` with the D1 and + DO bindings from `wrangler.toml.example` in the metadata), including the + DO migration `new_sqlite_classes = ["BundleFlight"]` on first deploy. +2. Sets the secrets as `secret_text` bindings: `REHYDRATOR_API_KEY` + (operator-chosen), `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, + `GOOGLE_REFRESH_TOKEN` (values from Bitwarden — see the reaper README's + credential notes; the Google pair/refresh token are the same grant the + reaper uses). +3. Enables the workers.dev subdomain for the script, or attaches a route. + +Until `sync-manifest` has run at least once against a manifest containing +verified bundles, every `/fetch/` is a well-formed 404. diff --git a/pkgs/cold-store-reaper/worker/dist/worker.js b/pkgs/cold-store-reaper/worker/dist/worker.js new file mode 100644 index 000000000..1bdd09174 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/dist/worker.js @@ -0,0 +1,779 @@ +// node_modules/fzstd/esm/index.mjs +var ab = ArrayBuffer; +var u8 = Uint8Array; +var u16 = Uint16Array; +var i16 = Int16Array; +var i32 = Int32Array; +var slc = function(v, s, e) { + if (u8.prototype.slice) + return u8.prototype.slice.call(v, s, e); + if (s == null || s < 0) + s = 0; + if (e == null || e > v.length) + e = v.length; + var n = new u8(e - s); + n.set(v.subarray(s, e)); + return n; +}; +var fill = function(v, n, s, e) { + if (u8.prototype.fill) + return u8.prototype.fill.call(v, n, s, e); + if (s == null || s < 0) + s = 0; + if (e == null || e > v.length) + e = v.length; + for (; s < e; ++s) + v[s] = n; + return v; +}; +var cpw = function(v, t, s, e) { + if (u8.prototype.copyWithin) + return u8.prototype.copyWithin.call(v, t, s, e); + if (s == null || s < 0) + s = 0; + if (e == null || e > v.length) + e = v.length; + while (s < e) { + v[t++] = v[s++]; + } +}; +var ec = [ + "invalid zstd data", + "window size too large (>2046MB)", + "invalid block type", + "FSE accuracy too high", + "match distance too far back", + "unexpected EOF" +]; +var err = function(ind, msg, nt) { + var e = new Error(msg || ec[ind]); + e.code = ind; + if (Error.captureStackTrace) + Error.captureStackTrace(e, err); + if (!nt) + throw e; + return e; +}; +var rb = function(d, b, n) { + var i = 0, o = 0; + for (; i < n; ++i) + o |= d[b++] << (i << 3); + return o; +}; +var b4 = function(d, b) { + return (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0; +}; +var rzfh = function(dat, w) { + var n3 = dat[0] | dat[1] << 8 | dat[2] << 16; + if (n3 == 3126568 && dat[3] == 253) { + var flg = dat[4]; + var ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6; + if (flg & 8) + err(0); + var bt = 6 - ss; + var db = df == 3 ? 4 : df; + var di = rb(dat, bt, db); + bt += db; + var fsb = fcf ? 1 << fcf : ss; + var fss = rb(dat, bt, fsb) + (fcf == 1 && 256); + var ws = fss; + if (!ss) { + var wb = 1 << 10 + (dat[5] >> 3); + ws = wb + (wb >> 3) * (dat[5] & 7); + } + if (ws > 2145386496) + err(1); + var buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12); + buf[0] = 1, buf[4] = 4, buf[8] = 8; + return { + b: bt + fsb, + y: 0, + l: 0, + d: di, + w: w && w != 1 ? w : buf.subarray(12), + e: ws, + o: new i32(buf.buffer, 0, 3), + u: fss, + c: cc, + m: Math.min(131072, ws) + }; + } else if ((n3 >> 4 | dat[3] << 20) == 25481893) { + return b4(dat, 4) + 8; + } + err(0); +}; +var msb = function(val) { + var bits = 0; + for (; 1 << bits <= val; ++bits) + ; + return bits - 1; +}; +var rfse = function(dat, bt, mal) { + var tpos = (bt << 3) + 4; + var al = (dat[bt] & 15) + 5; + if (al > mal) + err(3); + var sz = 1 << al; + var probs = sz, sym = -1, re = -1, i = -1, ht = sz; + var buf = new ab(512 + (sz << 2)); + var freq = new i16(buf, 0, 256); + var dstate = new u16(buf, 0, 256); + var nstate = new u16(buf, 512, sz); + var bb1 = 512 + (sz << 1); + var syms = new u8(buf, bb1, sz); + var nbits = new u8(buf, bb1 + sz); + while (sym < 255 && probs > 0) { + var bits = msb(probs + 1); + var cbt = tpos >> 3; + var msk = (1 << bits + 1) - 1; + var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk; + var msk1fb = (1 << bits) - 1; + var msv = msk - probs - 1; + var sval = val & msk1fb; + if (sval < msv) + tpos += bits, val = sval; + else { + tpos += bits + 1; + if (val > msk1fb) + val -= msv; + } + freq[++sym] = --val; + if (val == -1) { + probs += val; + syms[--ht] = sym; + } else + probs -= val; + if (!val) { + do { + var rbt = tpos >> 3; + re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3; + tpos += 2; + sym += re; + } while (re == 3); + } + } + if (sym > 255 || probs) + err(0); + var sympos = 0; + var sstep = (sz >> 1) + (sz >> 3) + 3; + var smask = sz - 1; + for (var s = 0; s <= sym; ++s) { + var sf = freq[s]; + if (sf < 1) { + dstate[s] = -sf; + continue; + } + for (i = 0; i < sf; ++i) { + syms[sympos] = s; + do { + sympos = sympos + sstep & smask; + } while (sympos >= ht); + } + } + if (sympos) + err(0); + for (i = 0; i < sz; ++i) { + var ns = dstate[syms[i]]++; + var nb = nbits[i] = al - msb(ns); + nstate[i] = (ns << nb) - sz; + } + return [tpos + 7 >> 3, { + b: al, + s: syms, + n: nbits, + t: nstate + }]; +}; +var rhu = function(dat, bt) { + var i = 0, wc = -1; + var buf = new u8(292), hb = dat[bt]; + var hw = buf.subarray(0, 256); + var rc = buf.subarray(256, 268); + var ri = new u16(buf.buffer, 268); + if (hb < 128) { + var _a = rfse(dat, bt + 1, 6), ebt = _a[0], fdt = _a[1]; + bt += hb; + var epos = ebt << 3; + var lb = dat[bt]; + if (!lb) + err(0); + var st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1; + var fpos = (++bt << 3) - 8 + msb(lb); + for (; ; ) { + fpos -= btr1; + if (fpos < epos) + break; + var cbt = fpos >> 3; + st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1; + hw[++wc] = fdt.s[st1]; + fpos -= btr2; + if (fpos < epos) + break; + cbt = fpos >> 3; + st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1; + hw[++wc] = fdt.s[st2]; + btr1 = fdt.n[st1]; + st1 = fdt.t[st1]; + btr2 = fdt.n[st2]; + st2 = fdt.t[st2]; + } + if (++wc > 255) + err(0); + } else { + wc = hb - 127; + for (; i < wc; i += 2) { + var byte = dat[++bt]; + hw[i] = byte >> 4; + hw[i + 1] = byte & 15; + } + ++bt; + } + var wes = 0; + for (i = 0; i < wc; ++i) { + var wt = hw[i]; + if (wt > 11) + err(0); + wes += wt && 1 << wt - 1; + } + var mb = msb(wes) + 1; + var ts = 1 << mb; + var rem = ts - wes; + if (rem & rem - 1) + err(0); + hw[wc++] = msb(rem) + 1; + for (i = 0; i < wc; ++i) { + var wt = hw[i]; + ++rc[hw[i] = wt && mb + 1 - wt]; + } + var hbuf = new u8(ts << 1); + var syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts); + ri[mb] = 0; + for (i = mb; i > 0; --i) { + var pv = ri[i]; + fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i)); + } + if (ri[0] != ts) + err(0); + for (i = 0; i < wc; ++i) { + var bits = hw[i]; + if (bits) { + var code = ri[bits]; + fill(syms, i, code, ri[bits] = code + (1 << mb - bits)); + } + } + return [bt, { + n: nb, + b: mb, + s: syms + }]; +}; +var dllt = rfse(/* @__PURE__ */ new u8([ + 81, + 16, + 99, + 140, + 49, + 198, + 24, + 99, + 12, + 33, + 196, + 24, + 99, + 102, + 102, + 134, + 70, + 146, + 4 +]), 0, 6)[1]; +var dmlt = rfse(/* @__PURE__ */ new u8([ + 33, + 20, + 196, + 24, + 99, + 140, + 33, + 132, + 16, + 66, + 8, + 33, + 132, + 16, + 66, + 8, + 33, + 68, + 68, + 68, + 68, + 68, + 68, + 68, + 68, + 36, + 9 +]), 0, 6)[1]; +var doct = rfse(/* @__PURE__ */ new u8([ + 32, + 132, + 16, + 66, + 102, + 70, + 68, + 68, + 68, + 68, + 36, + 73, + 2 +]), 0, 5)[1]; +var b2bl = function(b, s) { + var len = b.length, bl = new i32(len); + for (var i = 0; i < len; ++i) { + bl[i] = s; + s += 1 << b[i]; + } + return bl; +}; +var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([ + 0, + 0, + 0, + 0, + 16843009, + 50528770, + 134678020, + 202050057, + 269422093 +])).buffer, 0, 36); +var llbl = /* @__PURE__ */ b2bl(llb, 0); +var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16843009, + 50528770, + 117769220, + 185207048, + 252579084, + 16 +])).buffer, 0, 53); +var mlbl = /* @__PURE__ */ b2bl(mlb, 3); +var dhu = function(dat, out, hu) { + var len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b; + if (!lb) + err(0); + var st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1; + for (; pos > eb && i < ss; ) { + var cbt = pos >> 3; + var val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7); + st = (st << btr | val) & msk; + out[++i] = hu.s[st]; + pos -= btr = hu.n[st]; + } + if (pos != eb || i + 1 != ss) + err(0); +}; +var dhu4 = function(dat, out, hu) { + var bt = 6; + var ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2; + dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu); + dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu); + dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu); + dhu(dat.subarray(bt), out.subarray(sz3), hu); +}; +var rzb = function(dat, st, out) { + var _a; + var bt = st.b; + var b0 = dat[bt], btype = b0 >> 1 & 3; + st.l = b0 & 1; + var sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13; + var ebt = (bt += 3) + sz; + if (btype == 1) { + if (bt >= dat.length) + return; + st.b = bt + 1; + if (out) { + fill(out, dat[bt], st.y, st.y += sz); + return out; + } + return fill(new u8(sz), dat[bt]); + } + if (ebt > dat.length) + return; + if (btype == 0) { + st.b = ebt; + if (out) { + out.set(dat.subarray(bt, ebt), st.y); + st.y += sz; + return out; + } + return slc(dat, bt, ebt); + } + if (btype == 2) { + var b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3; + var lss = b3 >> 4, lcs = 0, s4 = 0; + if (lbt < 2) { + if (sf & 1) + lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12); + else + lss = b3 >> 3; + } else { + s4 = sf; + if (sf < 2) + lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2; + else if (sf == 2) + lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6; + else + lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10; + } + ++bt; + var buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m); + var spl = buf.length - lss; + if (lbt == 0) + buf.set(dat.subarray(bt, bt += lss), spl); + else if (lbt == 1) + fill(buf, dat[bt++], spl); + else { + var hu = st.h; + if (lbt == 2) { + var hud = rhu(dat, bt); + lcs += bt - (bt = hud[0]); + st.h = hu = hud[1]; + } else if (!hu) + err(0); + (s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu); + } + var ns = dat[bt++]; + if (ns) { + if (ns == 255) + ns = (dat[bt++] | dat[bt++] << 8) + 32512; + else if (ns > 127) + ns = ns - 128 << 8 | dat[bt++]; + var scm = dat[bt++]; + if (scm & 3) + err(0); + var dts = [dmlt, doct, dllt]; + for (var i = 2; i > -1; --i) { + var md = scm >> (i << 1) + 2 & 3; + if (md == 1) { + var rbuf = new u8([0, 0, dat[bt++]]); + dts[i] = { + s: rbuf.subarray(2, 3), + n: rbuf.subarray(0, 1), + t: new u16(rbuf.buffer, 0, 1), + b: 0 + }; + } else if (md == 2) { + _a = rfse(dat, bt, 9 - (i & 1)), bt = _a[0], dts[i] = _a[1]; + } else if (md == 3) { + if (!st.t) + err(0); + dts[i] = st.t[i]; + } + } + var _b = st.t = dts, mlt = _b[0], oct = _b[1], llt = _b[2]; + var lb = dat[ebt - 1]; + if (!lb) + err(0); + var spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0; + var lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1; + cbt = (spos -= oct.b) >> 3; + var ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1; + cbt = (spos -= mlt.b) >> 3; + var mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1; + for (++ns; --ns; ) { + var llc = llt.s[lst]; + var lbtr = llt.n[lst]; + var mlc = mlt.s[mst]; + var mbtr = mlt.n[mst]; + var ofc = oct.s[ost]; + var obtr = oct.n[ost]; + cbt = (spos -= ofc) >> 3; + var ofp = 1 << ofc; + var off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1); + cbt = (spos -= mlb[mlc]) >> 3; + var ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1); + cbt = (spos -= llb[llc]) >> 3; + var ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1); + cbt = (spos -= lbtr) >> 3; + lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1); + cbt = (spos -= mbtr) >> 3; + mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1); + cbt = (spos -= obtr) >> 3; + ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1); + if (off > 3) { + st.o[2] = st.o[1]; + st.o[1] = st.o[0]; + st.o[0] = off -= 3; + } else { + var idx = off - (ll != 0); + if (idx) { + off = idx == 3 ? st.o[0] - 1 : st.o[idx]; + if (idx > 1) + st.o[2] = st.o[1]; + st.o[1] = st.o[0]; + st.o[0] = off; + } else + off = st.o[0]; + } + for (var i = 0; i < ll; ++i) { + buf[oubt + i] = buf[spl + i]; + } + oubt += ll, spl += ll; + var stin = oubt - off; + if (stin < 0) { + var len = -stin; + var bs = st.e + stin; + if (len > ml) + len = ml; + for (var i = 0; i < len; ++i) { + buf[oubt + i] = st.w[bs + i]; + } + oubt += len, ml -= len, stin = 0; + } + for (var i = 0; i < ml; ++i) { + buf[oubt + i] = buf[stin + i]; + } + oubt += ml; + } + if (oubt != spl) { + while (spl < buf.length) { + buf[oubt++] = buf[spl++]; + } + } else + oubt = buf.length; + if (out) + st.y += oubt; + else + buf = slc(buf, 0, oubt); + } else if (out) { + st.y += lss; + if (spl) { + for (var i = 0; i < lss; ++i) { + buf[i] = buf[spl + i]; + } + } + } else if (spl) + buf = slc(buf, spl); + st.b = ebt; + return buf; + } + err(2); +}; +var cct = function(bufs, ol) { + if (bufs.length == 1) + return bufs[0]; + var buf = new u8(ol); + for (var i = 0, b = 0; i < bufs.length; ++i) { + var chk = bufs[i]; + buf.set(chk, b); + b += chk.length; + } + return buf; +}; +function decompress(dat, buf) { + var bufs = [], nb = +!buf; + var bt = 0, ol = 0; + for (; dat.length; ) { + var st = rzfh(dat, nb || buf); + if (typeof st == "object") { + if (nb) { + buf = null; + if (st.w.length == st.u) { + bufs.push(buf = st.w); + ol += st.u; + } + } else { + bufs.push(buf); + st.e = 0; + } + for (; !st.l; ) { + var blk = rzb(dat, st, buf); + if (!blk) + err(5); + if (buf) + st.e = st.y; + else { + bufs.push(blk); + ol += blk.length; + cpw(st.w, 0, blk.length); + st.w.set(blk, st.w.length - blk.length); + } + } + bt = st.b + st.c * 4; + } else + bt = st; + dat = dat.subarray(bt); + } + return cct(bufs, ol); +} + +// src/decoder.js +function decompressFrame(frameBytes) { + return decompress(frameBytes); +} + +// src/index.js +var TOKEN_URL = "https://oauth2.googleapis.com/token"; +var DRIVE_FILES_URL = "https://www.googleapis.com/drive/v3/files"; +var FETCH_ROUTE = /^\/fetch\/([0-9a-fA-F]{64})$/; +var HttpError = class extends Error { + constructor(status, message) { + super(message); + this.status = status; + } +}; +function jsonResponse(obj, status = 200) { + return new Response(JSON.stringify(obj), { + status, + headers: { "Content-Type": "application/json" } + }); +} +function errorResponse(message, status) { + return jsonResponse({ error: message }, status); +} +async function sha256Hex(bytes) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} +var BundleFlight = class { + constructor(state, env) { + this.state = state; + this.env = env; + this.inflight = /* @__PURE__ */ new Map(); + } + async fetch(request) { + let job; + try { + job = await request.json(); + } catch { + return errorResponse("invalid JSON job body", 400); + } + try { + let pending = this.inflight.get(job.sha256); + if (!pending) { + pending = this.rehydrate(job).finally(() => { + this.inflight.delete(job.sha256); + }); + this.inflight.set(job.sha256, pending); + } + const result = await pending; + return new Response(result.bytes, { + status: 200, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(result.bytes.length), + "X-Cold-Store-Sha256": result.sha256, + "X-Cold-Store-Bundle": result.bundleId + } + }); + } catch (err2) { + if (err2 instanceof HttpError) { + return errorResponse(err2.message, err2.status); + } + return errorResponse(`rehydration failed: ${err2.message}`, 502); + } + } + async accessToken() { + const cached = await this.state.storage.get("google_token"); + if (cached && cached.expires_at_ms > Date.now() + 6e4) { + return cached.token; + } + const form = new URLSearchParams({ + client_id: this.env.GOOGLE_CLIENT_ID, + client_secret: this.env.GOOGLE_CLIENT_SECRET, + refresh_token: this.env.GOOGLE_REFRESH_TOKEN, + grant_type: "refresh_token" + }); + const resp = await fetch(TOKEN_URL, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: form + }); + if (!resp.ok) { + throw new HttpError(502, `google token exchange failed (HTTP ${resp.status})`); + } + const payload = await resp.json(); + const entry = { + token: payload.access_token, + expires_at_ms: Date.now() + (payload.expires_in ?? 3600) * 1e3 + }; + await this.state.storage.put("google_token", entry); + return entry.token; + } + async rehydrate(job) { + const { drive_file_id, offset, frame_length, sha256, bundle_id } = job; + const token = await this.accessToken(); + const range = `bytes=${offset}-${offset + frame_length - 1}`; + const resp = await fetch(`${DRIVE_FILES_URL}/${drive_file_id}?alt=media`, { + headers: { Authorization: `Bearer ${token}`, Range: range } + }); + if (resp.status !== 206 && resp.status !== 200) { + throw new HttpError(502, `drive range download failed (HTTP ${resp.status})`); + } + let frame = new Uint8Array(await resp.arrayBuffer()); + if (resp.status === 200) { + frame = frame.slice(offset, offset + frame_length); + } + if (frame.length !== frame_length) { + throw new HttpError( + 502, + `drive range download short: got ${frame.length} of ${frame_length} bytes` + ); + } + let plain; + try { + plain = decompressFrame(frame); + } catch { + throw new HttpError(502, "zstd frame decompression failed"); + } + const digest = await sha256Hex(plain); + if (digest !== String(sha256).toLowerCase()) { + throw new HttpError(502, "sha256 mismatch \u2014 refusing to return unverified bytes"); + } + return { bytes: plain, bundleId: bundle_id, sha256: digest }; + } +}; +var index_default = { + async fetch(request, env) { + const url = new URL(request.url); + if (request.method === "GET" && url.pathname === "/health") { + return jsonResponse({ ok: true }); + } + const match = url.pathname.match(FETCH_ROUTE); + if (request.method === "GET" && match) { + if (!env.REHYDRATOR_API_KEY || request.headers.get("Authorization") !== `Bearer ${env.REHYDRATOR_API_KEY}`) { + return errorResponse("unauthorized", 401); + } + const sha = match[1].toLowerCase(); + const row = await env.MANIFEST.prepare( + "SELECT o.hash AS sha256, o.size AS size, m.offset AS offset, m.length AS frame_length, m.bundle_id AS bundle_id, b.drive_file_id AS drive_file_id FROM objects o JOIN members m ON m.object_hash = o.hash JOIN bundles b ON b.bundle_id = m.bundle_id WHERE o.hash = ? AND b.state = 'verified' ORDER BY b.uploaded_at DESC LIMIT 1" + ).bind(sha).first(); + if (!row) { + return errorResponse("unknown object", 404); + } + const stub = env.BUNDLE_FLIGHT.get(env.BUNDLE_FLIGHT.idFromName(row.bundle_id)); + return stub.fetch("https://bundle-flight/rehydrate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(row) + }); + } + return errorResponse("not found", 404); + } +}; +export { + BundleFlight, + index_default as default +}; diff --git a/pkgs/cold-store-reaper/worker/package-lock.json b/pkgs/cold-store-reaper/worker/package-lock.json new file mode 100644 index 000000000..b5145a100 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/package-lock.json @@ -0,0 +1,508 @@ +{ + "name": "cold-store-rehydrator", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cold-store-rehydrator", + "version": "1.0.0", + "dependencies": { + "fzstd": "^0.1.1" + }, + "devDependencies": { + "esbuild": "^0.28.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/fzstd": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/fzstd/-/fzstd-0.1.1.tgz", + "integrity": "sha512-dkuVSOKKwh3eas5VkJy1AW1vFpet8TA/fGmVA5krThl8YcOVE/8ZIoEA1+U1vEn5ckxxhLirSdY837azmbaNHA==", + "license": "MIT" + } + } +} diff --git a/pkgs/cold-store-reaper/worker/package.json b/pkgs/cold-store-reaper/worker/package.json new file mode 100644 index 000000000..b8bbdca71 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/package.json @@ -0,0 +1,17 @@ +{ + "dependencies": { + "fzstd": "^0.1.1" + }, + "description": "Cold-store rehydrator Worker: serves single objects out of Google Drive cold bundles (see ../README.md and worker/README.md).", + "devDependencies": { + "esbuild": "^0.28.0" + }, + "name": "cold-store-rehydrator", + "private": true, + "scripts": { + "build": "esbuild src/index.js --bundle --format=esm --platform=neutral --target=es2022 --outfile=dist/worker.js", + "test": "node test/run-local-proof.mjs" + }, + "type": "module", + "version": "1.0.0" +} diff --git a/pkgs/cold-store-reaper/worker/src/decoder.js b/pkgs/cold-store-reaper/worker/src/decoder.js new file mode 100644 index 000000000..4646f285c --- /dev/null +++ b/pkgs/cold-store-reaper/worker/src/decoder.js @@ -0,0 +1,13 @@ +import { decompress } from "fzstd"; + +// Decompress exactly one zstd frame. The reaper stores every object as its +// own self-contained zstd frame inside a bundle file, so a frame pulled out +// by byte range decompresses on its own — no bundle context needed. +// +// This module is the single decoder implementation for the whole package: +// the Worker bundles it (esbuild -> dist/worker.js) and the local proof +// test (test/run-local-proof.mjs) imports it directly, so the code that is +// tested locally is byte-for-byte the code that ships. +export function decompressFrame(frameBytes) { + return decompress(frameBytes); +} diff --git a/pkgs/cold-store-reaper/worker/src/index.js b/pkgs/cold-store-reaper/worker/src/index.js new file mode 100644 index 000000000..22831db74 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/src/index.js @@ -0,0 +1,206 @@ +// Cold-store rehydrator Worker (design section 6, v1). +// +// GET /health -> { ok: true } (no auth) +// GET /fetch/ -> the object's original bytes (Bearer auth) +// +// The Worker is a thin front door: it looks the object up in the D1 +// manifest (binding MANIFEST), then hands the job to a Durable Object +// (binding BUNDLE_FLIGHT) keyed by bundle id, which single-flights all +// concurrent requests for the same bundle. The DO exchanges the Google +// refresh token for an access token (cached in DO storage), Range-pulls +// exactly the member's zstd frame out of the bundle file in Drive, +// decompresses it, and verifies the plaintext sha256 before anything is +// returned. Unverified bytes are never served: any Drive, decompression, +// or hash failure is a 502. +// +// Secrets (set by the operator, never committed): REHYDRATOR_API_KEY, +// GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_REFRESH_TOKEN. + +import { decompressFrame } from "./decoder.js"; + +const TOKEN_URL = "https://oauth2.googleapis.com/token"; +const DRIVE_FILES_URL = "https://www.googleapis.com/drive/v3/files"; +const FETCH_ROUTE = /^\/fetch\/([0-9a-fA-F]{64})$/; + +class HttpError extends Error { + constructor(status, message) { + super(message); + this.status = status; + } +} + +function jsonResponse(obj, status = 200) { + return new Response(JSON.stringify(obj), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function errorResponse(message, status) { + return jsonResponse({ error: message }, status); +} + +async function sha256Hex(bytes) { + const digest = await crypto.subtle.digest("SHA-256", bytes); + return [...new Uint8Array(digest)] + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); +} + +// --------------------------------------------------------------------------- +// Durable Object: one instance per bundle id. Concurrent fetches for the +// same bundle funnel through here; an in-flight map joins identical member +// requests onto one Drive pull instead of stampeding Drive per request. +// --------------------------------------------------------------------------- +export class BundleFlight { + constructor(state, env) { + this.state = state; + this.env = env; + this.inflight = new Map(); // sha256 -> Promise<{bytes, bundleId, sha256}> + } + + async fetch(request) { + let job; + try { + job = await request.json(); + } catch { + return errorResponse("invalid JSON job body", 400); + } + try { + let pending = this.inflight.get(job.sha256); + if (!pending) { + pending = this.rehydrate(job).finally(() => { + this.inflight.delete(job.sha256); + }); + this.inflight.set(job.sha256, pending); + } + const result = await pending; + // A fresh Response per caller: a Response body can be consumed once, + // so joined callers must not share one instance. + return new Response(result.bytes, { + status: 200, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(result.bytes.length), + "X-Cold-Store-Sha256": result.sha256, + "X-Cold-Store-Bundle": result.bundleId, + }, + }); + } catch (err) { + if (err instanceof HttpError) { + return errorResponse(err.message, err.status); + } + return errorResponse(`rehydration failed: ${err.message}`, 502); + } + } + + async accessToken() { + const cached = await this.state.storage.get("google_token"); + if (cached && cached.expires_at_ms > Date.now() + 60_000) { + return cached.token; + } + const form = new URLSearchParams({ + client_id: this.env.GOOGLE_CLIENT_ID, + client_secret: this.env.GOOGLE_CLIENT_SECRET, + refresh_token: this.env.GOOGLE_REFRESH_TOKEN, + grant_type: "refresh_token", + }); + const resp = await fetch(TOKEN_URL, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: form, + }); + if (!resp.ok) { + throw new HttpError(502, `google token exchange failed (HTTP ${resp.status})`); + } + const payload = await resp.json(); + const entry = { + token: payload.access_token, + expires_at_ms: Date.now() + (payload.expires_in ?? 3600) * 1000, + }; + await this.state.storage.put("google_token", entry); + return entry.token; + } + + async rehydrate(job) { + const { drive_file_id, offset, frame_length, sha256, bundle_id } = job; + const token = await this.accessToken(); + const range = `bytes=${offset}-${offset + frame_length - 1}`; + const resp = await fetch(`${DRIVE_FILES_URL}/${drive_file_id}?alt=media`, { + headers: { Authorization: `Bearer ${token}`, Range: range }, + }); + if (resp.status !== 206 && resp.status !== 200) { + throw new HttpError(502, `drive range download failed (HTTP ${resp.status})`); + } + let frame = new Uint8Array(await resp.arrayBuffer()); + if (resp.status === 200) { + // Drive ignored the Range header and sent the whole bundle: slice out + // exactly the member's frame before doing anything with it. + frame = frame.slice(offset, offset + frame_length); + } + if (frame.length !== frame_length) { + throw new HttpError( + 502, + `drive range download short: got ${frame.length} of ${frame_length} bytes`, + ); + } + let plain; + try { + plain = decompressFrame(frame); + } catch { + throw new HttpError(502, "zstd frame decompression failed"); + } + const digest = await sha256Hex(plain); + if (digest !== String(sha256).toLowerCase()) { + throw new HttpError(502, "sha256 mismatch — refusing to return unverified bytes"); + } + return { bytes: plain, bundleId: bundle_id, sha256: digest }; + } +} + +// --------------------------------------------------------------------------- +// Worker entry point. +// --------------------------------------------------------------------------- +export default { + async fetch(request, env) { + const url = new URL(request.url); + + if (request.method === "GET" && url.pathname === "/health") { + return jsonResponse({ ok: true }); + } + + const match = url.pathname.match(FETCH_ROUTE); + if (request.method === "GET" && match) { + if ( + !env.REHYDRATOR_API_KEY || + request.headers.get("Authorization") !== `Bearer ${env.REHYDRATOR_API_KEY}` + ) { + return errorResponse("unauthorized", 401); + } + const sha = match[1].toLowerCase(); + const row = await env.MANIFEST.prepare( + "SELECT o.hash AS sha256, o.size AS size, m.offset AS offset," + + " m.length AS frame_length, m.bundle_id AS bundle_id," + + " b.drive_file_id AS drive_file_id" + + " FROM objects o" + + " JOIN members m ON m.object_hash = o.hash" + + " JOIN bundles b ON b.bundle_id = m.bundle_id" + + " WHERE o.hash = ? AND b.state = 'verified'" + + " ORDER BY b.uploaded_at DESC LIMIT 1", + ) + .bind(sha) + .first(); + if (!row) { + return errorResponse("unknown object", 404); + } + const stub = env.BUNDLE_FLIGHT.get(env.BUNDLE_FLIGHT.idFromName(row.bundle_id)); + return stub.fetch("https://bundle-flight/rehydrate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(row), + }); + } + + return errorResponse("not found", 404); + }, +}; diff --git a/pkgs/cold-store-reaper/worker/test/run-local-proof.mjs b/pkgs/cold-store-reaper/worker/test/run-local-proof.mjs new file mode 100644 index 000000000..4ea2f0232 --- /dev/null +++ b/pkgs/cold-store-reaper/worker/test/run-local-proof.mjs @@ -0,0 +1,82 @@ +// Local proof for the rehydrator Worker (run: npm test). +// +// 1. Uses the real Python reaper (`cold_store.ColdStore.pack`) to build a +// bundle from a sample file, and reads the member's offset/length/hash +// back out of the manifest SQLite it writes. +// 2. Slices exactly that frame out of the bundle file. +// 3. Decodes it with ../src/decoder.js — the identical module esbuild +// bundles into dist/worker.js — and asserts the plaintext sha256. +// +// Exits non-zero on any mismatch; prints PASS on success. + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { decompressFrame } from "../src/decoder.js"; + +const pkgDir = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +const PACK_SNIPPET = ` +import json, sqlite3, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from cold_store import ColdStore +home, sample = Path(sys.argv[2]), sys.argv[3] +store = ColdStore(home) +store.init() +res = store.pack([sys.argv[4], sample]) # decoy member first -> offset > 0 +conn = store.db() +row = conn.execute( + "SELECT object_hash, offset, length FROM members WHERE bundle_id = ?" + " ORDER BY offset DESC LIMIT 1", + (res["bundle_id"],), +).fetchone() +print(json.dumps({ + "bundle_path": res["path"], + "hash": row["object_hash"], + "offset": row["offset"], + "length": row["length"], +})) +`; + +const work = mkdtempSync(join(tmpdir(), "cold-store-proof-")); +const otherPath = join(work, "other.txt"); +writeFileSync(otherPath, "an unrelated first member, just to shift the offset\n"); +const samplePath = join(work, "sample.txt"); +// Line-numbered content: realistic size, does not collapse to a stub frame. +const sample = Buffer.from( + Array.from( + { length: 2000 }, + (_, i) => `rehydrator local proof line ${i} — cold store`, + ).join("\n"), +); +writeFileSync(samplePath, sample); + +const out = execFileSync( + "python3", + ["-c", PACK_SNIPPET, pkgDir, join(work, "home"), samplePath, otherPath], + { encoding: "utf8" }, +); +const info = JSON.parse(out.trim().split("\n").pop()); + +const bundle = readFileSync(info.bundle_path); +const frame = bundle.subarray(info.offset, info.offset + info.length); +const plain = Buffer.from(decompressFrame(new Uint8Array(frame))); +const digest = createHash("sha256").update(plain).digest("hex"); + +if (digest !== info.hash) { + console.error(`FAIL: sha256 mismatch: decoded ${digest}, manifest ${info.hash}`); + process.exit(1); +} +if (!plain.equals(sample)) { + console.error("FAIL: decoded bytes differ from the packed sample"); + process.exit(1); +} +console.log( + `PASS: frame at offset ${info.offset} (${info.length} bytes) decoded via ` + + `src/decoder.js (fzstd) and sha256-verified against the manifest (${digest.slice(0, 16)}...)`, +); diff --git a/pkgs/cold-store-reaper/worker/wrangler.toml.example b/pkgs/cold-store-reaper/worker/wrangler.toml.example new file mode 100644 index 000000000..6e33a41ca --- /dev/null +++ b/pkgs/cold-store-reaper/worker/wrangler.toml.example @@ -0,0 +1,33 @@ +# Reference configuration for the cold-store rehydrator Worker. +# +# Deployment is API-driven (the operator uploads dist/worker.js through the +# Cloudflare REST API and attaches these bindings there); this file documents +# the exact binding names/ids the code expects. If wrangler is used instead, +# copy this to wrangler.toml and fill in nothing else — the values below are +# the live ones (database ids are not secrets). +name = "cold-store-rehydrator" +main = "dist/worker.js" +compatibility_date = "2026-10-25" + +# D1 manifest: objects -> members -> bundles lookups. +[[d1_databases]] +binding = "MANIFEST" +database_name = "cold-store-manifest" +database_id = "15d5f460-835c-4f24-b229-222908a5b30c" + +# Single-flight per bundle id. +[[durable_objects.bindings]] +name = "BUNDLE_FLIGHT" +class_name = "BundleFlight" + +[[migrations]] +tag = "v1" +new_sqlite_classes = ["BundleFlight"] + +# Secrets — set out-of-band (CF API secret_text bindings, or +# `wrangler secret put `). NEVER commit values. +# REHYDRATOR_API_KEY bearer token callers must present on /fetch/ +# GOOGLE_CLIENT_ID OAuth client of the "Cold Store Reaper" app +# GOOGLE_CLIENT_SECRET ditto (Bitwarden: google.cold-store-reaper.client-secret) +# GOOGLE_REFRESH_TOKEN the one-time grant's refresh token +# (Bitwarden: google.cold-store-reaper.refresh-token) From 5daf225561cff46ef93a6c53f4aeeb5550daa0e3 Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 00:51:28 +0000 Subject: [PATCH 4/6] =?UTF-8?q?docs(cold-store-reaper):=20v2=20=E2=80=94?= =?UTF-8?q?=20D1=20manifest=20sync,=20DB=20backups,=20rehydrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents sync-manifest (token handling, overrides, dry-run), the DB-backup commands with retention explicitly noted as NOT enforced yet, and pointers to the worker. Also treefmt: exclude the worker's esbuild dist output from formatting. --- pkgs/cold-store-reaper/README.md | 68 ++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/pkgs/cold-store-reaper/README.md b/pkgs/cold-store-reaper/README.md index ca392e95c..e82a2a861 100644 --- a/pkgs/cold-store-reaper/README.md +++ b/pkgs/cold-store-reaper/README.md @@ -76,17 +76,71 @@ cold-store upload b_<...> # resumable upload + veri cold-store fetch ./restored.sql # range-pull one object back cold-store status # counts, cold bytes, last-24h transfers, # 700 GB/day hard-stop + 3 TiB budget checks +cold-store sync-manifest # push the local manifest into D1 +cold-store sync-manifest --dry-run # ... or just print the statement counts ``` +## DB backups (design section 5) + +Database dumps are a separate, simpler class: single self-describing files, +uploaded **unbundled** into `db-backups//` under the app-owned root +(`ensure-folders` creates the `supabase/`, `turso/`, and `d1/` subfolders, +idempotently). Files land with dated names — +`--` — and every upload writes a +`backup-upload` row to the transfer ledger. + +```sh +cold-store put-backup supabase pgdump-2026-10-26.sql.zst # prints the Drive file id +cold-store list-backups # all DBs, newest first +cold-store list-backups --db turso +``` + +**Retention is NOT enforced yet**: the design's 30 daily / 12 weekly / +12 monthly policy (trash, not delete) is pending a later version. Until +then, backups accumulate and pruning is manual. + +## Manifest sync → D1 + +The local SQLite manifest is the reaper's working copy; the hosted copy +lives in the Cloudflare D1 database `cold-store-manifest`, where the +rehydrator Worker reads it. `sync-manifest` pushes all five tables +(objects, bundles, members, transfer_ledger, meta) with batched, +idempotent upserts (`INSERT … ON CONFLICT() DO UPDATE`, primary keys +taken from `schema.sql`), so re-running is always safe and a failed run +resumes by simply running again. `--dry-run` prints per-table row and +statement counts and contacts nothing. + +The API token comes from the environment only — materialize it from +Bitwarden, never store it: + +```sh +CLOUDFLARE_API_TOKEN= cold-store sync-manifest +``` + +Account/database default to the cold-store D1 database; override with +`COLD_STORE_CF_ACCOUNT_ID` / `COLD_STORE_CF_D1_DATABASE_ID` (env) or the +`cf_account_id` / `cf_d1_database_id` fields of `state.json`. The D1 base +URL is overridable via `COLD_STORE_D1_BASE` (used by tests). + +## Rehydrator Worker + +`worker/` holds the Cloudflare Worker that serves cold objects back: +`GET /fetch/` (bearer auth) → D1 manifest lookup → a per-bundle +Durable Object single-flights a Drive Range pull of the one zstd frame, +decompresses it with a vendored pure-JS decoder, verifies the sha256, and +returns the bytes. See `worker/README.md` for endpoints, secrets, and the +API-driven deploy procedure. + ## Configuration -| Knob | Default | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | -| Home dir | `$COLD_STORE_HOME` or `~/.config/cold-store` (or `--home`) | -| Bundle target size (design constant) | 8 GiB (`pack` packs what it is given) | -| Upload chunk size | 8 MiB | -| Endpoints | Google; overridable via `COLD_STORE_TOKEN_URL`, `COLD_STORE_DRIVE_BASE`, `COLD_STORE_UPLOAD_BASE` (used by tests) | -| zstd binary | `/usr/local/bin/zstd`, or `$COLD_STORE_ZSTD` | +| Knob | Default | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Home dir | `$COLD_STORE_HOME` or `~/.config/cold-store` (or `--home`) | +| Bundle target size (design constant) | 8 GiB (`pack` packs what it is given) | +| Upload chunk size | 8 MiB | +| Endpoints | Google; overridable via `COLD_STORE_TOKEN_URL`, `COLD_STORE_DRIVE_BASE`, `COLD_STORE_UPLOAD_BASE` (used by tests) | +| D1 sync | `CLOUDFLARE_API_TOKEN` from env only; account/db via `COLD_STORE_CF_ACCOUNT_ID` / `COLD_STORE_CF_D1_DATABASE_ID` or `state.json`; base via `COLD_STORE_D1_BASE` | +| zstd binary | `/usr/local/bin/zstd`, or `$COLD_STORE_ZSTD` | ## Tests From 80f4a896df0a90f158b7192a568acd2a18004927 Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 00:51:28 +0000 Subject: [PATCH 5/6] chore(treefmt): exclude cold-store-reaper worker dist bundle --- treefmt.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/treefmt.toml b/treefmt.toml index 075925c33..f313d1e8d 100755 --- a/treefmt.toml +++ b/treefmt.toml @@ -7,6 +7,7 @@ excludes = [ "pkgs/dht/**", # WIP package with syntax errors ".opencode/**", # managed by opencode, may have permission issues "**/package-lock.json", # npm-generated; biome ignores it by default, so passing it in fails the run + "pkgs/cold-store-reaper/worker/dist/**", # esbuild bundle output; deployed as built, formatting it would rewrite the artifact "pkgs/mcpx/docs/parity.md", # generated; TestParityDocumentIsCurrent compares it byte-for-byte with the generator, so prettier's table padding can never be applied "pkgs/mcpx/docs/story.md", # quotes AGENTS.md verbatim; TestStoryDocumentIsCurrent compares the quoted lines byte-for-byte with AGENTS.md, whose *emphasis* prettier would rewrite to _emphasis_ ] From c8d2cc2e1d423785ad33e40428671e68fa9fcfe2 Mon Sep 17 00:00:00 2001 From: Muse Sweep Date: Mon, 26 Oct 2026 01:04:34 +0000 Subject: [PATCH 6/6] style(treefmt): taplo comment alignment for the excludes array --- treefmt.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/treefmt.toml b/treefmt.toml index f313d1e8d..5148817f9 100755 --- a/treefmt.toml +++ b/treefmt.toml @@ -3,13 +3,13 @@ # Exclude pkgs/id/ — it has its own treefmt/formatter [global] excludes = [ - "pkgs/id/**", # has its own treefmt/formatter - "pkgs/dht/**", # WIP package with syntax errors - ".opencode/**", # managed by opencode, may have permission issues - "**/package-lock.json", # npm-generated; biome ignores it by default, so passing it in fails the run + "pkgs/id/**", # has its own treefmt/formatter + "pkgs/dht/**", # WIP package with syntax errors + ".opencode/**", # managed by opencode, may have permission issues + "**/package-lock.json", # npm-generated; biome ignores it by default, so passing it in fails the run "pkgs/cold-store-reaper/worker/dist/**", # esbuild bundle output; deployed as built, formatting it would rewrite the artifact - "pkgs/mcpx/docs/parity.md", # generated; TestParityDocumentIsCurrent compares it byte-for-byte with the generator, so prettier's table padding can never be applied - "pkgs/mcpx/docs/story.md", # quotes AGENTS.md verbatim; TestStoryDocumentIsCurrent compares the quoted lines byte-for-byte with AGENTS.md, whose *emphasis* prettier would rewrite to _emphasis_ + "pkgs/mcpx/docs/parity.md", # generated; TestParityDocumentIsCurrent compares it byte-for-byte with the generator, so prettier's table padding can never be applied + "pkgs/mcpx/docs/story.md", # quotes AGENTS.md verbatim; TestStoryDocumentIsCurrent compares the quoted lines byte-for-byte with AGENTS.md, whose *emphasis* prettier would rewrite to _emphasis_ ] # [formatter.]