fix(route-proxy): per-leaf first-response timeout (#5294)

bin/opencode-route-proxy.mjs

@@ -33,6 +33,9 @@
 //   OPENCODE_ROUTE_PROXY_PORT        listen port (default 18750)
 //   OPENCODE_ROUTE_PROXY_HOST        listen host (default 127.0.0.1)
 //   OPENCODE_ROUTE_PROXY_TIMEOUT_MS  per-request cap (default 1200000)
+//   OPENCODE_ROUTE_PROXY_LEAF_TIMEOUT_MS  first-response cap per leaf
+//                                         (default 3000 ms; below the client
+//                                         no-output window)
 //   OPENCODE_ROUTE_PROXY_CONFIG      opencode.jsonc path (default:
 //                                    ~/.local/share/opencode/opencode.jsonc,
 //                                    else ../.opencode/opencode.jsonc
@@ -60,6 +63,10 @@ import { parseProxyMillis, parseProxyPort } from "./proxy-port.mjs";
 const PORT = parseProxyPort("OPENCODE_ROUTE_PROXY_PORT", 18750);
 const HOST = process.env.OPENCODE_ROUTE_PROXY_HOST || "127.0.0.1";
 const TIMEOUT_MS = parseProxyMillis("OPENCODE_ROUTE_PROXY_TIMEOUT_MS", 1_200_000);
+const LEAF_TIMEOUT_MS = parseProxyMillis(
+  "OPENCODE_ROUTE_PROXY_LEAF_TIMEOUT_MS",
+  3_000,
+);
 const HERE = dirname(fileURLToPath(import.meta.url));
 const CONFIG_PATH =
   process.env.OPENCODE_ROUTE_PROXY_CONFIG ||
@@ -962,18 +969,35 @@ const runCompletion = async (routeId, body, clientSignal, hooks = {}) => {
       const next = attempts[i + 1];
       const failedOverTo = next ? `${next.providerId}/${next.modelKey}` : null;
       let res;
+      const leafDeadlineCtl = new AbortController();
+      const leafTimer = setTimeout(
+        () =>
+          leafDeadlineCtl.abort(
+            new Error("opencode-route-proxy leaf first-response timeout"),
+          ),
+        LEAF_TIMEOUT_MS,
+      );
+      const leafSignal = AbortSignal.any([signal, leafDeadlineCtl.signal]);
       try {
-        res = await callLeaf(cfg, leaf, body, wantStream, signal, sessionLabel);
+        res = await callLeaf(
+          cfg,
+          leaf,
+          body,
+          wantStream,
+          leafSignal,
+          sessionLabel,
+        );
       } catch (err) {
         const ended = Date.now();
+        const leafTimedOut = !signal.aborted && leafDeadlineCtl.signal.aborted;
         recordAttempt({
           started,
           ended,
           routeId,
           providerId: leaf.providerId,
           modelId: leaf.modelKey,
           variant: leaf.variant,
-          outcome: signal.aborted ? "timeout" : "failure",
+          outcome: signal.aborted || leafTimedOut ? "timeout" : "failure",
           errorTag: err.tag || "Transport",
           errorCode: err.code ?? null,
           errorMessage: String(err.message || err).slice(0, 500),
@@ -986,6 +1010,8 @@ const runCompletion = async (routeId, body, clientSignal, hooks = {}) => {
         lastErr = err;
         if (signal.aborted) throw err;
         continue;
+      } finally {
+        clearTimeout(leafTimer);
       }
       if (!res.ok) {
         const ended = Date.now();

bin/opencode-route-proxy.test.mjs

@@ -126,7 +126,9 @@ const ROUTE_ATTEMPT_SCHEMA = `CREATE TABLE route_attempt (
   cost_usd real)`;
 
 const startProxy = async (arg = {}) => {
-  const timeout = typeof arg === "string" ? arg : "15000";
+  const timeout =
+    typeof arg === "string" ? arg : String(arg.timeoutMs ?? "15000");
+  const leafTimeout = typeof arg === "object" ? arg.leafTimeoutMs : undefined;
   const pinnedNow = arg && typeof arg === "object" ? arg.nowMs : undefined;
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "route-proxy-test-"));
   const leafA = await makeLeaf("LEAF-A", "key-a");
@@ -234,6 +236,9 @@ const startProxy = async (arg = {}) => {
     OPENCODE_ROUTE_PROXY_CONFIG: configPath,
     OPENCODE_ROUTE_PROXY_DB: dbPath,
     OPENCODE_ROUTE_PROXY_TIMEOUT_MS: timeout,
+    ...(leafTimeout !== undefined
+      ? { OPENCODE_ROUTE_PROXY_LEAF_TIMEOUT_MS: String(leafTimeout) }
+      : {}),
     ...(pinnedNow
       ? { OPENCODE_ROUTE_PROXY_NOW_MS: String(pinnedNow) }
       : {}),
@@ -312,6 +317,38 @@ test("configured deadline aborts a slow leaf", async () => {
   }
 });
 
+test(
+  "per-leaf first-response timeout fails over and records the attempt",
+  async () => {
+    const p = await startProxy({ timeoutMs: "500", leafTimeoutMs: "100" });
+    try {
+      p.leafA.mode = "slow";
+      const start = Date.now();
+      const res = await p.post({
+        model: "opencode-route/t-ordered",
+        messages: MSGS,
+      });
+      assert.equal(res.status, 200);
+      const json = await res.json();
+      assert.equal(json.choices[0].message.content, "LEAF-B");
+      assert.ok(
+        Date.now() - start < 2500,
+        "per-leaf timeout did not shed promptly",
+      );
+      assert.equal(p.leafA.hits, 1);
+      assert.equal(p.leafB.hits, 1);
+      const rows = p.attempts();
+      assert.equal(rows.length, 2);
+      assert.equal(rows[0].outcome, "timeout");
+      assert.equal(rows[0].error_tag, "Transport");
+      assert.equal(rows[0].failed_over_to, "leafB/m-b");
+      assert.equal(rows[1].outcome, "success");
+    } finally {
+      await p.cleanup();
+    }
+  },
+);
+
 test("health and model list are served locally", async () => {
   const p = await startProxy();
   try {