perf(ci): cache server tests individually across shards

.github/workflows/ci.yml

@@ -420,7 +420,18 @@ jobs:
         env:
           T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
           T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
-        run: vp run --filter t3 test --shard ${{ matrix.shard }}/${{ strategy.job-total }}
+        run: >-
+          vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }}
+          --uncached apps/server/integration/transferBudgetV2.integration.test.ts
+          --verify-hits 1 --report "${{ runner.temp }}/test-cache-${{ matrix.shard }}.json" apps/server
+
+      - name: Upload test cache measurements
+        if: always() && steps.changes.outputs.test_server != 'false'
+        uses: actions/upload-artifact@v7
+        with:
+          name: test-cache-${{ matrix.shard }}
+          path: ${{ runner.temp }}/test-cache-${{ matrix.shard }}.json
+          if-no-files-found: error
 
       # integration/transferBudgetV2.integration.test.ts writes the report; exactly one shard
       # produces these files. Gating the upload on their presence keeps a

package.json

@@ -33,6 +33,7 @@
     "knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts",
     "lint:mobile": "node scripts/mobile-native-static-check.ts",
     "test": "vp run -r test",
+    "test:cached": "node scripts/test-cached.ts",
     "test:perf:v2-wire": "vp test run apps/server/src/orchestration-v2/WireProjection.test.ts apps/server/src/orchestration-v2/threadHistoryPaging.test.ts apps/server/src/orchestration-v2/ThreadStream.test.ts apps/server/src/orchestration-v2/ThreadTransportPerformance.test.ts apps/server/src/orchestration-v2/ShellStream.test.ts apps/server/src/orchestration-v2/CommandPolicy.test.ts apps/server/src/orchestration-v2/runtimeLayer.test.ts apps/server/src/ws.test.ts packages/contracts/src/orchestrationV2.test.ts packages/client-runtime/src/operations/commands.test.ts packages/client-runtime/src/operations/commands.performance.test.ts packages/client-runtime/src/state/boundedThreadSnapshotHttp.test.ts packages/client-runtime/src/state/threadHistoryController.test.ts packages/client-runtime/src/state/threadHistoryMerge.test.ts packages/client-runtime/src/state/threads-sync.test.ts packages/client-runtime/src/state/shellReducer.test.ts",
     "test:resource-monitor": "cargo test --locked --manifest-path native/resource-monitor/Cargo.toml",
     "test:desktop-smoke": "vp run --filter @t3tools/desktop smoke-test",

scripts/lib/cached-tests.test.ts

@@ -0,0 +1,140 @@
+// @effect-diagnostics nodeBuiltinImport:off - verifies native task caching in isolated subprocesses.
+import { describe, expect, it } from "vite-plus/test";
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFSP from "node:fs/promises";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import * as NodeURL from "node:url";
+
+import { cachedTestTasks, matchesTestPath } from "./cached-tests.ts";
+
+describe("cachedTestTasks", () => {
+  it("selects every test for a checkout-root path", () => {
+    const root = NodePath.resolve(import.meta.dirname, "../..");
+    for (const path of [".", root]) {
+      expect(
+        matchesTestPath("apps/server/src/auth/utils.test.ts", [
+          NodePath.relative(root, NodePath.resolve(root, path)),
+        ]),
+      ).toBe(true);
+    }
+    expect(matchesTestPath("apps/server/src/auth/utils.test.ts", ["apps/server"])).toBe(true);
+    expect(matchesTestPath("apps/server-other/a.test.ts", ["apps/server"])).toBe(false);
+  });
+  it("keeps a file's cache task stable when unrelated tests are added", () => {
+    const first = cachedTestTasks(JSON.stringify({ files: ["src/b.test.ts"], groupSize: 1 }));
+    const second = cachedTestTasks(
+      JSON.stringify({ files: ["src/a.test.ts", "src/b.test.ts"], groupSize: 1 }),
+    );
+    const key = Object.keys(first).find((key) => key !== "test-cache")!;
+    expect(second[key]).toEqual(first[key]);
+  });
+
+  it("keeps shell metacharacters in filenames out of the command", () => {
+    const tasks = cachedTestTasks(
+      JSON.stringify({ files: ["src/a'$(touch injected).test.ts"], groupSize: 1 }),
+    );
+    const task = Object.entries(tasks).find(([key]) => key !== "test-cache")![1];
+    expect(task.command).toBe("vp test run 'src/a'\"'\"'$(touch injected).test.ts'");
+  });
+
+  it("rejects paths outside the checkout and empty groups", () => {
+    for (const request of [
+      { files: ["../elsewhere.test.ts"], groupSize: 1 },
+      { files: ["/absolute.test.ts"], groupSize: 1 },
+      { files: ["src/a.test.ts"], groupSize: 0 },
+    ]) {
+      expect(() => cachedTestTasks(JSON.stringify(request))).toThrow();
+    }
+  });
+
+  it("accepts spec files discovered by Vitest", () => {
+    const tasks = cachedTestTasks(JSON.stringify({ files: ["src/example.spec.ts"], groupSize: 1 }));
+    expect(
+      Object.values(tasks).some((task) => task.command === "vp test run 'src/example.spec.ts'"),
+    ).toBe(true);
+  });
+
+  it("does not add tasks without an explicit request", () => {
+    expect(cachedTestTasks(undefined)).toEqual({});
+  });
+
+  it("reuses individual results, invalidates filesystem inputs, and never caches failures", async () => {
+    const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "cached-tests-"));
+    const root = NodePath.resolve(import.meta.dirname, "../..");
+    const run = (files = ["a.test.ts", "b.test.ts"], uncached: Array<string> = []) =>
+      new Promise<number | null>((resolve, reject) => {
+        const child = NodeChildProcess.spawn(
+          process.execPath,
+          [NodePath.join(root, "node_modules/vite-plus/dist/bin.js"), "run", "test-cache"],
+          {
+            cwd: directory,
+            env: {
+              ...process.env,
+              NODE_OPTIONS: "",
+              VP_REMOTE_CACHE: "off",
+              T3_CACHED_TESTS: JSON.stringify({ files, groupSize: 1, uncached }),
+              VITE_CACHE_PATH: NodePath.join(directory, "cache"),
+              PATH: `${NodePath.join(root, "node_modules/.bin")}${NodePath.delimiter}${process.env.PATH}`,
+            },
+            stdio: "ignore",
+          },
+        );
+        child.once("error", reject);
+        child.once("close", resolve);
+      });
+    const summary = async () => {
+      const value: { tasks: Array<{ result: Record<string, unknown> }> } = JSON.parse(
+        await NodeFSP.readFile(NodePath.join(directory, "cache/v18/last-summary.json"), "utf8"),
+      );
+      return value.tasks.filter((task) => "CacheHit" in task.result).length;
+    };
+    try {
+      await NodeFSP.mkdir(NodePath.join(directory, "node_modules"));
+      await NodeFSP.symlink(
+        NodePath.join(root, "node_modules/vite-plus"),
+        NodePath.join(directory, "node_modules/vite-plus"),
+        "dir",
+      );
+      await NodeFSP.writeFile(NodePath.join(directory, "package.json"), '{"type":"module"}');
+      await NodeFSP.writeFile(
+        NodePath.join(directory, "vite.config.ts"),
+        `import {cachedTestTasks} from ${JSON.stringify(NodeURL.pathToFileURL(NodePath.join(root, "scripts/lib/cached-tests.ts")).href)}; export default {run:{tasks:cachedTestTasks(process.env.T3_CACHED_TESTS, import.meta.dirname)},test:{include:["*.test.ts"]}};`,
+      );
+      await NodeFSP.writeFile(
+        NodePath.join(directory, "a.test.ts"),
+        'import {test,expect} from "vite-plus/test"; import {readFileSync} from "node:fs"; test("fixture",()=>expect(readFileSync("fixture.txt","utf8")).toMatch(/^hello/));',
+      );
+      await NodeFSP.writeFile(
+        NodePath.join(directory, "b.test.ts"),
+        'import {test,expect} from "vite-plus/test"; test("independent",()=>expect(1).toBe(1));',
+      );
+      await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello");
+      expect(await run()).toBe(0);
+      expect(await run()).toBe(0);
+      expect(await summary()).toBe(2);
+      expect(await run(["b.test.ts"])).toBe(0);
+      expect(await summary()).toBe(1);
+      expect(await run()).toBe(0);
+      expect(await summary()).toBe(2);
+      await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello changed");
+      expect(await run()).toBe(0);
+      expect(await summary()).toBe(1);
+      await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "broken");
+      expect(await run()).toBe(1);
+      expect(await run()).toBe(1);
+      expect(await summary()).toBe(1);
+      await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello again");
+      await NodeFSP.writeFile(
+        NodePath.join(directory, "b.test.ts"),
+        'import {test} from "vite-plus/test";import {writeFileSync} from "node:fs";test("report",()=>writeFileSync("report.json","{}"));',
+      );
+      expect(await run(undefined, ["b.test.ts"])).toBe(0);
+      await NodeFSP.unlink(NodePath.join(directory, "report.json"));
+      expect(await run(undefined, ["b.test.ts"])).toBe(0);
+      expect(await NodeFSP.readFile(NodePath.join(directory, "report.json"), "utf8")).toBe("{}");
+    } finally {
+      await NodeFSP.rm(directory, { recursive: true, force: true });
+    }
+  });
+});

scripts/lib/cached-tests.ts

@@ -0,0 +1,118 @@
+// @effect-diagnostics nodeBuiltinImport:off - Vite loads this before dependency installation and service initialization.
+import * as NodeCrypto from "node:crypto";
+import * as NodeFS from "node:fs";
+import * as NodePath from "node:path";
+
+interface CachedTestTask {
+  command: string;
+  cwd?: string;
+  dependsOn?: Array<string>;
+  cache:
+    | false
+    | {
+        input: Array<string | { auto: true }>;
+        output: Array<string>;
+      };
+}
+
+export function matchesTestPath(file: string, filters: Array<string>) {
+  return (
+    filters.length === 0 ||
+    filters.some((filter) => filter === "" || file === filter || file.startsWith(`${filter}/`))
+  );
+}
+
+export function cachedTestTasks(
+  raw: string | undefined,
+  root = NodePath.resolve(import.meta.dirname, "../.."),
+): Record<string, CachedTestTask> {
+  if (!raw) return {};
+  const request: unknown = JSON.parse(raw);
+  if (
+    typeof request !== "object" ||
+    request === null ||
+    !("files" in request) ||
+    !Array.isArray(request.files) ||
+    !("groupSize" in request) ||
+    typeof request.groupSize !== "number" ||
+    !Number.isSafeInteger(request.groupSize) ||
+    request.groupSize < 1
+  ) {
+    throw new Error("Invalid cached test request");
+  }
+  const uncached = "uncached" in request ? request.uncached : [];
+  if (!Array.isArray(uncached) || !uncached.every((file) => typeof file === "string")) {
+    throw new Error("Invalid uncached test files");
+  }
+  const files: Array<string> = [];
+  for (const file of request.files) {
+    if (
+      typeof file !== "string" ||
+      NodePath.isAbsolute(file) ||
+      file.split(/[\\/]/).includes("..") ||
+      !/\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)
+    ) {
+      throw new Error("Cached tests must be test files inside the checkout");
+    }
+    files.push(file);
+  }
+  const tasks: Record<string, CachedTestTask> = {};
+  const sorted = [...new Set(files)].sort();
+  const packages = new Map<string, Array<string>>();
+  for (const file of sorted) {
+    let directory = NodePath.dirname(NodePath.join(root, file));
+    while (directory !== root && !NodeFS.existsSync(NodePath.join(directory, "package.json"))) {
+      directory = NodePath.dirname(directory);
+    }
+    const entries = packages.get(directory) ?? [];
+    entries.push(file);
+    packages.set(directory, entries);
+  }
+  for (const [directory, entries] of packages) {
+    const manifest: unknown = JSON.parse(
+      NodeFS.readFileSync(NodePath.join(directory, "package.json"), "utf8"),
+    );
+    const command =
+      directory !== root &&
+      typeof manifest === "object" &&
+      manifest !== null &&
+      "scripts" in manifest &&
+      typeof manifest.scripts === "object" &&
+      manifest.scripts !== null &&
+      "test" in manifest.scripts &&
+      typeof manifest.scripts.test === "string"
+        ? manifest.scripts.test
+        : "vp test run";
+    if (!command.startsWith("vp test run")) {
+      throw new Error(`Unsupported test command in ${directory}`);
+    }
+    const cacheable = entries.filter((file) => !uncached.includes(file));
+    const groups: Array<Array<string>> = [];
+    for (let index = 0; index < cacheable.length; index += request.groupSize) {
+      groups.push(cacheable.slice(index, index + request.groupSize));
+    }
+    groups.push(...entries.filter((file) => uncached.includes(file)).map((file) => [file]));
+    for (const group of groups) {
+      const key = NodeCrypto.createHash("sha256").update(group.join("\0")).digest("hex");
+      tasks[`test-cache:${key}`] = {
+        command: `${command} ${group.map((file) => `'${NodePath.relative(directory, NodePath.join(root, file)).replaceAll("'", "'\"'\"'")}'`).join(" ")}`,
+        ...(directory === root ? {} : { cwd: NodePath.relative(root, directory) }),
+        cache: uncached.includes(group[0]!)
+          ? false
+          : {
+              // Vitest reads and rewrites these caches while running otherwise unchanged tests.
+              input: [
+                { auto: true },
+                "!**/node_modules/.vite/**",
+                "!**/node_modules/.vitest-cache/**",
+              ],
+              output: [],
+            },
+      };
+    }
+  }
+  return {
+    ...tasks,
+    "test-cache": { command: 'node -e ""', dependsOn: Object.keys(tasks), cache: false },
+  };
+}

scripts/lib/sharded-tests.test.ts

@@ -0,0 +1,55 @@
+// @effect-diagnostics nodeBuiltinImport:off - verifies Vitest discovery in a disposable project.
+import { expect, it } from "vite-plus/test";
+import * as NodeFSP from "node:fs/promises";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import { shardedTestFiles } from "./sharded-tests.ts";
+
+it("uses the configured sequencer and discovers every file exactly once", async () => {
+  const root = await NodeFSP.realpath(
+    await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "test-shards-")),
+  );
+  try {
+    await NodeFSP.mkdir(NodePath.join(root, "node_modules"));
+    await NodeFSP.symlink(
+      NodePath.resolve(import.meta.dirname, "../../node_modules/vite-plus"),
+      NodePath.join(root, "node_modules/vite-plus"),
+      "dir",
+    );
+    await NodeFSP.writeFile(NodePath.join(root, "package.json"), '{"type":"module"}');
+    await NodeFSP.writeFile(
+      NodePath.join(root, "vite.config.ts"),
+      `import {BaseSequencer} from "vite-plus/test/node";
+       class Sequencer extends BaseSequencer {
+         async shard(files) {
+           const {index,count}=this.ctx.config.shard;
+           return files.sort((a,b)=>a.moduleId.localeCompare(b.moduleId)).filter((_,i)=>i%count===index-1);
+         }
+       }
+       export default {test:{include:["included/*.{test,spec}.ts"],sequence:{sequencer:Sequencer}}};`,
+    );
+    await NodeFSP.mkdir(NodePath.join(root, "included"));
+    for (const file of [
+      "included/a.test.ts",
+      "included/b.test.ts",
+      "included/c.test.ts",
+      "included/d.spec.ts",
+      "excluded.test.ts",
+    ]) {
+      await NodeFSP.writeFile(NodePath.join(root, file), "export {};");
+    }
+    const first = await shardedTestFiles(root, "1/2");
+    const second = await shardedTestFiles(root, "2/2");
+    expect(first).toEqual(
+      ["a", "c"].map((file) => NodePath.join(root, `included/${file}.test.ts`)),
+    );
+    expect(second).toEqual([
+      NodePath.join(root, "included/b.test.ts"),
+      NodePath.join(root, "included/d.spec.ts"),
+    ]);
+    expect(new Set([...first, ...second]).size).toBe(4);
+    await expect(shardedTestFiles(root, "0/2")).rejects.toThrow("Shard must be");
+  } finally {
+    await NodeFSP.rm(root, { recursive: true, force: true });
+  }
+});

scripts/lib/sharded-tests.ts

@@ -0,0 +1,21 @@
+// @effect-diagnostics nodeBuiltinImport:off - Vitest discovery runs outside the application runtime.
+import * as NodePath from "node:path";
+import { createVitest } from "vite-plus/test/node";
+
+export async function shardedTestFiles(root: string, shard: string) {
+  const match = /^(\d+)\/(\d+)$/.exec(shard);
+  const index = Number(match?.[1]);
+  const count = Number(match?.[2]);
+  if (!Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index < 1 || index > count) {
+    throw new Error("Shard must be index/count with 1 <= index <= count");
+  }
+  const context = await createVitest("test", { root, watch: false, shard });
+  try {
+    const files = await context.globTestSpecifications();
+    const sequencer = new context.config.sequence.sequencer(context);
+    const selected = await sequencer.shard(files);
+    return selected.map((file) => NodePath.resolve(file.moduleId));
+  } finally {
+    await context.close();
+  }
+}