diff --git a/src/__tests__/drift/bedrock-misbehavior-live.drift.ts b/src/__tests__/drift/bedrock-misbehavior-live.drift.ts index 223f52a4..9f769bc0 100644 --- a/src/__tests__/drift/bedrock-misbehavior-live.drift.ts +++ b/src/__tests__/drift/bedrock-misbehavior-live.drift.ts @@ -20,6 +20,7 @@ import { Readable, Transform } from "node:stream"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { LLMock } from "../../llmock.js"; import type { Fixture } from "../../types.js"; +import { boundedDiagnosticLine } from "./bounded-diagnostic.js"; import { withClaudeFault } from "../helpers/misbehavior-claude-stage3.js"; const cells = [ @@ -419,36 +420,36 @@ async function runAwsCell( networkEnded, decoded, }; + const { rawWire, ...observed } = observation; console.log( - JSON.stringify({ - cell, - request: cell.wire === "invoke" ? invokeInput : converseInput, - ...observation, - rawWire: { encoding: "base64", bytes: observation.rawWire.toString("base64") }, - }), + boundedDiagnosticLine( + { cell, request: cell.wire === "invoke" ? invokeInput : converseInput, ...observed }, + rawWire, + ), ); return observation; } catch (error) { const errorMetadata = error instanceof Error && "$metadata" in error ? error.$metadata : undefined; + const diagnostic = { + cell, + request: cell.wire === "invoke" ? invokeInput : converseInput, + requestCount, + http, + metadata: errorMetadata ?? metadata, + sdkResponse, + sdkEvents, + decodedEvents: cell.wire === "invoke" ? events : converseEvents, + networkEnded, + error: error instanceof Error ? { name: error.name, message: error.message } : String(error), + }; console.log( - JSON.stringify({ - cell, - request: cell.wire === "invoke" ? invokeInput : converseInput, - requestCount, - http, - metadata: errorMetadata ?? metadata, - sdkResponse, - sdkEvents, - decodedEvents: cell.wire === "invoke" ? events : converseEvents, - networkEnded, - rawWire: - !live && limits.diagnosticWire === "summary" - ? { omitted: true, byteLength: Buffer.concat(chunks).length } - : { encoding: "base64", bytes: Buffer.concat(chunks).toString("base64") }, - error: - error instanceof Error ? { name: error.name, message: error.message } : String(error), - }), + !live && limits.diagnosticWire === "summary" + ? JSON.stringify({ + ...diagnostic, + rawWire: { omitted: true, byteLength: Buffer.concat(chunks).length }, + }) + : boundedDiagnosticLine(diagnostic, Buffer.concat(chunks)), ); throw failure ?? error; } finally { diff --git a/src/__tests__/drift/bounded-diagnostic.test.ts b/src/__tests__/drift/bounded-diagnostic.test.ts new file mode 100644 index 00000000..3ac3be51 --- /dev/null +++ b/src/__tests__/drift/bounded-diagnostic.test.ts @@ -0,0 +1,146 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { MAX_EVIDENCE_RECORD_BYTES } from "../../../scripts/drift-evidence-reporter.js"; +import { boundedDiagnosticLine, fitsEvidenceRecord } from "./bounded-diagnostic.js"; + +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); + +// Shape of the live Bedrock invoke K9 stream=true record observed on the +// scheduled main drift run: ~318 SDK chunk events plus ~90KiB of raw wire. +function liveK9StreamRecord(chunkCount = 318, chunkBytes = 280) { + const sdkEvents = Array.from({ length: chunkCount }, (_, index) => ({ + chunk: { bytes: new Uint8Array(chunkBytes).fill(97 + (index % 26)) }, + })); + const decoded = { wire: "invoke", stream: true, events: [{ type: "message_stop" }] }; + const record = { cell: { wire: "invoke", fault: "K9", stream: true }, sdkEvents, decoded }; + const wire = Buffer.alloc(chunkCount * chunkBytes, 0x62); + return { record, wire }; +} + +describe("boundedDiagnosticLine", () => { + it("keeps a small record complete, with byte arrays as base64", () => { + const wire = Buffer.from("raw-wire"); + const line = boundedDiagnosticLine( + { cell: "x", sdkEvents: [{ chunk: { bytes: new Uint8Array([123, 125]) } }] }, + wire, + ); + expect(JSON.parse(line)).toEqual({ + cell: "x", + sdkEvents: [{ chunk: { bytes: { encoding: "base64", bytes: "e30=" } } }], + rawWire: { encoding: "base64", bytes: wire.toString("base64") }, + }); + }); + + it("bounds the live K9 stream record that the reporter used to drop", () => { + const { record, wire } = liveK9StreamRecord(); + // The pre-fix line: JSON.stringify with the raw wire inline. + const unbounded = JSON.stringify({ + ...record, + rawWire: { encoding: "base64", bytes: wire.toString("base64") }, + }); + expect(fitsEvidenceRecord(unbounded)).toBe(false); + + const line = boundedDiagnosticLine(record, wire); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.decoded).toEqual(record.decoded); + expect(parsed.sdkEvents).toHaveLength(record.sdkEvents.length); + expect(parsed.sdkEvents[0].chunk.bytes).toEqual({ + encoding: "base64", + bytes: Buffer.from(record.sdkEvents[0].chunk.bytes).toString("base64"), + }); + expect(parsed.rawWire).toEqual({ encoding: "base64", bytes: wire.toString("base64") }); + }); + + it("digests the raw wire when the base64 events and wire together exceed the limit", () => { + const { record, wire } = liveK9StreamRecord(318, 400); + const line = boundedDiagnosticLine(record, wire); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.decoded).toEqual(record.decoded); + expect(parsed.sdkEvents).toHaveLength(318); + expect(parsed.rawWire).toEqual({ + omitted: true, + byteLength: wire.length, + sha256: sha256(wire), + }); + }); + + it("digests the SDK events too when they alone exceed the record limit", () => { + const { record, wire } = liveK9StreamRecord(400, 1024); + const line = boundedDiagnosticLine(record, wire); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.decoded).toEqual(record.decoded); + expect(parsed.sdkEvents).toMatchObject({ omitted: true, count: 400 }); + expect(parsed.sdkEvents.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(parsed.rawWire).toMatchObject({ omitted: true, byteLength: wire.length }); + }); +}); + +describe("boundedDiagnosticLine never returns an oversized line", () => { + const deltas = (count: number) => + Array.from({ length: count }, (_, index) => ({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: `delta ${index} `.repeat(8) }, + })); + + it("digests decoded events when they alone exceed the record limit", () => { + const decoded = { wire: "invoke", stream: true, events: deltas(3000) }; + const record = { cell: { wire: "invoke", fault: "K9", stream: true }, sdkEvents: [], decoded }; + expect(fitsEvidenceRecord(JSON.stringify(record))).toBe(false); + + const line = boundedDiagnosticLine(record, Buffer.from("wire")); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.cell).toEqual(record.cell); + const bytes = Buffer.from(JSON.stringify(decoded)); + expect(parsed.decoded).toEqual({ + omitted: true, + byteLength: bytes.length, + sha256: sha256(bytes), + }); + }); + + it("digests decodedEvents on the error path the same way", () => { + const record = { + cell: { wire: "invoke", fault: "K9", stream: true }, + sdkEvents: [], + decodedEvents: deltas(3000), + error: { name: "Error", message: "boom" }, + }; + const line = boundedDiagnosticLine(record, Buffer.from("wire")); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.decodedEvents).toMatchObject({ omitted: true }); + expect(parsed.error).toEqual(record.error); + }); + + it("falls back to a minimal line when other fields still exceed the limit", () => { + const record = { + cell: { wire: "converse", fault: "K5", stream: false }, + sdkEvents: [], + sdkResponse: { text: "y".repeat(MAX_EVIDENCE_RECORD_BYTES) }, + error: { name: "Error", message: "z".repeat(MAX_EVIDENCE_RECORD_BYTES) }, + }; + const line = boundedDiagnosticLine(record, Buffer.from("wire")); + expect(fitsEvidenceRecord(line)).toBe(true); + const parsed = JSON.parse(line); + expect(parsed.cell).toEqual(record.cell); + expect(parsed.omitted).toEqual(expect.arrayContaining(["sdkResponse", "error"])); + expect(parsed.byteLength).toBeGreaterThan(MAX_EVIDENCE_RECORD_BYTES); + expect(parsed.sha256).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +describe("fitsEvidenceRecord", () => { + it("applies the reporter limit to the JSON-encoded console record", () => { + // Quotes are escaped once more inside the record, so a line under the + // limit by raw length can still exceed it once encoded. + const quoted = '"'.repeat(MAX_EVIDENCE_RECORD_BYTES / 2); + expect(Buffer.byteLength(quoted)).toBeLessThan(MAX_EVIDENCE_RECORD_BYTES); + expect(fitsEvidenceRecord(quoted)).toBe(false); + expect(fitsEvidenceRecord("x".repeat(1024))).toBe(true); + }); +}); diff --git a/src/__tests__/drift/bounded-diagnostic.ts b/src/__tests__/drift/bounded-diagnostic.ts new file mode 100644 index 00000000..f7701541 --- /dev/null +++ b/src/__tests__/drift/bounded-diagnostic.ts @@ -0,0 +1,87 @@ +import { createHash } from "node:crypto"; +import { MAX_EVIDENCE_RECORD_BYTES } from "../../../scripts/drift-evidence-reporter.js"; + +// Vitest task ids are short; 64 characters leaves a margin for the envelope. +const TASK_ID_ALLOWANCE = "x".repeat(64); + +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); + +// Without this, JSON.stringify writes a Uint8Array as {"0":123,"1":34,...}, +// which is about ten times the size of its base64 form. +const bytesAsBase64 = (_key: string, value: unknown) => + value instanceof Uint8Array && !Buffer.isBuffer(value) + ? { encoding: "base64", bytes: Buffer.from(value).toString("base64") } + : value; + +/** Mirrors the evidence reporter's console record encoding and size check. */ +export function fitsEvidenceRecord(line: string, taskId = TASK_ID_ALLOWANCE): boolean { + const encoded = JSON.stringify({ + kind: "console", + taskId, + stream: "stdout", + content: line + "\n", + }); + return Buffer.byteLength(encoded) <= MAX_EVIDENCE_RECORD_BYTES; +} + +const digest = (value: unknown) => { + const bytes = Buffer.from(JSON.stringify(value, bytesAsBase64) ?? "null"); + return { omitted: true, byteLength: bytes.length, sha256: sha256(bytes) }; +}; + +/** + * Serialize one AWS drift diagnostic console line so the evidence reporter can + * still bind it to its test. + * + * The reporter replaces any console record over MAX_EVIDENCE_RECORD_BYTES with + * `console-unavailable`. That taints the owning test, and one tainted live + * assertion makes the collector withhold the whole observation artifact, so + * the native drift gate cannot grade the run. + * + * SDK event bytes are written as base64. While the line is still too large, + * the raw wire bytes, then the SDK events, then the decoded events are + * replaced by their size and SHA-256. If it still does not fit, the line keeps + * only the cell, the names of the omitted fields, and the size and SHA-256 of + * the complete line. Every returned line fits the reporter limit. + */ +export function boundedDiagnosticLine( + record: Record & { sdkEvents: unknown[] }, + wire: Buffer, +): string { + const rawWire = { encoding: "base64", bytes: wire.toString("base64") }; + const wireDigest = { omitted: true, byteLength: wire.length, sha256: sha256(wire) }; + const eventsDigest = { count: record.sdkEvents.length, ...digest(record.sdkEvents) }; + const decodedDigests = Object.fromEntries( + (["decoded", "decodedEvents"] as const) + .filter((key) => key in record) + .map((key) => [key, digest(record[key])]), + ); + const full = JSON.stringify({ ...record, rawWire }, bytesAsBase64); + const candidates = [ + () => full, + () => ({ ...record, rawWire: wireDigest }), + () => ({ ...record, sdkEvents: eventsDigest, rawWire: wireDigest }), + () => ({ + ...record, + sdkEvents: eventsDigest, + ...decodedDigests, + rawWire: wireDigest, + }), + ]; + for (const candidate of candidates) { + const value = candidate(); + const line = typeof value === "string" ? value : JSON.stringify(value, bytesAsBase64); + if (fitsEvidenceRecord(line)) return line; + } + const fullBytes = Buffer.from(full); + const summary = { + omitted: [...Object.keys(record).filter((key) => key !== "cell"), "rawWire"], + byteLength: fullBytes.length, + sha256: sha256(fullBytes), + }; + const minimal = JSON.stringify({ cell: record.cell, ...summary }, bytesAsBase64); + // `cell` is a short fixed tuple; drop it too rather than ever exceed the limit. + return fitsEvidenceRecord(minimal) + ? minimal + : JSON.stringify({ ...summary, omitted: [...summary.omitted, "cell"] }); +}