diff --git a/README.md b/README.md index 7644283c..2fc6f6fd 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ Dispatch can optionally run issue grooming itself by calling an OpenAI-compatibl | `DISPATCH_GROOMER_MODEL` | Conditional | Model name sent to the chat completions API. Required when hosted grooming is enabled. | | `DISPATCH_GROOMER_TIMEOUT_MS` | No | LLM request timeout. Defaults to a scaled value of `60s + 5s/KB of maxContextBytes`, clamped to 60s–300s. | | `DISPATCH_GROOMER_MAX_CONTEXT_BYTES` | No | Issue context budget sent to the model. Defaults to `8192`. | +| `DISPATCH_CI_FAILURES_INTERVAL_MS` | No | Interval for the `ci-failures` scheduler job, which files an issue when a workflow fails twice in a row on a repo's default branch and closes it when the workflow goes green. Defaults to `1800000` (30m). | +| `DISPATCH_CI_FAILURE_LABELS` | No | Comma-separated labels applied to those issues. Defaults to `type/bug,status/ready`. Set to an empty string to apply none. | | `DISPATCH_GROOMER_CONTEXT_MODE` | No | Exploration budget preset: `small`, `medium` (default), `large`. | | `DISPATCH_GROOMER_MODEL_CONTEXT_TOKENS` | No | The model's real context window in tokens. Derives the exploration budget from it and ignores the preset. Recommended for self-hosted models. | | `DISPATCH_GROOMER_DRY_RUN` | No | Defaults to `true`; when true, returns a mutation plan without GitHub or DB writes. | diff --git a/src/app/api/ci-failures/sync/route.ts b/src/app/api/ci-failures/sync/route.ts new file mode 100644 index 00000000..2e9f9e61 --- /dev/null +++ b/src/app/api/ci-failures/sync/route.ts @@ -0,0 +1,202 @@ +import { NextRequest, NextResponse } from "next/server"; +import { errorResponse } from "@/lib/api-errors"; +import { authorizeRequest } from "@/lib/auth"; +import { getTrackedRepos } from "@/lib/config"; +import { + fetchRepositoryMetadata, + fetchRecentRunsAllWorkflows, + fetchRunJobs, + fetchFailedJobLogExcerpt, + fetchIssues, + createIssue, + addIssueComment, + closeIssue, +} from "@/lib/github"; +import { + buildCloseComment, + buildIssueDraft, + classifyWorkflow, + computeFailureSignature, + decideAction, + extractFailureMarker, + groupDefaultBranchRuns, + type CiRun, + type FiledIssue, +} from "@/lib/ci-failure-ingestion"; +import { enforceRateLimit } from "@/lib/rate-limit"; +import { acquireLock, releaseLock, type AcquiredLock, type LockConflict } from "@/lib/sync-lock"; + +/** + * Files an issue when a workflow fails twice in a row on a repo's default + * branch, and closes it when the workflow goes green again. + * + * `pr-followup` covers failing checks on a pull request. This covers the same + * failure with no PR attached, which previously reached nothing at all. + * + * Holds no state: consecutiveness comes from the workflow's own recent runs, + * and "already filed" from a marker in the issue body. See + * `lib/ci-failure-ingestion.ts` for the policy and why each part of it is the + * way it is. + */ + +/** Labels applied to a filed issue. Configurable so this carries no + * assumption about any one deployment's taxonomy. */ +function issueLabels(): string[] { + const raw = process.env.DISPATCH_CI_FAILURE_LABELS; + if (raw === undefined) return ["type/bug", "status/ready"]; + return raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +} + +async function filedIssuesFor(repoFullName: string): Promise { + const issues = await fetchIssues(repoFullName, { includeClosed: true }); + const filed: FiledIssue[] = []; + for (const issue of issues) { + const signature = extractFailureMarker(issue.body); + if (!signature) continue; + filed.push({ + number: issue.number, + state: issue.state === "closed" ? "closed" : "open", + signature, + }); + } + return filed; +} + +export async function POST(request: NextRequest) { + const auth = await authorizeRequest(request); + if (!auth.authorized) { + return errorResponse("Unauthorized", 401); + } + + const limited = enforceRateLimit(`ci-failures-sync:${auth.actor}`, { + limit: 10, + windowMs: 60_000, + }); + if (limited) return limited; + + let lock: AcquiredLock | LockConflict | undefined; + lock = await acquireLock("ci-failures"); + if (!lock?.locked) { + return NextResponse.json( + { error: "CI failure sync is already running", locked: true }, + { status: 409 }, + ); + } + + const filedIssues: { repo: string; number: number; workflow: string }[] = []; + const closedIssues: { repo: string; number: number; workflow: string }[] = []; + const skipped: { repo: string; workflow: string; reason: string }[] = []; + const errors: { repo: string; error: string }[] = []; + + try { + for (const repoFullName of await getTrackedRepos()) { + try { + const [meta, runs] = await Promise.all([ + fetchRepositoryMetadata(repoFullName), + fetchRecentRunsAllWorkflows(repoFullName, 50), + ]); + const histories = groupDefaultBranchRuns(runs as CiRun[], meta.defaultBranch); + if (histories.length === 0) continue; + + // One listing per repo, reused for every workflow: the marker lookup + // is the same set of issues each time. + const filed = await filedIssuesFor(repoFullName); + + for (const history of histories) { + const state = classifyWorkflow(history); + + // Only a repeated failure needs the job and its log, so the + // expensive calls stay off the healthy path entirely. + let signature: string | null = null; + let jobName = ""; + let logExcerpt = ""; + if (state.kind === "repeated-failure") { + const jobs = await fetchRunJobs(repoFullName, state.latest.id); + const failing = jobs.find((j) => j.conclusion === "failure"); + if (!failing) { + skipped.push({ + repo: repoFullName, + workflow: history.workflowName, + reason: "run failed but no job reported failure", + }); + continue; + } + jobName = failing.name; + logExcerpt = await fetchFailedJobLogExcerpt(repoFullName, failing.id).catch(() => ""); + signature = computeFailureSignature({ + repoFullName, + workflowName: history.workflowName, + jobName, + logExcerpt, + }); + } + + const action = decideAction(state, signature, filed); + if (action.action === "none") { + skipped.push({ + repo: repoFullName, + workflow: history.workflowName, + reason: action.reason, + }); + continue; + } + + if (action.action === "close") { + const green = state.kind === "healthy" ? state.latest : null; + if (!green) continue; + await addIssueComment(repoFullName, action.issueNumber, buildCloseComment(green)); + await closeIssue(repoFullName, action.issueNumber); + closedIssues.push({ + repo: repoFullName, + number: action.issueNumber, + workflow: history.workflowName, + }); + continue; + } + + if (state.kind !== "repeated-failure") continue; + const draft = buildIssueDraft({ + repoFullName, + workflowName: history.workflowName, + jobName, + signature: action.signature, + latest: state.latest, + previous: state.previous, + logExcerpt, + supersedes: action.supersedes, + }); + const created = await createIssue(repoFullName, { + title: draft.title, + body: draft.body, + labels: issueLabels(), + }); + // Keep the local view current so a second workflow in the same repo + // with the same signature does not file a duplicate in this pass. + filed.push({ number: created.number, state: "open", signature: action.signature }); + filedIssues.push({ + repo: repoFullName, + number: created.number, + workflow: history.workflowName, + }); + } + } catch (error) { + // Per-repo isolation: one unreachable repo must not abort the pass. + errors.push({ repo: repoFullName, error: String(error) }); + } + } + } finally { + if (lock && lock.locked) { + await releaseLock(lock.runId).catch(() => undefined); + } + } + + return NextResponse.json({ + filed: filedIssues, + closed: closedIssues, + skipped, + errors, + }); +} diff --git a/src/lib/ci-failure-ingestion.test.ts b/src/lib/ci-failure-ingestion.test.ts new file mode 100644 index 00000000..d6537279 --- /dev/null +++ b/src/lib/ci-failure-ingestion.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; +import { + buildCloseComment, + buildFailureMarker, + buildIssueDraft, + classifyWorkflow, + computeFailureSignature, + decideAction, + extractFailureMarker, + groupDefaultBranchRuns, + type CiRun, + type FiledIssue, +} from "./ci-failure-ingestion"; + +function run(over: Partial = {}): CiRun { + return { + id: 1, + name: "Release", + status: "completed", + conclusion: "failure", + head_branch: "main", + head_sha: "deadbeefcafe", + html_url: "https://example.test/run/1", + updated_at: "2026-09-04T02:00:00Z", + ...over, + }; +} + +describe("computeFailureSignature", () => { + const base = { + repoFullName: "o/r", + workflowName: "Release", + jobName: "Build", + logExcerpt: "error: openssl 3.5.5-1ubuntu3.3 is vulnerable", + }; + + it("is stable for the same failure", () => { + expect(computeFailureSignature(base)).toBe(computeFailureSignature({ ...base })); + }); + + it("ignores run-specific noise so consecutive failures match", () => { + // Without normalisation these differ every run and nothing is ever filed, + // because no two failures ever look consecutive. + const a = computeFailureSignature({ + ...base, + logExcerpt: "2026-09-04T02:00:00Z run 33830329465 failed after 66.4s sha256:abc123def456", + }); + const b = computeFailureSignature({ + ...base, + logExcerpt: "2026-09-05T09:13:11Z run 33999111222 failed after 71.2s sha256:fff999eee888", + }); + expect(a).toBe(b); + }); + + it("separates different jobs, workflows and repos", () => { + expect(computeFailureSignature({ ...base, jobName: "Test" })).not.toBe( + computeFailureSignature(base), + ); + expect(computeFailureSignature({ ...base, workflowName: "Nightly" })).not.toBe( + computeFailureSignature(base), + ); + expect(computeFailureSignature({ ...base, repoFullName: "o/other" })).not.toBe( + computeFailureSignature(base), + ); + }); + + it("separates genuinely different errors", () => { + expect(computeFailureSignature({ ...base, logExcerpt: "permission denied" })).not.toBe( + computeFailureSignature(base), + ); + }); +}); + +describe("failure marker", () => { + it("round-trips", () => { + expect(extractFailureMarker(`body\n${buildFailureMarker("abc123")}`)).toBe("abc123"); + }); + + it("returns null for a body without one", () => { + expect(extractFailureMarker("just an issue")).toBeNull(); + expect(extractFailureMarker(null)).toBeNull(); + expect(extractFailureMarker(undefined)).toBeNull(); + }); +}); + +describe("groupDefaultBranchRuns", () => { + it("keeps only completed runs on the default branch", () => { + const grouped = groupDefaultBranchRuns( + [ + run({ id: 1 }), + run({ id: 2, head_branch: "feature" }), + run({ id: 3, status: "in_progress", conclusion: null }), + run({ id: 4, name: "Other" }), + ], + "main", + ); + expect(grouped.map((g) => g.workflowName).sort()).toEqual(["Other", "Release"]); + expect(grouped.find((g) => g.workflowName === "Release")!.runs).toHaveLength(1); + }); + + it("orders each workflow newest first", () => { + const grouped = groupDefaultBranchRuns( + [ + run({ id: 1, updated_at: "2026-09-01T00:00:00Z" }), + run({ id: 2, updated_at: "2026-09-04T00:00:00Z" }), + run({ id: 3, updated_at: "2026-09-02T00:00:00Z" }), + ], + "main", + ); + expect(grouped[0].runs.map((r) => r.id)).toEqual([2, 3, 1]); + }); +}); + +describe("classifyWorkflow", () => { + it("calls a single failure a first failure", () => { + const s = classifyWorkflow({ workflowName: "Release", runs: [run({ id: 2 })] }); + expect(s.kind).toBe("first-failure"); + }); + + it("calls two failures in a row repeated", () => { + const s = classifyWorkflow({ + workflowName: "Release", + runs: [run({ id: 3 }), run({ id: 2 })], + }); + expect(s.kind).toBe("repeated-failure"); + }); + + it("does not call a failure after a success repeated", () => { + const s = classifyWorkflow({ + workflowName: "Release", + runs: [run({ id: 3 }), run({ id: 2, conclusion: "success" })], + }); + expect(s.kind).toBe("first-failure"); + }); + + it("reports green", () => { + const s = classifyWorkflow({ + workflowName: "Release", + runs: [run({ id: 3, conclusion: "success" })], + }); + expect(s.kind).toBe("healthy"); + }); + + it("treats cancelled and skipped as unknown, not failure", () => { + for (const conclusion of ["cancelled", "skipped", "neutral", "timed_out"]) { + const s = classifyWorkflow({ workflowName: "R", runs: [run({ conclusion })] }); + expect(s.kind).toBe("unknown"); + } + }); + + it("reports unknown with no history", () => { + expect(classifyWorkflow({ workflowName: "R", runs: [] }).kind).toBe("unknown"); + }); +}); + +describe("decideAction", () => { + const repeated = classifyWorkflow({ + workflowName: "Release", + runs: [run({ id: 3 }), run({ id: 2 })], + }); + const healthy = classifyWorkflow({ + workflowName: "Release", + runs: [run({ id: 3, conclusion: "success" })], + }); + + it("files on a repeated failure with nothing open", () => { + expect(decideAction(repeated, "sig1", [])).toEqual({ + action: "file", + signature: "sig1", + supersedes: null, + }); + }); + + it("does not file twice for the same signature", () => { + const filed: FiledIssue[] = [{ number: 7, state: "open", signature: "sig1" }]; + expect(decideAction(repeated, "sig1", filed)).toEqual({ + action: "none", + reason: "already filed as #7", + }); + }); + + it("files fresh and links the old one when a closed failure returns", () => { + const filed: FiledIssue[] = [{ number: 7, state: "closed", signature: "sig1" }]; + expect(decideAction(repeated, "sig1", filed)).toEqual({ + action: "file", + signature: "sig1", + supersedes: 7, + }); + }); + + it("files when an open issue exists for a different failure of the same workflow", () => { + const filed: FiledIssue[] = [{ number: 7, state: "open", signature: "other" }]; + expect(decideAction(repeated, "sig1", filed)).toMatchObject({ action: "file" }); + }); + + it("never files on a first failure", () => { + const first = classifyWorkflow({ workflowName: "Release", runs: [run({ id: 3 })] }); + expect(decideAction(first, "sig1", [])).toMatchObject({ action: "none" }); + }); + + it("closes an open issue when the workflow goes green", () => { + const filed: FiledIssue[] = [{ number: 7, state: "open", signature: "sig1" }]; + expect(decideAction(healthy, null, filed)).toEqual({ + action: "close", + issueNumber: 7, + signature: "sig1", + }); + }); + + it("closes an open issue even if the current signature differs", () => { + // Green means no failure of this workflow is outstanding, whatever the + // open issue was originally about. + const filed: FiledIssue[] = [{ number: 7, state: "open", signature: "old" }]; + expect(decideAction(healthy, "new", filed)).toMatchObject({ action: "close", issueNumber: 7 }); + }); + + it("does nothing when green with nothing open", () => { + expect(decideAction(healthy, null, [])).toMatchObject({ action: "none" }); + }); + + it("does nothing without a signature", () => { + expect(decideAction(repeated, null, [])).toMatchObject({ action: "none" }); + }); +}); + +describe("buildIssueDraft", () => { + const opts = { + repoFullName: "o/r", + workflowName: "Release", + jobName: "Vulnerability Scan", + signature: "sig1", + latest: run({ id: 3, html_url: "https://example.test/3" }), + previous: run({ id: 2, html_url: "https://example.test/2" }), + logExcerpt: "openssl 3.5.5-1ubuntu3.3 fixed in 3.5.5-1ubuntu3.4", + supersedes: null, + }; + + it("names both runs, the job, and carries the marker", () => { + const d = buildIssueDraft(opts); + expect(d.title).toContain("Release"); + expect(d.title).toContain("Vulnerability Scan"); + expect(d.body).toContain("https://example.test/3"); + expect(d.body).toContain("https://example.test/2"); + expect(d.body).toContain("openssl 3.5.5-1ubuntu3.3"); + expect(extractFailureMarker(d.body)).toBe("sig1"); + }); + + it("says why a single red run was not filed", () => { + expect(buildIssueDraft(opts).body).toContain("A single red run is not filed"); + }); + + it("links the superseded issue when the failure returned", () => { + const d = buildIssueDraft({ ...opts, supersedes: 7 }); + expect(d.body).toContain("#7"); + expect(d.body).toContain("did not hold"); + }); + + it("tolerates a missing log excerpt", () => { + const d = buildIssueDraft({ ...opts, logExcerpt: "" }); + expect(d.body).toContain("(no log excerpt available)"); + }); + + it("caps a huge excerpt", () => { + const d = buildIssueDraft({ ...opts, logExcerpt: "x".repeat(50_000) }); + expect(d.body.length).toBeLessThan(6_000); + }); +}); + +describe("buildCloseComment", () => { + it("names the run that cleared it", () => { + const c = buildCloseComment(run({ conclusion: "success", html_url: "https://example.test/9" })); + expect(c).toContain("https://example.test/9"); + expect(c).toContain("green again"); + expect(c).toContain("a fresh issue is filed"); + }); +}); diff --git a/src/lib/ci-failure-ingestion.ts b/src/lib/ci-failure-ingestion.ts new file mode 100644 index 00000000..c6255cdb --- /dev/null +++ b/src/lib/ci-failure-ingestion.ts @@ -0,0 +1,242 @@ +/** + * Turn repeated CI failures on a repository's default branch into issues. + * + * `pr-followup` already ingests a failing check when it happens on a pull + * request. The same failure on the default branch reaches nothing: it has no + * PR, so nothing ingests it and nothing files anything. A release build that + * breaks after a merge is therefore invisible to the queue. + * + * The policy this implements, and why: + * + * - **Only a second consecutive failure files.** One red run is weak evidence: + * network blips and rate limits are common and clear themselves. Two in a + * row with the same signature is a condition, not an accident. + * - **Every workflow is eligible.** The repeat rule is the noise filter. An + * allowlist would silently exclude whichever workflow nobody remembered to + * add, which is the failure this exists to catch. + * - **A green run closes the issue.** An auto-filed issue nobody auto-closes + * goes stale the moment CI recovers, and a queue of stale auto-issues trains + * people to ignore the label. + * - **A recurrence files fresh, linking the old one.** A failure returning + * after its issue closed means the fix did not hold — new information, and it + * should re-enter the queue with its own priority rather than reopening an + * issue whose body describes a different occurrence. + * + * Deliberately holds no state of its own. Consecutiveness is read from the + * workflow's own recent runs, and "already filed" from a marker in the issue + * body — so there is no table to migrate and an instance that has never seen a + * repo behaves exactly like one that has. + */ + +import { createHash } from "crypto"; + +/** Marker embedded in an auto-filed issue body, mirroring the AI reviewer's. */ +const MARKER_PREFIX = "dispatch-ci-failure"; +const MARKER_RE = //i; + +/** A workflow run, narrowed to what this module needs. */ +export interface CiRun { + id: number; + name: string; + status: string; + conclusion: string | null; + head_branch: string; + head_sha: string; + html_url: string; + updated_at: string; +} + +export interface FailureSignatureInput { + repoFullName: string; + workflowName: string; + jobName: string; + logExcerpt: string; +} + +/** + * Stable identity for "this workflow is failing for this reason". + * + * The log excerpt is normalised before hashing so run-specific noise — ids, + * timestamps, hex digests, durations — does not make every occurrence look + * like a new failure. Without that, consecutive failures never match and + * nothing is ever filed. + */ +export function computeFailureSignature(input: FailureSignatureInput): string { + const normalised = (input.logExcerpt || "") + .replace(/\b[0-9a-f]{7,64}\b/gi, "") + .replace(/\b\d{4}-\d{2}-\d{2}[T ][\d:.]+Z?\b/g, "") + .replace(/\b\d+(\.\d+)?(ms|s|m|h)\b/gi, "") + .replace(/\b\d{3,}\b/g, "") + .replace(/\s+/g, " ") + .trim() + .slice(0, 2000); + return createHash("sha256") + .update(`${input.repoFullName}\n${input.workflowName}\n${input.jobName}\n${normalised}`) + .digest("hex") + .slice(0, 16); +} + +export function buildFailureMarker(signature: string): string { + return ``; +} + +export function extractFailureMarker(body: string | null | undefined): string | null { + const m = MARKER_RE.exec(body || ""); + return m ? m[1].toLowerCase() : null; +} + +/** Runs for one workflow on the default branch, newest first. */ +export interface WorkflowHistory { + workflowName: string; + runs: CiRun[]; +} + +/** + * Group completed default-branch runs by workflow, newest first. + * + * In-progress runs are excluded rather than treated as neither-failed-nor- + * passed: a queued run says nothing about the condition, and letting it sit + * between two failures would break the consecutiveness test. + */ +export function groupDefaultBranchRuns(runs: CiRun[], defaultBranch: string): WorkflowHistory[] { + const byWorkflow = new Map(); + for (const run of runs) { + if (run.head_branch !== defaultBranch) continue; + if (run.status !== "completed") continue; + if (!run.conclusion) continue; + const list = byWorkflow.get(run.name) ?? []; + list.push(run); + byWorkflow.set(run.name, list); + } + return [...byWorkflow.entries()].map(([workflowName, list]) => ({ + workflowName, + runs: list.sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at)), + })); +} + +export type WorkflowState = + | { kind: "healthy"; latest: CiRun } + | { kind: "first-failure"; latest: CiRun } + | { kind: "repeated-failure"; latest: CiRun; previous: CiRun } + | { kind: "unknown" }; + +/** + * Classify a workflow from its recent default-branch history. + * + * `healthy` is what closes an existing issue; `repeated-failure` is what files + * one. `first-failure` deliberately does neither — it is the transient case, + * and acting on it is what would flood the queue. + */ +export function classifyWorkflow(history: WorkflowHistory): WorkflowState { + const [latest, previous] = history.runs; + if (!latest) return { kind: "unknown" }; + if (latest.conclusion === "success") return { kind: "healthy", latest }; + if (latest.conclusion !== "failure") return { kind: "unknown" }; + if (!previous || previous.conclusion !== "failure") { + return { kind: "first-failure", latest }; + } + return { kind: "repeated-failure", latest, previous }; +} + +export interface FiledIssue { + number: number; + state: "open" | "closed"; + signature: string; +} + +export type IngestAction = + | { action: "none"; reason: string } + | { action: "file"; signature: string; supersedes: number | null } + | { action: "close"; issueNumber: number; signature: string }; + +/** + * Decide what to do about one workflow, given its state and any issues already + * filed for it. + * + * Pure so the policy is testable without touching GitHub — every branch here + * is a decision someone argued about on #931, and each one has a test. + */ +export function decideAction( + state: WorkflowState, + signature: string | null, + filed: FiledIssue[], +): IngestAction { + const openForSignature = filed.find((i) => i.state === "open" && i.signature === signature); + + if (state.kind === "healthy") { + // Close whatever is open for this workflow, whichever signature it carries: + // the workflow is green, so no failure of it is outstanding. + const anyOpen = filed.find((i) => i.state === "open"); + return anyOpen + ? { action: "close", issueNumber: anyOpen.number, signature: anyOpen.signature } + : { action: "none", reason: "workflow is green and nothing is open" }; + } + if (state.kind === "first-failure") { + return { action: "none", reason: "first failure; waiting for a second to rule out a transient" }; + } + if (state.kind === "unknown" || !signature) { + return { action: "none", reason: "no actionable state" }; + } + if (openForSignature) { + return { action: "none", reason: `already filed as #${openForSignature.number}` }; + } + // A closed issue for the same signature means the fix did not hold. File + // fresh and link it rather than reopening a body that describes the earlier + // occurrence and may already carry a merged PR. + const closedForSignature = filed.find((i) => i.state === "closed" && i.signature === signature); + return { action: "file", signature, supersedes: closedForSignature?.number ?? null }; +} + +export interface IssueDraft { + title: string; + body: string; +} + +/** Render the issue an actionable failure produces. */ +export function buildIssueDraft(opts: { + repoFullName: string; + workflowName: string; + jobName: string; + signature: string; + latest: CiRun; + previous: CiRun; + logExcerpt: string; + supersedes: number | null; +}): IssueDraft { + const { workflowName, jobName, latest, previous, logExcerpt, supersedes } = opts; + const excerpt = (logExcerpt || "").trim().slice(0, 4000) || "(no log excerpt available)"; + const lines = [ + `\`${workflowName}\` has failed twice in a row on the default branch, in the same job and for the same reason.`, + "", + `- Latest: [${latest.html_url}](${latest.html_url}) (\`${latest.head_sha.slice(0, 8)}\`)`, + `- Previous: [${previous.html_url}](${previous.html_url}) (\`${previous.head_sha.slice(0, 8)}\`)`, + `- Failing job: \`${jobName}\``, + "", + "A single red run is not filed — this one repeated, so it is a condition rather than a transient.", + "", + "```", + excerpt, + "```", + ]; + if (supersedes !== null) { + lines.push( + "", + `This failure was filed before as #${supersedes} and closed. It has returned, so the earlier fix did not hold.`, + ); + } + lines.push("", buildFailureMarker(opts.signature)); + return { + title: `CI: ${workflowName} failing on the default branch (${jobName})`, + body: lines.join("\n"), + }; +} + +export function buildCloseComment(greenRun: CiRun): string { + return [ + `Closing: \`${greenRun.name}\` is green again on the default branch.`, + "", + `Cleared by [${greenRun.html_url}](${greenRun.html_url}) (\`${greenRun.head_sha.slice(0, 8)}\`).`, + "", + "If it fails again a fresh issue is filed, linking this one.", + ].join("\n"); +} diff --git a/src/lib/github-facades.test.ts b/src/lib/github-facades.test.ts index 238320d1..908e996c 100644 --- a/src/lib/github-facades.test.ts +++ b/src/lib/github-facades.test.ts @@ -19,6 +19,7 @@ describe("github domain modules expose expected exports", () => { "addIssueComment", "addIssueLabel", "closeIssue", + "createIssue", "fetchIssue", "fetchIssueComments", "fetchIssues", diff --git a/src/lib/github-issues.ts b/src/lib/github-issues.ts index 94a58c2a..67be5cff 100644 --- a/src/lib/github-issues.ts +++ b/src/lib/github-issues.ts @@ -229,3 +229,28 @@ export async function closeIssue( throw new Error(`GitHub API error: ${response.status} ${text}`); } } + +/** + * Open a new issue. Used by the CI-failure ingester; every other issue in + * Dispatch arrives from GitHub rather than being created by it. + */ +export async function createIssue( + repoFullName: string, + input: { title: string; body: string; labels?: string[] }, +): Promise<{ number: number; html_url: string }> { + const response = await fetchWithRetry(`${GITHUB_API}/repos/${repoFullName}/issues`, { + method: "POST", + headers: { ...(await getHeadersAsync()), "Content-Type": "application/json" }, + body: JSON.stringify({ + title: input.title, + body: input.body, + ...(input.labels?.length ? { labels: input.labels } : {}), + }), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`Failed to create issue in ${repoFullName}: ${response.status} ${text}`); + } + const data = await response.json(); + return { number: data.number, html_url: data.html_url }; +} diff --git a/src/lib/github.ts b/src/lib/github.ts index b750a8f8..45987e0c 100644 --- a/src/lib/github.ts +++ b/src/lib/github.ts @@ -13,6 +13,7 @@ export { type GitHubIssueComment, fetchIssueComments, addIssueComment, + createIssue, updateIssueComment, addIssueLabel, type UpdateIssueFields, diff --git a/src/lib/scheduler.test.ts b/src/lib/scheduler.test.ts index 701877d6..8e5ccd1a 100644 --- a/src/lib/scheduler.test.ts +++ b/src/lib/scheduler.test.ts @@ -45,14 +45,16 @@ describe("schedulerConfigFromEnv", () => { expect(schedulerConfigFromEnv({ PORT: "" }).baseUrl).toBe("http://127.0.0.1:3000"); }); - it("configures sync + groomer + pr-followup + prune-closed + reconcile + stale-work with defaults", () => { + it("configures sync + groomer + pr-followup + ci-failures + prune-closed + reconcile + stale-work with defaults", () => { const jobs = schedulerConfigFromEnv({}).jobs; - expect(jobs.map((j) => j.name)).toEqual(["sync", "groomer", "pr-followup", "prune-closed", "reconcile", "stale-work"]); + expect(jobs.map((j) => j.name)).toEqual(["sync", "groomer", "pr-followup", "ci-failures", "prune-closed", "reconcile", "stale-work"]); const byName = (n: string) => jobs.find((j) => j.name === n)!; expect(byName("groomer").path).toBe("/api/groomer/run"); expect(byName("groomer").intervalMs).toBe(10 * 60 * 1000); expect(byName("pr-followup").path).toBe("/api/pr-followup/sync"); expect(byName("pr-followup").intervalMs).toBe(15 * 60 * 1000); + expect(byName("ci-failures").path).toBe("/api/ci-failures/sync"); + expect(byName("ci-failures").intervalMs).toBe(30 * 60 * 1000); expect(byName("prune-closed").path).toBe("/api/issues/prune-closed"); expect(byName("prune-closed").intervalMs).toBe(24 * 60 * 60 * 1000); expect(byName("reconcile").path).toBe("/api/issues/reconcile"); diff --git a/src/lib/scheduler.ts b/src/lib/scheduler.ts index f26e4bd9..8ebad623 100644 --- a/src/lib/scheduler.ts +++ b/src/lib/scheduler.ts @@ -52,6 +52,7 @@ const DEFAULT_PR_FOLLOWUP_INTERVAL_MS = 15 * 60 * 1000; // 15m const DEFAULT_PRUNE_CLOSED_INTERVAL_MS = 24 * 60 * 60 * 1000; // daily const DEFAULT_RECONCILE_INTERVAL_MS = 30 * 60 * 1000; // 30m const DEFAULT_STALE_WORK_INTERVAL_MS = 5 * 60 * 1000; // 5m +const DEFAULT_CI_FAILURES_INTERVAL_MS = 30 * 60 * 1000; // 30m const DEFAULT_STARTUP_DELAY_MS = 5 * 1000; function intFromEnv(raw: string | undefined, fallback: number): number { @@ -95,6 +96,15 @@ export function schedulerConfigFromEnv(env: Record): body: {}, intervalMs: jobIntervalFromEnv(env.DISPATCH_PR_FOLLOWUP_INTERVAL_MS, DEFAULT_PR_FOLLOWUP_INTERVAL_MS), }, + { + name: "ci-failures", + path: "/api/ci-failures/sync", + body: {}, + intervalMs: jobIntervalFromEnv( + env.DISPATCH_CI_FAILURES_INTERVAL_MS, + DEFAULT_CI_FAILURES_INTERVAL_MS, + ), + }, { name: "prune-closed", path: "/api/issues/prune-closed", diff --git a/src/lib/sync-lock.ts b/src/lib/sync-lock.ts index 208ff26c..9a4846df 100644 --- a/src/lib/sync-lock.ts +++ b/src/lib/sync-lock.ts @@ -74,7 +74,8 @@ export type SyncType = | "automation" | "pr-followup" | "reconcile" - | "stale-work"; + | "stale-work" + | "ci-failures"; /** * Lock key per sync type. One row per key in the same `sync_lock` table that @@ -99,6 +100,7 @@ const LOCK_KEY_BY_TYPE: Record = { "pr-followup": "pr-followup", reconcile: "reconcile", "stale-work": "stale-work", + "ci-failures": "ci-failures", }; /**