diff --git a/scripts/merge-queue-fail-fast.py b/scripts/merge-queue-fail-fast.py index 0791acf16..452208d22 100644 --- a/scripts/merge-queue-fail-fast.py +++ b/scripts/merge-queue-fail-fast.py @@ -13,6 +13,13 @@ failed and cancelled `needs`, and reports failure within seconds, and the queue evicts the entry and rebuilds the ones behind it. +A job whose runner died is the exception (see `runner_lost`): it says +nothing about the PR, and cancelling the run for it turned one dead runner +into an eviction plus a burst of cancelled jobs. Those are left alone, so +`ci-ok` decides when the run finishes. The job can't be re-run instead: +`POST .../actions/jobs/{id}/rerun` is refused until the whole run has +completed, and by then `ci-ok` has already reported. + The script never fails its own job: any API error is a warning and the watch carries on (or gives up), leaving the run to finish as before. """ @@ -31,14 +38,32 @@ # Job conclusions that make `ci-ok` fail. `cancelled` is left out: it # means someone (the queue, a janitor, a person) is already cancelling. FATAL = {"failure", "timed_out"} +# Check-run annotations that mean the job's runner went away, not that a +# step failed. On Depot runners these arrive as a job `failure` (its step +# shows `cancelled`) about a minute into the job, whatever the job runs; +# "lost communication" is the same death noticed by GitHub's 10-minute +# heartbeat. Matched case-insensitively against failure-level messages. +RUNNER_LOSS = ("lost communication", "shutdown signal", "step canceled by github") def fatal_jobs(jobs): """Names of the completed jobs whose conclusion dooms `ci-ok`.""" - return sorted( - j["name"] + return sorted(j["name"] for j in failed_jobs(jobs)) + + +def failed_jobs(jobs): + return [ + j for j in jobs if j.get("status") == "completed" and j.get("conclusion") in FATAL and j.get("name") != GATE + ] + + +def runner_lost(annotations): + """True when a job's failure annotations say its runner died.""" + return any( + a.get("annotation_level") == "failure" and any(m in (a.get("message") or "").lower() for m in RUNNER_LOSS) + for a in annotations ) @@ -74,6 +99,11 @@ def list_jobs(repo, run_id, token): page += 1 +def annotations(repo, job_id, token): + # A job's id is also its check-run id. + return request("GET", f"repos/{repo}/check-runs/{job_id}/annotations?per_page=100", token) + + def summary(text): print(text) path = os.environ.get("GITHUB_STEP_SUMMARY") @@ -89,6 +119,7 @@ def main(): interval = int(os.environ.get("POLL_SECONDS", "30")) deadline = time.monotonic() + int(os.environ.get("WATCH_MINUTES", "100")) * 60 run = None + lost = {} # job id -> runner_lost verdict; a completed job's annotations don't change while time.monotonic() < deadline: try: if run is None: @@ -100,7 +131,15 @@ def main(): if request("GET", f"repos/{repo}/actions/runs/{run['id']}", token)["status"] == "completed": print("CI run completed; nothing to do") return 0 - failed = fatal_jobs(list_jobs(repo, run["id"], token)) + failed = [] + for j in failed_jobs(list_jobs(repo, run["id"], token)): + if j["id"] not in lost: + lost[j["id"]] = runner_lost(annotations(repo, j["id"], token)) + if lost[j["id"]]: + summary(f"Not cancelling for `{j['name']}`: its runner died ({j.get('html_url', '')}); `ci-ok` decides.") + if not lost[j["id"]]: + failed.append(j["name"]) + failed.sort() if failed: request("POST", f"repos/{repo}/actions/runs/{run['id']}/cancel", token) summary( diff --git a/scripts/tests/test_merge_queue_fail_fast.py b/scripts/tests/test_merge_queue_fail_fast.py index afc5590e6..7efefb9f3 100644 --- a/scripts/tests/test_merge_queue_fail_fast.py +++ b/scripts/tests/test_merge_queue_fail_fast.py @@ -1,9 +1,12 @@ """Tests for scripts/merge-queue-fail-fast.py.""" import importlib.util +import io import re import unittest +from contextlib import redirect_stdout from pathlib import Path +from unittest import mock ROOT = Path(__file__).parents[2] CI = ROOT / ".github" / "workflows" / "ci.yml" @@ -37,6 +40,73 @@ def test_cancelled_jobs_and_the_gate_itself_are_ignored(self): self.assertEqual(ff.fatal_jobs(jobs), []) +def note(message, level="failure"): + return {"annotation_level": level, "message": message} + + +class RunnerLost(unittest.TestCase): + """Annotations GitHub attached to jobs whose Depot runner died (2026-10-09).""" + + def test_depot_runner_deaths_are_recognised(self): + for message in ( + "Step canceled by GitHub (see: https://depot.dev/docs/github-actions/troubleshooting#error-step-canceled-by-github)", + "The runner has received a shutdown signal. This can happen when the runner service is stopped, " + "or a manually started runner is canceled.", + "The self-hosted runner lost communication with the server. Verify the machine is running and has " + "a healthy network connection.", + ): + with self.subTest(message=message[:30]): + self.assertTrue(ff.runner_lost([note("Node.js 20 is deprecated.", "warning"), note(message)])) + + def test_a_real_failure_is_not_a_runner_loss(self): + self.assertFalse(ff.runner_lost([])) + self.assertFalse(ff.runner_lost([note("Process completed with exit code 101.")])) + + def test_only_failure_level_annotations_count(self): + self.assertFalse(ff.runner_lost([note("Step canceled by GitHub", "warning")])) + + +class Watch(unittest.TestCase): + """main() against a fake API: which failed jobs make it cancel the run.""" + + def watch(self, notes_by_job): + jobs = [ + {"id": i, "name": f"job{i}", "status": "completed", "conclusion": "failure", "html_url": f"u{i}"} + for i in notes_by_job + ] + calls = [] + + def request(method, path, token): + calls.append((method, path)) + if "/annotations" in path: + return notes_by_job[int(path.split("/")[-2])] + if path.endswith("/jobs?filter=latest&per_page=100&page=1"): + return {"jobs": jobs} + if "/workflows/ci.yml/runs" in path: + return {"workflow_runs": [{"id": 7, "html_url": "run"}]} + if path.endswith("/runs/7"): + # Report completion on the second poll so the watch ends. + done = sum(1 for _, p in calls if p.endswith("/runs/7")) > 1 + return {"status": "completed" if done else "in_progress"} + return None + + env = {"GITHUB_REPOSITORY": "o/r", "HEAD_SHA": "abc", "GH_TOKEN": "t", "POLL_SECONDS": "0"} + with mock.patch.dict(ff.os.environ, env, clear=True), mock.patch.object(ff, "request", request), \ + mock.patch.object(ff.time, "sleep"), redirect_stdout(io.StringIO()): + ff.main() + return calls + + def test_a_runner_loss_does_not_cancel_the_run(self): + calls = self.watch({1: [note("Step canceled by GitHub (see: https://depot.dev/...)")]}) + self.assertNotIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls) + # The verdict is cached: annotations are fetched once across polls. + self.assertEqual(sum("/annotations" in p for _, p in calls), 1) + + def test_a_real_failure_still_cancels_the_run(self): + calls = self.watch({1: [note("The runner has received a shutdown signal.")], 2: [note("exit code 1")]}) + self.assertIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls) + + class CiWorkflowContract(unittest.TestCase): """Cancelling on a failed job is safe only while ci.yml keeps these."""