From 4329170775b9c20e1a74d22e3164dc6298fb3203 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 13:15:41 +0000 Subject: [PATCH 1/3] Retry PDM backtest cases on transport errors The PDM matrix runs against production PyPI and the public patch API. Over the last 7 days 25 pdm-compatibility runs failed on one random cell each, on unrelated PRs. The version, OS, shape, mode and check differed every time (rescanIdempotent, appliedExactlyOne, rescanAfterRelockApplies, ...). Each check judges a CLI scan, install or rollback. `Run` retries a command once, and only on a non-zero exit. The CLI usually reports an exhausted patch API fetch in its JSON while exiting zero, so the cell just fails a later check. Port backtest-poetry.py's case-level retry (#596). A case is re-run from a fresh directory, at most three attempts, only when every failed check recorded transport evidence from the operation it judged. Evidence is a failed command's request error, PyPI give-up, patch API 5xx or exhausted 429, or the same in the CLI's JSON error records. Functional failures are never retried, even when a later step raises a transport error. Failed attempts' logs go under attempts/ and are uploaded. A failing case now prints its failed checks' notes, since the job log alone never said why. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01N8YeCUhdg2tKdqyyY7z3sV --- .github/workflows/pdm-compatibility.yml | 1 + docs/testing/pdm-compatibility.md | 11 ++ scripts/backtest-pdm.py | 207 +++++++++++++++++++---- scripts/tests/test_backtest_harnesses.py | 114 +++++++++++++ 4 files changed, 304 insertions(+), 29 deletions(-) diff --git a/.github/workflows/pdm-compatibility.yml b/.github/workflows/pdm-compatibility.yml index 6af960cea..6f760d7c8 100644 --- a/.github/workflows/pdm-compatibility.yml +++ b/.github/workflows/pdm-compatibility.yml @@ -165,6 +165,7 @@ jobs: native-pdm/cases/**/cli-output.json native-pdm/cases/**/pdm.lock native-pdm/cases/**/*.log + native-pdm/attempts/** retention-days: 14 # The hermetic Rust capstone (wiremock Socket API that also serves the diff --git a/docs/testing/pdm-compatibility.md b/docs/testing/pdm-compatibility.md index 383fec433..70028e12b 100644 --- a/docs/testing/pdm-compatibility.md +++ b/docs/testing/pdm-compatibility.md @@ -140,5 +140,16 @@ every Windows cell used to skip, and a run whose cells all skip or whose PDM bootstrap fails is now an error. The matrix needs no Socket API token (the `urllib3@1.26.18` patch is a free tier). +The matrix runs against production PyPI and the public patch API, so a case +is re-run from a fresh directory, at most three attempts in total, when every +failed check is explained by a terminal transport failure in the operation it +judged: PDM, pip or uv giving up on PyPI, or the CLI's request error, patch +API 5xx or exhausted 429 retry, including one the CLI reports in its JSON +with exit code zero. Recovered retry warnings do not count, and a functional +failure is never retried, even when a later step raises a transport error. +Failed attempt logs stay under `attempts/--//`, and +the final row lists them in `transportRetries`. A failing case prints each +failed check's recorded detail to the job log. + Full run results belong with the source revision and toolchain versions in CI artifacts or a local output directory. See the [testing guide](README.md#ci-and-results). diff --git a/scripts/backtest-pdm.py b/scripts/backtest-pdm.py index c952e6cb8..0c02b3b85 100644 --- a/scripts/backtest-pdm.py +++ b/scripts/backtest-pdm.py @@ -328,6 +328,132 @@ def json_or_empty(self): return {} +# Terminal transport diagnostics (the set backtest-poetry.py retries on): +# PDM/pip/uv giving up on PyPI (requests/urllib3/httpx connection errors, +# "too many 503 error responses") or the CLI's own report of a request error, +# a patch API 5xx, or a 429 its retry loop gave up on. `Run`'s own one-shot +# retry only covers a non-zero exit; the CLI reports most of these in its JSON +# envelope while exiting zero, so the cell just fails a later check. +TRANSPORT_FAILURE = re.compile( + r"too many 5\d\d error responses|Max retries exceeded with url|" + r"NewConnectionError|ConnectTimeoutError|ReadTimeoutError|ProtocolError\(|" + r"raise ConnectionError\(|requests\.exceptions\.ConnectionError|ClosedPoolError|" + r"httpx\.(?:ConnectError|ReadError|RemoteProtocolError|ConnectTimeout|ReadTimeout)|" + r"Temporary failure in name resolution|nodename nor servname provided|Connection reset by peer|RemoteDisconnected|" + r"error sending request for url \(|API request failed with status 5\d\d\b|Rate limit exceeded \(HTTP 429" +) + + +def transport_diagnostic(text): + # A recovered pip/PDM/CLI retry warning is not a terminal failure, even + # when a later part of the same command fails for another reason. + text = "\n".join(line for line in text.splitlines() if "retrying" not in line.lower()) + match = TRANSPORT_FAILURE.search(text) + return match.group(0) if match else None + + +def operation_transport_failure(run): + """Transport evidence from the one operation a check judged, not from + other case logs. A failed command's output counts; a zero-exit CLI run + counts only through its JSON error records (an `error`, a failed or + skipped event, an `api_batch_failed` / `patch_details_failed` warning), + never through arbitrary successful output.""" + if run is None: + return None + if not run.ok(): + return transport_diagnostic(run.out) + + def diagnostic(value): + if isinstance(value, dict): + if value.get("error"): + found = transport_diagnostic(json.dumps(value["error"])) + if found: + return found + if (value.get("errorCode") or value.get("code") in ("api_batch_failed", "patch_details_failed") + or value.get("action") in ("failed", "skipped")): + found = transport_diagnostic(json.dumps(value)) + if found: + return found + return next((found for child in value.values() if (found := diagnostic(child))), None) + if isinstance(value, list): + return next((found for child in value if (found := diagnostic(child))), None) + return None + + return diagnostic(run.json_or_empty()) + + +def record_check(row, name, value, note=None, operation=None): + """Record one check. `operation`: the command (or tuple of commands) it + judges; a failed check whose operation shows a transport failure makes + the case retryable (see retry_transport).""" + row["checks"][name] = bool(value) + if note is not None: + row["info"][name] = note + operations = operation if isinstance(operation, tuple) else (operation,) + evidence = None if value else next((found for op in operations if (found := operation_transport_failure(op))), None) + if evidence: + row.setdefault("transportFailures", {})[name] = evidence + elif "transportFailures" in row: + row["transportFailures"].pop(name, None) + return bool(value) + + +def retry_transport(run_case, job, case, root, attempts=3, sleep=time.sleep): + """`run_case(job)`'s row, re-run from a clean case dir while the case + fails for transport reasons only: a FAIL row whose every failed check + recorded transport evidence from its own operation, or an exception whose + text carries a transport diagnostic. A functional failure is never + retried. A failed attempt's logs are kept under + /attempts/--// and listed on the row.""" + history = [] + for attempt in range(1, attempts + 1): + error = None + try: + row = run_case(job) + failed = [k for k, ok in row.get("checks", {}).items() if not ok] + causes = row.get("transportFailures", {}) + retryable = row.get("outcome") == "FAIL" and bool(failed) and all(k in causes for k in failed) + except Exception as e: + # A check that failed before the exception must itself be a + # transport failure for the case to be retried. + error, row = e, None + partial = getattr(e, "row", None) or {} + failed = [k for k, ok in partial.get("checks", {}).items() if not ok] + causes = partial.get("transportFailures", {}) + retryable = transport_diagnostic(str(e)) is not None and all(k in causes for k in failed) + if not retryable or attempt == attempts: + if error is not None: + if history: + error.transport_retries = history + raise error + if history: + row["transportRetries"] = history + if row.get("caseDir"): + save(Path(row["caseDir"]) / "result.json", row) + return row + evidence = root / "attempts" / "-".join(job) / str(attempt) + evidence.mkdir(parents=True, exist_ok=True) + for log in case.glob("*.log") if case.is_dir() else []: + shutil.copy2(log, evidence / log.name) + if row is not None and (case / "result.json").is_file(): + shutil.copy2(case / "result.json", evidence / "result.json") + history.append({ + "attempt": attempt, + "evidence": evidence.relative_to(root).as_posix(), + "failedChecks": failed, + "transport": (row or {}).get("transportFailures") or transport_diagnostic(str(error)), + }) + print(f"{' '.join(job)}: transport failure; retrying fresh case ({attempt}/{attempts})", flush=True) + sleep(10 * attempt) + + +def failure_details(row): + """One line per failed check with the note it recorded, so a job log + shows why a case failed without downloading the capture artifact.""" + info = row.get("info", {}) + return [f" {k}: {json.dumps(info[k], default=str)[:500]}" for k, ok in row.get("checks", {}).items() if not ok and k in info] + + def require(r, what): if not r.ok(): raise RuntimeError(f"{what} failed (exit {r.rc}):\n{r.tail(4000)}") @@ -889,7 +1015,7 @@ def manifestless_vex(check, info, version, case, project, lockname, pristine_loc rs = Run(install_cmd(version, "sync", sync_groups, None), vdir, venv_env, case / "vex-install.log", timeout=900, retry=True) res = oracle(version, vdir, venv_env, hashes, case / "vex-oracle.log") info["vexCheckoutInstall"] = {"exit": rs.rc, "patched": patched(res, hashes)} - check("vexCheckoutInstallsPatched", (rs.ok() or self_install_only(rs.out)) and patched(res, hashes), info["vexCheckoutInstall"]) + check("vexCheckoutInstallsPatched", (rs.ok() or self_install_only(rs.out)) and patched(res, hashes), info["vexCheckoutInstall"], rs) cenv_v = cli_env(version, vhome) def vex(log, *flags, via="vex"): @@ -916,15 +1042,15 @@ def omitted(env, reason): notes = info.setdefault("vex", {}) r, env_, doc = vex("vex-manifest-deleted.log") notes["manifestDeleted"] = {"exit": r.rc, "status": env_.get("status"), "error": (env_.get("error") or {}).get("code")} - check("vexManifestDeleted", r.ok() and attested(doc), notes["manifestDeleted"]) + check("vexManifestDeleted", r.ok() and attested(doc), notes["manifestDeleted"], r) for p in ledgers: p.unlink(missing_ok=True) r, env_, doc = vex("vex-ledgers-deleted.log") notes["ledgersDeleted"] = {"exit": r.rc, "status": env_.get("status"), "error": (env_.get("error") or {}).get("code")} - check("vexLedgersDeleted", r.ok() and attested(doc) and not (vdir / ".socket/manifest.json").exists(), notes["ledgersDeleted"]) + check("vexLedgersDeleted", r.ok() and attested(doc) and not (vdir / ".socket/manifest.json").exists(), notes["ledgersDeleted"], r) r, env_, doc = vex("vex-apply-embedded.log", via="apply") notes["applyEmbedded"] = {"exit": r.rc, "vex": env_.get("vex")} - check("vexApplyEmbedded", r.ok() and attested(doc), notes["applyEmbedded"]) + check("vexApplyEmbedded", r.ok() and attested(doc), notes["applyEmbedded"], r) r, env_, doc = vex("vex-offline.log", "--offline") notes["offlineNoLedger"] = {"exit": r.rc, "error": (env_.get("error") or {}).get("code")} check("vexOfflineUnavailable", r.rc == 1 and doc is None and omitted(env_, "record_unavailable"), notes["offlineNoLedger"]) @@ -951,17 +1077,29 @@ def prune(case, project): for path in (project / ".venv", project / "__pypackages__", case / "home", case / "native-cache", case / "saved-socket", project / ".socket"): shutil.rmtree(path, ignore_errors=True) + def case_dir(job): + version, shape, mode = job + return root / "cases" / version / shape / mode + def backtest(job): version, shape, mode = job - started = time.time() row = {"pdm": version, "python": python_for(version), "shape": shape, "mode": mode, "checks": {}, "info": {}, "passed": None, "outcome": None} + try: + return backtest_case(job, row) + except Exception as error: + # retry_transport must see the checks that already failed: a + # later transport exception must not retry away a functional + # failure. + error.row = row + raise + + def backtest_case(job, row): + version, shape, mode = job + started = time.time() checks, info = row["checks"], row["info"] - def check(name, value, note=None): - checks[name] = bool(value) - if note is not None: - info[name] = note - return bool(value) + def check(name, value, note=None, operation=None): + return record_check(row, name, value, note, operation) def finish(outcome): row["outcome"] = outcome @@ -991,7 +1129,7 @@ def native_broken(reason, tail): info["skip"] = meta["skip"] return finish("SKIP") lockname = meta["lockfile"] - case = root / "cases" / version / shape / mode + case = case_dir(job) if case.exists(): shutil.rmtree(case) case.mkdir(parents=True) @@ -1115,7 +1253,7 @@ def uninstall(log): row["expected"] = f"refused: lock_version {row['lockVersion']!r} is not supported by the rewriter" check("appliedZero", applied == 0, {"applied": applied, "codes": codes}) if shape != "custom-lockfile": - check("refusalCodeReported", bool(codes), codes) + check("refusalCodeReported", bool(codes), codes, r) check("lockUnchanged", lock_after == pristine_lock) if mode == "hosted": check("noPatchWiring", ledger_cleared(project, "hosted")) @@ -1131,7 +1269,7 @@ def uninstall(log): r, ok = native_sync("native-install.log") if not ok and applied == 0: return native_broken("pristine lock", r.out) - check("nativeInstallOk", ok, {"exit": r.rc, "tail": r.tail(300)}) + check("nativeInstallOk", ok, {"exit": r.rc, "tail": r.tail(300)}, r) res = oracle(version, project, penv, {}, case / "oracle-native.log") check("nativeInstallsPackage", res.get("installed") is (not excluded), res) info["installedOrigin"] = res.get("origin") @@ -1155,7 +1293,7 @@ def uninstall(log): if mode == "agent": found = envelope.get("apply", {}).get("found", 0) info["found"] = found - if not check("appliedExactlyOne", applied == 1, {"applied": applied, "found": found, "codes": codes, "status": envelope.get("status")}): + if not check("appliedExactlyOne", applied == 1, {"applied": applied, "found": found, "codes": codes, "status": envelope.get("status")}, r): return finish("FAIL") check("lockUnchanged", lock_after == pristine_lock) after, before, uuid = record_hashes(project, "agent") @@ -1167,13 +1305,13 @@ def uninstall(log): r2 = Run(cli_cmd(project, "scan", *scan_mode), project, cenv, case / "rescan.log", timeout=900, retry=True) e2 = r2.json_or_empty() res2 = oracle(version, project, penv, after, case / "oracle-2.log") - check("rescanIdempotent", r2.ok() and patched(res2, after) and (project / lockname).read_bytes() == pristine_lock, {"exit": r2.rc, "applied": applied_count("agent", e2)}) + check("rescanIdempotent", r2.ok() and patched(res2, after) and (project / lockname).read_bytes() == pristine_lock, {"exit": r2.rc, "applied": applied_count("agent", e2)}, r2) rs, ok = native_sync("sync-again.log") res3 = oracle(version, project, penv, after, case / "oracle-3.log") - check("survivesSync", ok and patched(res3, after), {"exit": rs.rc, "oracle": res3}) + check("survivesSync", ok and patched(res3, after), {"exit": rs.rc, "oracle": res3}, rs) ri = Run(install, project, penv, case / "install-again.log", timeout=900, retry=True) res4 = oracle(version, project, penv, after, case / "oracle-4.log") - check("survivesInstall", (ri.ok() or self_install_only(ri.out)) and patched(res4, after), {"exit": ri.rc, "oracle": res4}) + check("survivesInstall", (ri.ok() or self_install_only(ri.out)) and patched(res4, after), {"exit": ri.rc, "oracle": res4}, ri) lock_now = (project / lockname).read_bytes() info["ordinaryInstall"] = {"exit": ri.rc, "lockStable": lock_now == pristine_lock, "baselineFresh": row["baselineFresh"]} # The agent never touches the lock; a regenerated lock here is @@ -1182,17 +1320,19 @@ def uninstall(log): check("lockUnchangedAfterInstalls", lock_now == pristine_lock or row["baselineFresh"] is not True, info["ordinaryInstall"]) rb = Run(cli_cmd(project, "rollback"), project, cenv, case / "rollback.log", timeout=900) erb = rb.json_or_empty() - check("rollbackExit0", rb.ok(), rb.tail(600) if not rb.ok() else None) + check("rollbackExit0", rb.ok(), rb.tail(600) if not rb.ok() else None, rb) res5 = oracle(version, project, penv, before, case / "oracle-rollback.log") - check("rollbackRestoresUpstreamBytes", patched(res5, before), res5) - check("rollbackClearsManifest", ledger_cleared(project, "agent")) + # Both follow from the rollback: it restores the upstream bytes + # (fetched from the patch API) and then clears the manifest. + check("rollbackRestoresUpstreamBytes", patched(res5, before), res5, rb) + check("rollbackClearsManifest", ledger_cleared(project, "agent"), None, rb) check("rollbackKeepsPyproject", (project / "pyproject.toml").read_bytes() == pristine_pyproject) check("rollbackKeepsLock", (project / lockname).read_bytes() == lock_now) info["rollbackEnvelope"] = {k: erb.get(k) for k in ("status", "rolledBack", "failed") if k in erb} return finish("PASS" if all(checks.values()) else "FAIL") # ------------------------------------------------ hosted / vendored - if not check("appliedExactlyOne", applied == 1, {"applied": applied, "codes": codes, "status": envelope.get("status")}): + if not check("appliedExactlyOne", applied == 1, {"applied": applied, "codes": codes, "status": envelope.get("status")}, r): return finish("FAIL") check("lockRewritten", lock_after != pristine_lock) if shape == "crlf": @@ -1211,7 +1351,7 @@ def uninstall(log): # idempotent re-scan r2 = Run(cli_cmd(project, "scan", *scan_mode), project, cenv, case / "rescan.log", timeout=900, retry=True) e2 = r2.json_or_empty() - check("rescanIdempotent", r2.ok() and (project / lockname).read_bytes() == lock_after and (project / "pyproject.toml").read_bytes() == pristine_pyproject, {"exit": r2.rc, "applied": applied_count(mode, e2), "status": e2.get("status")}) + check("rescanIdempotent", r2.ok() and (project / lockname).read_bytes() == lock_after and (project / "pyproject.toml").read_bytes() == pristine_pyproject, {"exit": r2.rc, "applied": applied_count(mode, e2), "status": e2.get("status")}, r2) # lock-driven install of the patched artifact uninstall("uninstall.log") r, ok = native_sync("install.log") @@ -1226,7 +1366,7 @@ def uninstall(log): info["nativeBaseline"] = {"exit": rb0.rc, "ok": ok0, "tail": rb0.tail(300)} if not ok0: return native_broken("pristine lock fails too", rb0.out) - check("nativeInstallOk", ok, info["install"]) + check("nativeInstallOk", ok, info["install"], r) res = oracle(version, project, penv, after, case / "oracle-1.log") info["installedOrigin"] = res.get("origin") if excluded: @@ -1240,7 +1380,7 @@ def uninstall(log): ri = Run(install, project, penv, case / "ordinary-install.log", timeout=900, retry=True) ordinary_stable = (project / lockname).read_bytes() == lock_after info["ordinaryInstall"] = {"exit": ri.rc, "lockStable": ordinary_stable, "baselineFresh": row["baselineFresh"], "tail": ri.tail(300)} - check("ordinaryInstallOk", ri.ok() or self_install_only(ri.out), info["ordinaryInstall"]) + check("ordinaryInstallOk", ri.ok() or self_install_only(ri.out), info["ordinaryInstall"], ri) check("ordinaryInstallKeepsLock", ordinary_stable or row["baselineFresh"] is not True, info["ordinaryInstall"]) if not ordinary_stable: shutil.copyfile(project / lockname, case / "ordinary-result.lock") @@ -1270,7 +1410,7 @@ def uninstall(log): check("integrityRejected", tam.rc != 0 and not info["tamper"]["installedPatchedAnyway"], info["tamper"]) uninstall("tamper-uninstall2.log") r, ok = native_sync("reinstall.log") - check("reinstallOk", ok, r.tail(300)) + check("reinstallOk", ok, r.tail(300), r) # relock -> re-scan -> rollback (must restore the relocked bytes) saved = case / "saved-socket" # v5 hosted mode may leave no `.socket/` at all. @@ -1298,7 +1438,7 @@ def uninstall(log): failures = (erb1.get("hosted") or {}).get("failed") or erb1.get("vendoredFailed") or [] rollback_note = {"exit": rb1.rc, "status": erb1.get("status"), "failed": failures[:3], "lockEqualsRelocked": (project / lockname).read_bytes() == relocked} if target_kept: - check("rescanAfterRelockApplies", rs.ok() and marker in rescanned, info["rescanAfterRelock"]) + check("rescanAfterRelockApplies", rs.ok() and marker in rescanned, info["rescanAfterRelock"], rs) if mode == "vendored": # The re-scan re-wires the COMMITTED wheel (no service # call, no rebuild): the patched sha the first scan @@ -1315,8 +1455,11 @@ def uninstall(log): reuse_event = "vendor_artifact_reused" in info["rescanAfterRelock"]["codes"] reused = sha_kept and (reuse_event or not rewired) info["rescanAfterRelock"].update({"shaKept": sha_kept, "rewired": rewired, "reuseEvent": reuse_event, "reusesWheel": reused}) - check("rescanReusesWheel", reused, info["rescanAfterRelock"]) - check("rollbackAfterRelockPristine", rb1.ok() and rollback_note["lockEqualsRelocked"], rollback_note) + check("rescanReusesWheel", reused, info["rescanAfterRelock"], rs) + # A re-scan that never re-wired the relocked lock leaves the + # first scan's state for this rollback to undo, so it fails + # with the re-scan: judge it by the re-scan's transport too. + check("rollbackAfterRelockPristine", rb1.ok() and rollback_note["lockEqualsRelocked"], rollback_note, (rb1, rs)) else: # The relock resolved urllib3 away from 1.26.18 (PDM < 2.0 # has no overrides, so the pinned transitive resolution does @@ -1375,7 +1518,7 @@ def persist(): persist() with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool: - pending = {pool.submit(backtest, job): job for job in jobs} + pending = {pool.submit(retry_transport, backtest, job, case_dir(job), root): job for job in jobs} for fut in concurrent.futures.as_completed(pending): job = pending[fut] try: @@ -1383,8 +1526,14 @@ def persist(): results.append(row) failed = [k for k, ok in row["checks"].items() if not ok] say(*job, row["outcome"], ",".join(failed), f"{row.get('durationSeconds', '?')}s") + if row["outcome"] == "FAIL": + details = failure_details(row) + if details: + say("\n".join(details)) except Exception as e: err = {"pdm": job[0], "shape": job[1], "mode": job[2], "outcome": "ERROR", "error": str(e)[-3000:], "trace": traceback.format_exc()[-2000:]} + if getattr(e, "transport_retries", None): + err["transportRetries"] = e.transport_retries errors.append(err) results.append({"pdm": job[0], "python": python_for(job[0]), "shape": job[1], "mode": job[2], "outcome": "ERROR", "passed": False, "checks": {}, "info": {"error": str(e)[-600:]}}) say(*job, "ERROR", str(e)[-300:].replace("\n", " ")) diff --git a/scripts/tests/test_backtest_harnesses.py b/scripts/tests/test_backtest_harnesses.py index 381442507..e0e380536 100644 --- a/scripts/tests/test_backtest_harnesses.py +++ b/scripts/tests/test_backtest_harnesses.py @@ -305,6 +305,120 @@ def test_failure_details_list_only_failed_checks_with_notes(self): self.assertEqual(poetry.failure_details(row), [' appliedExactlyOne: {"applied": 0, "status": "error"}']) +class PdmTransportRetryTests(unittest.TestCase): + JOB = ("2.17.3", "space-unicode", "vendored") + + @staticmethod + def operation(rc=0, out=""): + run = object.__new__(pdm.Run) + run.cmd, run.rc, run.out = ["socket-patch", "scan"], rc, out + return run + + def drive(self, outcomes, root): + """Drive retry_transport with one scripted outcome per attempt: + `checks` maps a check to (passed, operation).""" + case = root / "cases" / "2.17.3" / "space-unicode" / "vendored" + calls = [] + + def run_case(_job): + outcome = outcomes[len(calls)] + calls.append(outcome) + if case.exists(): + pdm.shutil.rmtree(case) + case.mkdir(parents=True) + (case / "rescan-after-relock.log").write_text(outcome.get("log", "")) + row = {"checks": {}, "info": {}, "caseDir": str(case)} + for name, (ok, op) in outcome.get("checks", {}).items(): + pdm.record_check(row, name, ok, {"exit": getattr(op, "rc", None)}, op) + if "raise" in outcome: + error = RuntimeError(outcome["raise"]) + error.row = row + raise error + row["outcome"] = "PASS" if all(row["checks"].values()) else "FAIL" + return row + + sleeps = [] + try: + result = pdm.retry_transport(run_case, self.JOB, case, root, sleep=sleeps.append) + except RuntimeError as error: + result = error + return result, calls, sleeps + + def test_zero_exit_cli_transport_warning_is_retried_from_a_fresh_case(self): + # The CLI exits 0 and reports the exhausted patch API fetch in its + # envelope; `Run`'s exit-code retry never sees it. + envelope = json.dumps({"status": "partial_failure", "warnings": [{"code": "patch_details_failed", + "message": "API request failed with status 503: service unavailable"}]}) + blip = self.operation(0, envelope) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + row, calls, sleeps = self.drive([ + {"checks": {"rescanAfterRelockApplies": (False, blip), "rollbackAfterRelockPristine": (False, (self.operation(0, "{}"), blip))}, + "log": "first attempt"}, + {"checks": {"rescanAfterRelockApplies": (True, self.operation())}}, + ], root) + self.assertEqual((row["outcome"], len(calls), sleeps), ("PASS", 2, [10])) + retry = row["transportRetries"][0] + self.assertEqual(retry["failedChecks"], ["rescanAfterRelockApplies", "rollbackAfterRelockPristine"]) + self.assertEqual((root / retry["evidence"] / "rescan-after-relock.log").read_text(), "first attempt") + self.assertEqual(retry["evidence"], "attempts/2.17.3-space-unicode-vendored/1") + saved = json.loads((Path(row["caseDir"]) / "result.json").read_text()) + self.assertEqual(saved["transportRetries"], row["transportRetries"]) + + def test_nonzero_exit_cli_request_errors_are_retried(self): + for out in ("error sending request for url (https://patches-api.socket.dev/patch/view/x)", + "Rate limit exceeded (HTTP 429, gave up after 3 retries). Please try again later.", + "httpx.ConnectError: [Errno -3] Temporary failure in name resolution"): + with self.subTest(out=out), tempfile.TemporaryDirectory() as temp: + row, calls, _ = self.drive([{"checks": {"appliedExactlyOne": (False, self.operation(1, out))}}, + {"checks": {"appliedExactlyOne": (True, self.operation())}}], Path(temp)) + self.assertEqual((row["outcome"], len(calls)), ("PASS", 2)) + + def test_functional_failures_are_never_retried(self): + with tempfile.TemporaryDirectory() as temp: + # No operation, a refusal, a recovered retry warning, and a zero + # exit whose transport text sits outside any error record. + for checks in ({"lockRewritten": (False, None)}, + {"appliedExactlyOne": (False, self.operation(1, "error: pypi_pdm_lock_unsupported"))}, + {"nativeInstallOk": (False, self.operation(1, "Connection reset by peer, retrying\nResolutionImpossible"))}, + {"rescanIdempotent": (False, self.operation(0, json.dumps({"note": "error sending request for url ("})))}): + with self.subTest(checks=list(checks)): + row, calls, sleeps = self.drive([{"checks": checks}], Path(temp)) + self.assertEqual((row["outcome"], len(calls), sleeps), ("FAIL", 1, [])) + self.assertNotIn("transportRetries", row) + + def test_one_unexplained_failed_check_blocks_the_retry(self): + with tempfile.TemporaryDirectory() as temp: + row, calls, _ = self.drive([{"checks": { + "appliedExactlyOne": (False, self.operation(1, "error sending request for url (https://x)")), + "lockUnchanged": (False, None)}}], Path(temp)) + self.assertEqual((row["outcome"], len(calls)), ("FAIL", 1)) + + def test_transport_exception_retries_only_without_an_earlier_functional_failure(self): + with tempfile.TemporaryDirectory() as temp: + row, calls, _ = self.drive([{"raise": "no JSON in output: error sending request for url (https://x)"}, + {"checks": {"appliedExactlyOne": (True, self.operation())}}], Path(temp)) + self.assertEqual((row["outcome"], len(calls)), ("PASS", 2)) + error, calls, _ = self.drive([{"checks": {"lockRewritten": (False, None)}, + "raise": "no JSON in output: error sending request for url (https://x)"}], Path(temp)) + self.assertIsInstance(error, RuntimeError) + self.assertEqual(len(calls), 1) + + def test_a_persistent_transport_failure_stays_red_after_three_attempts(self): + blip = (False, self.operation(1, "error sending request for url (https://x)")) + with tempfile.TemporaryDirectory() as temp: + row, calls, sleeps = self.drive([{"checks": {"appliedExactlyOne": blip}}] * 3, Path(temp)) + self.assertEqual((row["outcome"], len(calls), sleeps), ("FAIL", 3, [10, 20])) + self.assertEqual([r["attempt"] for r in row["transportRetries"]], [1, 2]) + error, calls, _ = self.drive([{"raise": "ReadTimeoutError: read timed out"}] * 3, Path(temp)) + self.assertEqual((len(calls), [r["attempt"] for r in error.transport_retries]), (3, [1, 2])) + + def test_failure_details_list_only_failed_checks_with_notes(self): + row = {"checks": {"appliedExactlyOne": False, "lockRewritten": True, "lockUnchanged": False}, + "info": {"appliedExactlyOne": {"applied": 0}, "lockRewritten": {"x": 1}}} + self.assertEqual(pdm.failure_details(row), [' appliedExactlyOne: {"applied": 0}']) + + class PipenvShimTests(unittest.TestCase): def test_parallel_first_use(self): # Force every worker to reach symlink creation before any can create From 6b0302a7216a4103808ef3622af6a83cde8e3985 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 13:24:13 +0000 Subject: [PATCH 2/3] Judge PDM rollback and VEX checks by their run Bugbot: the final hosted/vendored rollback checks, the unverifiable- write rollback, the refused-lock VEX and the reverted-lock VEX runs named no operation, so a transport failure there never made the case retryable. installedBytesPatched fails together with a blipped pdm sync and blocked the retry the same way. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01N8YeCUhdg2tKdqyyY7z3sV --- scripts/backtest-pdm.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/backtest-pdm.py b/scripts/backtest-pdm.py index 0c02b3b85..35035c090 100644 --- a/scripts/backtest-pdm.py +++ b/scripts/backtest-pdm.py @@ -1057,7 +1057,7 @@ def omitted(env, reason): for p, data in saved.items(): p.write_bytes(data) (vdir / lockname).write_bytes(pristine_lock) - reverted = {} + reverted, reverted_runs = {}, [] for flags in ((), ("--no-verify",), ("--offline", "--no-verify")): r, env_, doc = vex("vex-reverted%s.log" % "".join(flags).replace("--", "-"), *flags) if mode == "hosted": @@ -1065,8 +1065,9 @@ def omitted(env, reason): else: dead = r.rc == 1 and omitted(env_, unwired) reverted[" ".join(flags) or "default"] = {"exit": r.rc, "dead": dead, "attested": attested(doc)} + reverted_runs.append(r) notes["reverted"] = reverted - check("vexRevertedUnwired", bool(saved) == (mode != "hosted") and all(v["dead"] and not v["attested"] for v in reverted.values()), reverted) + check("vexRevertedUnwired", bool(saved) == (mode != "hosted") and all(v["dead"] and not v["attested"] for v in reverted.values()), reverted, tuple(reverted_runs)) if not args.keep_environments: for path in (vvenv, vhome, vcache): shutil.rmtree(path, ignore_errors=True) @@ -1242,7 +1243,7 @@ def uninstall(log): and (project / "pyproject.toml").read_bytes() == pristine_pyproject ) info["rollbackOutsideProject"] = {"exit": rb.rc, "restored": restored} - check("rollbackRestoresUnverifiableWrite", restored) + check("rollbackRestoresUnverifiableWrite", restored, None, rb) return finish("UNSUPPORTED" if restored else "FAIL") expected_refusal = mode != "agent" and (row["lockVersion"] not in SUPPORTED_LOCK_VERSIONS or shape in EXPECTED_NOOP_SHAPES) @@ -1287,7 +1288,7 @@ def uninstall(log): vdoc = json.loads(vout.read_text(encoding="utf-8")) if vout.exists() else {} named = [st for st in vdoc.get("statements", []) if any(sc.get("@id", "").startswith(PURL_BASE) for p in st.get("products", []) for sc in p.get("subcomponents", []))] info["vexRefused"] = {"exit": rv.rc, "error": (rv.json_or_empty().get("error") or {}).get("code"), "statements": len(named)} - check("vexRefusedAttestsNothing", rv.rc != 0 and not named, info["vexRefused"]) + check("vexRefusedAttestsNothing", rv.rc != 0 and not named, info["vexRefused"], rv) return finish("REFUSED-EXPECTED" if all(checks.values()) else "FAIL") if mode == "agent": @@ -1370,9 +1371,9 @@ def uninstall(log): res = oracle(version, project, penv, after, case / "oracle-1.log") info["installedOrigin"] = res.get("origin") if excluded: - check("installedBytesPatched", res.get("installed") is False, {"expected": "not installed (marker excludes this host)", "oracle": res}) + check("installedBytesPatched", res.get("installed") is False, {"expected": "not installed (marker excludes this host)", "oracle": res}, r) else: - check("installedBytesPatched", patched(res, after), res) + check("installedBytesPatched", patched(res, after), res, r) check("lockUnchangedByInstall", (project / lockname).read_bytes() == lock_after) if not excluded: manifestless_vex(check, info, version, case, project, lockname, pristine_lock, mode, uuid, after, groups) @@ -1474,12 +1475,14 @@ def uninstall(log): # final rollback restores every byte rb = Run(cli_cmd(project, "rollback"), project, cenv, case / "rollback.log", timeout=900) erb = rb.json_or_empty() - check("rollbackExit0", rb.ok(), rb.tail(600) if not rb.ok() else None) - check("rollbackRestoresLockBytes", (project / lockname).read_bytes() == pristine_lock) - check("rollbackKeepsPyproject", (project / "pyproject.toml").read_bytes() == pristine_pyproject) - check("rollbackClearsLedger", ledger_cleared(project, mode)) + # A rollback that stops on a transport error leaves the lock, ledger + # and wheel it had not yet restored: those follow from `rb`. + check("rollbackExit0", rb.ok(), rb.tail(600) if not rb.ok() else None, rb) + check("rollbackRestoresLockBytes", (project / lockname).read_bytes() == pristine_lock, None, rb) + check("rollbackKeepsPyproject", (project / "pyproject.toml").read_bytes() == pristine_pyproject, None, rb) + check("rollbackClearsLedger", ledger_cleared(project, mode), None, rb) if mode == "vendored": - check("rollbackRemovesVendoredWheel", not (project / ".socket/vendor/pypi" / (uuid or "x")).exists()) + check("rollbackRemovesVendoredWheel", not (project / ".socket/vendor/pypi" / (uuid or "x")).exists(), None, rb) info["rollbackEnvelope"] = {k: erb.get(k) for k in ("status", "rolledBack", "failed", "vendoredReverted") if k in erb} if erb.get("hosted"): info["rollbackEnvelope"]["hosted"] = {k: erb["hosted"].get(k) for k in ("reverted", "failed", "unsupported", "editedFiles")} From 80538db4f7e4fcf9bcbc49156ee8ecf5ccf4fb9f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 13:43:19 +0000 Subject: [PATCH 3/3] Port #851: fix vex alias tests broken on main Main is red since #605 (4646693): two commands::vex_consumed tests assume the name-keyed resolver never returns npm-aliased copies, and #605 taught it to find them. This fails socket-patch-cli --lib in coverage and test on every PR. Port #851's test-only fix so this PR can go green; it no-ops once #851 lands. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01N8YeCUhdg2tKdqyyY7z3sV --- .../src/commands/vex_consumed.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/vex_consumed.rs b/crates/socket-patch-cli/src/commands/vex_consumed.rs index b57d475fb..cb0c68023 100644 --- a/crates/socket-patch-cli/src/commands/vex_consumed.rs +++ b/crates/socket-patch-cli/src/commands/vex_consumed.rs @@ -715,8 +715,11 @@ mod tests { None, ) .await; - assert_eq!(installed_again, installed); - let (paths, calls) = tracked_npm_hosted(&common, &installed_again).await; + // Since #605 the name-keyed resolver probes bundled trees itself, so + // it already returns the aliases and the nested store's peers. Feed + // the earlier, alias-free set to keep exercising alias expansion; + // the resolver's own set is checked against the same result below. + let (paths, calls) = tracked_npm_hosted(&common, &installed).await; assert_eq!(calls.len(), 1); let mut inputs = calls[0].clone(); inputs.sort(); @@ -738,6 +741,9 @@ mod tests { .len(), paths.len() ); + let (mut resolved, _) = tracked_npm_hosted(&common, &installed_again).await; + resolved.sort(); + assert_eq!(resolved, expected, "the resolver's own copy set"); } #[cfg(unix)] @@ -768,14 +774,19 @@ mod tests { None, ) .await; - assert!(installed.is_empty(), "{installed:?}"); - let (mut paths, calls) = tracked_npm_hosted(&common, &installed).await; + // Since #605 the name-keyed resolver reaches the alias and its + // sibling peers on its own. An alias-only set (what an alias-blind + // resolver returns) must still expand to the same copies. + let (mut paths, calls) = tracked_npm_hosted(&common, &HashMap::new()).await; assert_eq!(calls, vec![vec![alias.clone()]]); let mut expected = peers; expected.push(alias); paths.sort(); expected.sort(); assert_eq!(paths, expected); + let (mut resolved, _) = tracked_npm_hosted(&common, &installed).await; + resolved.sort(); + assert_eq!(resolved, expected, "the resolver's own copy set"); } #[cfg(unix)]