diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bebde5fc..106f591a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -412,16 +412,19 @@ jobs: python3 scripts/ci-test-shard.py "$TEST_SHARD" 2 test-release: - # Not in the merge queue: each PR's head already ran this, and so did - # every PR ahead of it, so a merge group would re-spend ~30 min (23 of it - # compiling) on the same release-mode suite while the queue waits. It - # still runs on every PR and on main after each merge; `ci-ok` counts a - # skipped job as passing. + # Each PR and main push runs the complete release-mode suite. The merge + # queue already skips this job because each constituent PR ran it. if: github.event.pull_request.draft != true && github.event_name != 'merge_group' runs-on: ubuntu-latest - # Every tests/ target is its own optimized link (~240 test binaries). - # The manifest-less VEX suites share two multi-module binaries - # (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) to keep the count down. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + # Warm-cache runs still spend 20-23 minutes compiling ~240 optimized + # test binaries, followed by 5-6 minutes executing them. Split both + # compilation and execution with the same partitioner as `test`: + # shard 1 owns the unit tests/doctests and fewer integration targets. + # `ci-ok` waits for every shard and fails if any shard fails. timeout-minutes: 40 steps: - name: Checkout @@ -449,7 +452,10 @@ jobs: # semantics this job exists to validate; ~23m of LTO relinking gone. # Default features for the same reason as the `test` job; the # feature-gated suites' compile rot is e2e-build's. - run: cargo test --workspace --profile ci-release + env: + TEST_SHARD: ${{ matrix.shard }} + TEST_SHARD_COUNT: ${{ strategy.job-total }} + run: python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT" --locked --profile ci-release coverage: if: github.event.pull_request.draft != true @@ -1500,10 +1506,10 @@ jobs: # The exact release a leg names rather than the runner's Maven. The # tarball comes from Maven Central's CDN (well under a second); # archive.apache.org throttles bulk downloads to 1.5-5.5 minutes per - # leg. Its sha512 still comes from the Apache archive (Central has - # none for 3.6.3/3.8.9), so the bytes are checked against a digest - # from a second origin. --ssl-revoke-best-effort: see the `test` - # job's vexctl step. + # leg. If the CDN fails, use the slower archive tarball. Both + # sources must match the committed SHA512 in scripts/maven-sha512.json; + # an origin cannot replace both the archive and its expected digest. + # --ssl-revoke-best-effort: see the `test` job's vexctl step. shell: bash env: MAVEN_VERSION: ${{ matrix.maven || '3.9.16' }} @@ -1511,10 +1517,20 @@ jobs: major="${MAVEN_VERSION%%.*}" file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" - sha_url="https://archive.apache.org/dist/maven/maven-${major}/${MAVEN_VERSION}/binaries/${file}.sha512" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" - python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' + archive_url="https://archive.apache.org/dist/maven/maven-${major}/${MAVEN_VERSION}/binaries/${file}" + if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then + echo "::warning::Maven Central download failed; falling back to the Apache archive." + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$archive_url" -o "$RUNNER_TEMP/maven.tgz" + fi + python - "$MAVEN_VERSION" "$RUNNER_TEMP/maven.tgz" <<'PY' + import hashlib, json, sys + from pathlib import Path + # Pins come from Apache's release checksums, cross-checked against + # the Maven Central tarballs. Add a pin when adding a matrix version. + expected = json.loads(Path("scripts/maven-sha512.json").read_text())[sys.argv[1]] + if hashlib.sha512(Path(sys.argv[2]).read_bytes()).hexdigest() != expected: + raise SystemExit("Maven archive SHA512 mismatch") + PY # Python accepts native Windows paths for both archive and destination. python -m tarfile -e "$RUNNER_TEMP/maven.tgz" "$RUNNER_TEMP" launcher="$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin/mvn" diff --git a/.github/workflows/gradle-compatibility.yml b/.github/workflows/gradle-compatibility.yml index 70081afb1..847ba7521 100644 --- a/.github/workflows/gradle-compatibility.yml +++ b/.github/workflows/gradle-compatibility.yml @@ -42,6 +42,7 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: + - 'scripts/maven-sha512.json' - '.github/workflows/gradle-compatibility.yml' - 'scripts/ci-e2e-bundle.py' - 'Cargo.lock' @@ -256,14 +257,22 @@ jobs: # TLS handshake with CRYPT_E_REVOCATION_OFFLINE whenever the CA's # revocation server is unreachable. A revoked certificate still fails; # the body is checked against a digest right after. A no-op elsewhere. - # Tarball from Maven Central's CDN, digest from the Apache archive, - # which throttles the tarball itself to minutes (ci.yml's copy). + # Use Maven Central's fast CDN, falling back to the slower Apache + # archive. Both must match the committed digest (ci.yml's copy). file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" - sha_url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${file}.sha512" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" - python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' + archive_url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${file}" + if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then + echo "::warning::Maven Central download failed; falling back to the Apache archive." + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$archive_url" -o "$RUNNER_TEMP/maven.tgz" + fi + python - "$MAVEN_VERSION" "$RUNNER_TEMP/maven.tgz" <<'PY' + import hashlib, json, sys + from pathlib import Path + expected = json.loads(Path("scripts/maven-sha512.json").read_text())[sys.argv[1]] + if hashlib.sha512(Path(sys.argv[2]).read_bytes()).hexdigest() != expected: + raise SystemExit("Maven archive SHA512 mismatch") + PY python -m tarfile -e "$RUNNER_TEMP/maven.tgz" "$RUNNER_TEMP" launcher="$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin/mvn" if [ "$RUNNER_OS" = Windows ]; then launcher="${launcher}.cmd"; fi diff --git a/scripts/ci-test-shard.py b/scripts/ci-test-shard.py index 6aed76b4d..05336ce9b 100644 --- a/scripts/ci-test-shard.py +++ b/scripts/ci-test-shard.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Run one shard of ci.yml's `test` job: `cargo test --workspace` split over -`COUNT` runners so the macOS / Windows legs stop being the merge queue's -critical path (one leg spent ~10 min linking ~240 test binaries and ~15 min -running them). +"""Run one shard of ci.yml's `test` or `test-release` job: +`cargo test --workspace` split over `COUNT` runners. Each runner compiles +and executes only its assigned integration targets, shortening both the +debug and release-mode jobs without changing their compilation profiles. Shard 1 runs the unit tests (`--lib --bins`, ~4 min of the run on Windows) and the doctests; the integration-test targets (`cargo metadata`, kind @@ -15,6 +15,7 @@ any of them failed. python3 scripts/ci-test-shard.py 1 2 + python3 scripts/ci-test-shard.py 1 3 --locked --profile ci-release """ import json diff --git a/scripts/maven-sha512.json b/scripts/maven-sha512.json new file mode 100644 index 000000000..066ab5605 --- /dev/null +++ b/scripts/maven-sha512.json @@ -0,0 +1,9 @@ +{ + "3.6.3": "c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0", + "3.8.9": "4a490b7f331a0e7869b61da24600241e445339f2801ed94e32f835b63ed78597ad05ef8c1cce2501b4c2c3dcde30030eb395cd5756be739c20ac687ad6f82f0e", + "3.9.2": "900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db", + "3.9.3": "400fc5b6d000c158d5ee7937543faa06b6bda8408caa2444a9c947c21472fde0f0b64ac452b8cec8855d528c0335522ed5b6c8f77085811c7e29e1bedbb5daa2", + "3.9.4": "deaa39e16b2cf20f8cd7d232a1306344f04020e1f0fb28d35492606f647a60fe729cc40d3cba33e093a17aed41bd161fe1240556d0f1b80e773abd408686217e", + "3.9.16": "831a8591fe20c8243b1dbe7d71e3244f31d1665b0804b2e825e38cbbe5ce0cafb8338851f90780735568773e0a6cd07bbec107cda0b896b008b861075358b6f6", + "4.0.0-rc-6": "3fba58e1c345a5aa1dbacfa7aceaf7b1a0fa9626e368eec4814fa7a7ebf0fe74f0e41481faef77f95d8738f9c1365f918c8b8c94d7c28656f067db61a8af7f2e" +} diff --git a/scripts/tests/test_ci_maven_download.py b/scripts/tests/test_ci_maven_download.py new file mode 100644 index 000000000..033f8269f --- /dev/null +++ b/scripts/tests/test_ci_maven_download.py @@ -0,0 +1,130 @@ +"""Exercise the workflow installers against a fake CDN and Apache archive. + +The archive bytes must match the committed pin before extraction, even if +the download origin serves a matching checksum for substituted bytes. +""" + +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).parents[2] +WORKFLOWS = ("ci.yml", "gradle-compatibility.yml") +VERSION = "3.9.16" +spec = importlib.util.spec_from_file_location("ci_maven_rows", Path(__file__).with_name("test_ci_vlt_rows.py")) +rows = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rows) + + +def installer(workflow): + jobs = rows.jobs((ROOT / ".github/workflows" / workflow).read_text()) + step = next(text for job in jobs.values() for name, text in rows.steps(job) + if name.startswith("Install Maven")) + match = re.search(r"(?m)^ run: \|\n((?: .*\n|\n)+)", step + "\n") + return textwrap.dedent(match[1]) + + +def archive_bytes(contents): + data = io.BytesIO() + with tarfile.open(fileobj=data, mode="w:gz") as archive: + for name in ("mvn", "mvn.cmd"): + info = tarfile.TarInfo(f"apache-maven-{VERSION}/bin/{name}") + info.size = len(contents) + info.mode = 0o755 + archive.addfile(info, io.BytesIO(contents)) + return data.getvalue() + + +CURL = r''' +import hashlib, os, pathlib, sys +root = pathlib.Path(os.environ["FIXTURE_ROOT"]) +mode = os.environ["FIXTURE_MODE"] +url = next(a for a in sys.argv if a.startswith("https://")) +dest = pathlib.Path(sys.argv[sys.argv.index("-o") + 1]) +with (root / "requests").open("a") as log: + log.write(url + "\n") +if mode == "both-fail" or ("repo.maven.apache.org" in url and mode in ("fallback", "corrupt-fallback")): + dest.write_bytes(b"partial download") + sys.exit(22) +payload = (root / ("tampered.tgz" if mode.startswith("corrupt-") else "fixture.tgz")).read_bytes() +if url.endswith(".sha512"): + # An origin can replace both its tarball and its online digest. + payload = hashlib.sha512(payload).hexdigest().encode() +dest.write_bytes(payload) +''' + + +class MavenDownload(unittest.TestCase): + def test_every_matrix_version_has_a_sha512_pin(self): + pins = json.loads((ROOT / "scripts/maven-sha512.json").read_text()) + for workflow in WORKFLOWS: + text = (ROOT / ".github/workflows" / workflow).read_text() + versions = re.findall(r"(?:maven|MAVEN_VERSION): '([^']+)'", text) + self.assertTrue(versions) + self.assertFalse(set(versions) - pins.keys(), f"unpinned Maven version in {workflow}") + for version, digest in pins.items(): + self.assertRegex(digest, r"^[0-9a-f]{128}$", version) + + def test_workflow_downloads_fail_closed_before_extraction(self): + for workflow in WORKFLOWS: + script = installer(workflow) + for runner_os in ("Linux", "Windows"): + for mode in ("primary", "fallback", "corrupt-primary", "corrupt-fallback", "both-fail", "unpinned"): + with self.subTest(workflow=workflow, runner_os=runner_os, mode=mode): + self.check_installer(script, workflow, runner_os, mode) + + def check_installer(self, script, workflow, runner_os, mode): + with tempfile.TemporaryDirectory(prefix="maven-download-") as directory: + work = Path(directory) + trusted = archive_bytes(b"trusted launcher\n") + (work / "fixture.tgz").write_bytes(trusted) + (work / "tampered.tgz").write_bytes(archive_bytes(b"substituted launcher\n")) + (work / "scripts").mkdir() + pins = {} if mode == "unpinned" else {VERSION: hashlib.sha512(trusted).hexdigest()} + (work / "scripts/maven-sha512.json").write_text(json.dumps(pins)) + (work / "curl").write_text("#!" + sys.executable + "\n" + CURL) + (work / "curl").chmod(0o755) + (work / "python").symlink_to(sys.executable) + github_env = work / "github-env" + github_env.touch() + env = dict(os.environ, PATH=str(work) + os.pathsep + os.environ["PATH"], + FIXTURE_ROOT=str(work), FIXTURE_MODE=mode, RUNNER_TEMP=str(work), + RUNNER_OS=runner_os, MAVEN_VERSION=VERSION, GITHUB_ENV=str(github_env), + PYTHONOPTIMIZE="1") # Verification must not rely on assert. + result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=work, env=env, capture_output=True, text=True, timeout=30) + requests = (work / "requests").read_text().splitlines() + self.assertTrue(requests[0].startswith("https://repo.maven.apache.org/")) + self.assertFalse(any(url.endswith(".sha512") for url in requests), requests) + fallback = mode in ("fallback", "corrupt-fallback", "both-fail") + self.assertEqual(len(requests), 2 if fallback else 1, requests) + if fallback: + self.assertEqual(requests[1], f"https://archive.apache.org/dist/maven/maven-3/{VERSION}/binaries/apache-maven-{VERSION}-bin.tar.gz") + if mode in ("primary", "fallback"): + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + launcher = work / f"apache-maven-{VERSION}/bin" / ("mvn.cmd" if runner_os == "Windows" else "mvn") + self.assertEqual(launcher.read_bytes(), b"trusted launcher\n") + self.assertIn(f"SOCKET_PATCH_MAVEN_E2E_MVN={launcher}\n", github_env.read_text()) + if workflow == "gradle-compatibility.yml": + self.assertIn(f"SOCKET_PATCH_MAVEN_E2E_VERSION={VERSION}\n", github_env.read_text()) + self.assertIn("SOCKET_PATCH_MAVEN_E2E_REQUIRED=1\n", github_env.read_text()) + else: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse((work / f"apache-maven-{VERSION}").exists()) + self.assertFalse(github_env.read_text()) + if mode.startswith("corrupt-"): + self.assertIn("SHA512 mismatch", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_ci_test_shard.py b/scripts/tests/test_ci_test_shard.py index c35eaa61d..bb2d446e0 100644 --- a/scripts/tests/test_ci_test_shard.py +++ b/scripts/tests/test_ci_test_shard.py @@ -1,11 +1,15 @@ -"""ci-test-shard.py: the `test` job's shards together run exactly the old -single `cargo test --workspace` selection.""" +"""ci-test-shard.py: the debug/release shards preserve the workspace tests.""" import importlib.util import json +import os +import re +import shutil import subprocess +import tempfile import unittest from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).parents[2] spec = importlib.util.spec_from_file_location("ci_test_shard", ROOT / "scripts" / "ci-test-shard.py") @@ -49,6 +53,24 @@ def test_every_run_is_workspace_wide_and_keeps_going(self): self.assertEqual(args[:4], ["cargo", "test", "--workspace", "--no-fail-fast"]) self.assertIn("--locked", args) + def test_release_profile_is_kept_for_integration_unit_and_doc_tests(self): + for k in range(1, 4): + for args in shard.invocations(k, 3, self.NAMES, ["--locked", "--profile", "ci-release"]): + self.assertIn("--locked", args) + self.assertEqual(args[args.index("--profile") + 1], "ci-release") + + def test_failed_unit_or_integration_run_still_runs_docs_and_fails_the_shard(self): + metadata = {"workspace_members": ["a"], "packages": [ + {"id": "a", "targets": [{"name": "integration", "kind": ["test"]}]}]} + with patch.object(shard.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, stdout=json.dumps(metadata)), + subprocess.CompletedProcess([], 1), + subprocess.CompletedProcess([], 0), + ]) as run: + self.assertEqual(shard.main(["1", "3", "--locked", "--profile", "ci-release"]), 1) + self.assertEqual(run.call_count, 3) + self.assertIn("--doc", run.call_args_list[-1].args[0]) + def test_negative_bad_shard(self): with self.assertRaises(ValueError): shard.invocations(3, 2, self.NAMES) @@ -76,5 +98,78 @@ def test_the_checkout_has_integration_targets(self): self.assertGreater(len(names), 100) +class ReleaseWorkflow(unittest.TestCase): + def test_all_release_shards_are_required_and_use_the_matrix_size(self): + # Read the configured matrix, rather than assuming the workflow kept + # the same shard count as this test. A missing shard silently loses + # tests; an unguarded aggregate can turn a failed matrix green. + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + release = workflow.split("\n test-release:\n")[1].split("\n coverage:\n")[0] + count = json.loads(re.search(r"^ shard: (\[.*\])$", release, re.M)[1]) + self.assertEqual(count, list(range(1, len(count) + 1))) + self.assertGreater(len(count), 1) + self.assertIn("TEST_SHARD: ${{ matrix.shard }}", release) + self.assertIn("TEST_SHARD_COUNT: ${{ strategy.job-total }}", release) + self.assertIn('python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT" ' + '--locked --profile ci-release', release) + self.assertIn("fail-fast: false", release) + self.assertIn("github.event_name != 'merge_group'", release) + verdict = workflow.split("\n ci-ok:\n")[1] + needs = re.search(r"^ needs: \[(.*)\]$", verdict, re.M)[1].split(", ") + self.assertIn("test-release", needs) + self.assertIn("if: always()", verdict) + self.assertIn('if v["result"] not in ("success", "skipped")', verdict) + + +@unittest.skipUnless(shutil.which("cargo"), "cargo not available") +class CargoSelection(unittest.TestCase): + def test_three_release_shards_run_the_same_tests_as_cargo_workspace(self): + # Exercise Cargo, including duplicate target names across packages, + # binary/library units, ignored tests and doctests. This catches + # selector interactions that argument-list assertions cannot prove. + with tempfile.TemporaryDirectory(prefix="ci-release-shards-") as directory: + root = Path(directory) + files = { + "Cargo.toml": '[workspace]\nmembers=["one","two"]\nresolver="2"\n' + '[profile.ci-release]\ninherits="release"\nlto=false\n', + "one/Cargo.toml": '[package]\nname="one"\nversion="0.1.0"\nedition="2021"\n', + "one/src/lib.rs": '/// ```\n/// assert_eq!(one::answer(), 42);\n/// ```\n' + 'pub fn answer() -> u8 { 42 }\n' + '#[test] fn release_semantics() {\n' + ' assert!(!cfg!(debug_assertions));\n' + ' let n = std::hint::black_box(u8::MAX);\n' + ' assert_eq!(n + 1, 0);\n}\n', + "one/src/main.rs": 'fn main() {}\n#[test] fn binary_unit() {}\n', + "one/tests/shared.rs": '#[test] fn first_shared() {}\n' + '#[test] #[ignore] fn ignored_case() {}\n', + "one/tests/tail.rs": '#[test] fn tail_case() {}\n', + "two/Cargo.toml": '[package]\nname="two"\nversion="0.1.0"\nedition="2021"\n' + '[lib]\ntest=false\ndoctest=false\n', + "two/src/lib.rs": 'pub fn value() -> u8 { 1 }\n', + "two/tests/shared.rs": '#[test] fn second_shared() {}\n', + } + for name, content in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target")) + + def run(args): + result = subprocess.run(args, cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + + metadata = json.loads(run(["cargo", "metadata", "--offline", "--no-deps", "--format-version", "1"])) + run(["cargo", "generate-lockfile", "--offline"]) + extra = ["--offline", "--locked", "--profile", "ci-release"] + baseline = run(["cargo", "test", "--workspace", *extra]) + actual = "\n".join(run(args) for k in range(1, 4) + for args in shard.invocations(k, 3, shard.integration_targets(metadata), extra)) + pattern = re.compile(r"^test (.+) \.\.\. (ok|ignored)$", re.M) + expected = pattern.findall(baseline) + self.assertGreaterEqual(len(expected), 7) + self.assertCountEqual(pattern.findall(actual), expected) + + if __name__ == "__main__": unittest.main()