Block-structured AMR with batched fine advance, load-balance and active-box infrastructure

.github/scripts/check_caseopt_shared_slugs.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Assert the shared-target invariant that prebuild-case-optimization.sh relies on.
+
+That script builds syscheck, pre_process and post_process once, from benchmarks[0], behind a
+done-marker so concurrent shards skip them. That serialization only covers the benchmarks[0]
+slug. It is correct while every benchmark case hashes those three targets to that same slug:
+if two cases in *different* shards shared some other slug, both shards would build that
+staging directory at the same time, with no marker to serialize them.
+
+A target's slug covers Case.get_fpp, so it moves when the case changes any compile-time axis
+baked into it -- _prepend() bakes chemistry and eos_state_dependent (a Mie-Gruneisen, JWL or
+Vinet fluid flips the latter), and the pre_process half bakes analytic initial conditions.
+None of the current benchmarks trip any of those, which is precisely the property this check
+pins down so a future benchmark cannot quietly remove it.
+
+Usage: check_caseopt_shared_slugs.py <acc|mp|no> <case.py> [<case.py> ...]
+"""
+
+import os
+import sys
+
+SHARED_TARGETS = ("syscheck", "pre_process", "post_process")
+
+
+def main(argv):
+    if len(argv) < 3:
+        print(f"usage: {os.path.basename(argv[0])} <acc|mp|no> <case.py> [<case.py> ...]", file=sys.stderr)
+        return 2
+
+    interface, cases = argv[1], argv[2:]
+
+    sys.path.insert(0, os.path.join(os.getcwd(), "toolchain"))
+    from mfc import state
+    from mfc.state import MFCConfig
+
+    state.gCFG = MFCConfig(gpu=interface)
+    # Only the keys the loader and the case files actually read: load() forwards ARGS() to each
+    # case.py as its DICT, and the benchmarks read gpu/nodes/tasks_per_node from it.
+    state.gARG = {
+        "gpu": interface != "no",
+        "nodes": 1,
+        "tasks_per_node": 1,
+        "case_optimization": True,
+        "input": None,
+        "--": [],
+        "rdma_mpi": False,
+    }
+
+    from mfc.build import get_target
+    from mfc.run import input as mfc_input
+
+    slugs = {}
+    for case in cases:
+        state.gARG["input"] = case
+        ifile = mfc_input.load(case, [], {}, do_print=False)
+        slugs[case] = {name: get_target(name).get_slug(ifile) for name in SHARED_TARGETS}
+
+    reference = cases[0]
+    mismatches = []
+    for case in cases[1:]:
+        for name in SHARED_TARGETS:
+            if slugs[case][name] != slugs[reference][name]:
+                mismatches.append((name, case, slugs[case][name], slugs[reference][name]))
+
+    if mismatches:
+        print("ERROR: case-optimization benchmarks no longer share one slug per shared target.", file=sys.stderr)
+        print(f"       Reference case (benchmarks[0]): {reference}", file=sys.stderr)
+        for name, case, got, want in mismatches:
+            print(f"         {name}: {case}", file=sys.stderr)
+            print(f"           got  {got}", file=sys.stderr)
+            print(f"           want {want}", file=sys.stderr)
+        print(
+            "       prebuild-case-optimization.sh builds these three targets once, from the reference\n"
+            "       case, and lets other shards skip them -- which serializes only the reference slug.\n"
+            "       Give every shard its own shared-target build, or drop the offending case, before\n"
+            "       relying on the marker.",
+            file=sys.stderr,
+        )
+        return 1
+
+    print(f"Shared-target slugs agree across {len(cases)} benchmark case(s):")
+    for name in SHARED_TARGETS:
+        print(f"  {name}: {slugs[reference][name]}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

.github/scripts/prebuild-case-optimization.sh

@@ -59,6 +59,20 @@ esac
 # Concurrent shards must not build those shared staging dirs simultaneously:
 # shard 1 builds them first and drops a done marker; other shards wait for it,
 # after which their builds no-op in the shared dirs.
+#
+# That marker serializes only the benchmarks[0] slug, so the "hash identically"
+# claim above is load-bearing: two cases in different shards sharing some *other*
+# slug would build that staging dir concurrently, unserialized. The slug moves
+# with any compile-time axis in Case.get_fpp -- chemistry and eos_state_dependent
+# (a Mie-Gruneisen, JWL or Vinet fluid flips the latter) from _prepend(), analytic
+# ICs from the pre_process half. Check it up front, in every shard so they all
+# fail together, rather than leaving it to a future benchmark to discover.
+(
+    . ./mfc.sh venv > /dev/null
+    python3 .github/scripts/check_caseopt_shared_slugs.py \
+        "$([ "$job_interface" = "omp" ] && echo mp || echo acc)" "${benchmarks[@]}"
+)
+
 if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
     shared_marker_done="build/.prebuild-shared-targets-done"
     shared_marker_failed="build/.prebuild-shared-targets-failed"

.github/scripts/submit-slurm-job.sh

@@ -153,11 +153,12 @@ elif [ "$device" = "gpu" ]; then
 
     case "$cluster" in
         phoenix)
+            # 8 tasks: the largest test (AMR 3D pinned-cap multi-level, ppn=8) needs 8 MPI slots.
             # --exclude is rendered separately (see $node_exclude) so the
             # preflight can add a node to it and resubmit.
             sbatch_device_opts="\
 #SBATCH -p $gpu_partition
-#SBATCH --ntasks-per-node=4
+#SBATCH --ntasks-per-node=8
 #SBATCH -G${gpu_count}"
             node_exclude="atl1-1-03-007-29-0,atl1-1-03-007-31-0,atl1-1-01-002-28-0"
             ;;

.github/workflows/common/build.sh

@@ -54,16 +54,24 @@ if [ "$job_cluster" = "phoenix" ]; then
 fi
 
 # --- Variant selection ---
-# The suite needs two simulation binaries: the base one and a chemistry one (the
-# mechanism is compiled in). On amdflang each device link is ~1 hour, so building
-# both serially exceeds the 2 h walltime. $job_variant lets the caller split them
+# The suite needs three simulation binaries, one per always-on compile-time axis in
+# Case.get_fpp's _prepend(): the chemistry mechanism is compiled in, and so is
+# eos_state_dependent. Each distinct fpp hashes to its own build slug, for every
+# target including syscheck. On amdflang each device link is ~1 hour, so building
+# them serially exceeds the 2 h walltime. $job_variant lets the caller split them
 # into concurrent jobs that write to disjoint staging directories:
-#   base -> plain targets, what every non-chemistry test runs
+#   base -> the empty case, which bakes chemistry=False and eos_state_dependent=False
 #   chem -> the chemistry variant only
+#   eos  -> the eos_state_dependent=True variant (Mie-Gruneisen, JWL, Vinet). All such
+#           cases share one slug per target, so any one of them builds it for all.
 # Unset builds everything in this job (default for every other cluster).
+#
+# A variant missing from this list is now caught up front by the --no-build check in
+# mfc/test/test.py rather than by srun failing to exec a path that was never built.
 case "${job_variant:-}" in
     base) build_cmd=(./mfc.sh build -j 8 $build_opts) ;;
     chem) build_cmd=(./mfc.sh test -v --dry-run -a -j 8 -o Chemistry $build_opts) ;;
+    eos)  build_cmd=(./mfc.sh test -v --dry-run -a -j 8 -o eos=mie_gruneisen $build_opts) ;;
     "")   build_cmd=(./mfc.sh test -v --dry-run -a -j 8 $build_opts) ;;
     *)    echo "ERROR: unknown job_variant '$job_variant'"; exit 1 ;;
 esac

.github/workflows/test.yml

@@ -481,13 +481,14 @@ jobs:
 
       - name: Build (concurrent variants)
         if:   matrix.cluster == 'frontier_amd' && matrix.device == 'gpu'
-        # The suite needs a base and a chemistry simulation binary, and on amdflang
-        # each device LTO link is ~1 h -- building both serially overruns the 2 h
-        # walltime. They touch disjoint staging directories, so submit one SLURM job
-        # per variant and wait on both (mirrors the case-optimization pre-build).
+        # The suite needs a base, a chemistry and an eos_state_dependent simulation
+        # binary (see the variant list in common/build.sh), and on amdflang each device
+        # LTO link is ~1 h -- building them serially overruns the 2 h walltime. They
+        # touch disjoint staging directories, so submit one SLURM job per variant and
+        # wait on all (mirrors the case-optimization pre-build).
         run:  |
           pids=""
-          for v in base chem; do
+          for v in base chem eos; do
             bash .github/scripts/submit-slurm-job.sh .github/workflows/common/build.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} "${{ matrix.shard }}" "$v" &
             pids="$pids $!"
           done

.gitignore

@@ -37,6 +37,10 @@ docs/documentation/parameters.md
 /tests/*/**
 !/tests/*/golden.txt
 !/tests/*/golden-metadata.txt
+# Written by ./mfc.sh test when cases fail and deleted again on a clean run, so a committed copy
+# flips in and out of every working tree depending on the last run's outcome. The pattern above
+# only covers tests/<uuid>/, not tests/ itself.
+/tests/failed_uuids.txt
 
 # NVIDIA Nsight Compute
 *.nsys-rep

.lychee.toml

@@ -33,5 +33,6 @@ exclude = [
     "https://code\\.visualstudio\\.com/?$",           # Root page returns 403 to automated requests
     "https://stackoverflow\\.com",                    # Returns 403 to automated requests
     "https://marketplace\\.visualstudio\\.com",        # Returns 503 to automated requests
+    "_8md\\.html$",  # Doxygen auto-links backticked *.md filenames in prose to per-file pages it never generates for markdown inputs; the real md_*.html page links are still checked
     "https://web\\.eng\\.ucsd\\.edu",                    # San Diego mechanism page has an untrusted SSL cert
 ]

.typos.toml

@@ -7,6 +7,9 @@ extend-ignore-identifiers-re = [
 AttributeIDSupressMenu = "AttributeIDSupressMenu"
 
 [default.extend-words]
+# Cray CCE spells it this way in the lib-4425 runtime error; quoted verbatim in the AMR ledger so the
+# message stays greppable against what the machine actually prints.
+Unitialized = "Unitialized"
 INOUT = "INOUT"
 WRONLY = "WRONLY"
 nd = "nd"
@@ -22,6 +25,9 @@ TKE = "TKE"
 HSA = "HSA"
 infp = "infp"
 Sur = "Sur"
+thi = "thi"           # AMR clustering local: tagged-box hi index (tlo/thi)
+alo = "alo"           # AMR clustering local: accepted-box lo array (alo/ahi)
+thr = "thr"           # AMR clustering local: min-separation merge threshold
 equil = "equil"       # abbreviation for "equilibrium" (flamelet chemistry)
 chioces = "chioces"   # typo for "choices" - tests constraint key validation
 reqires = "reqires"   # typo for "requires" - tests dependency key validation

cmake/Fypp.cmake

@@ -120,6 +120,7 @@ macro(HANDLE_SOURCES target useCommon)
                                  -D MFC_COMPILER="${CMAKE_Fortran_COMPILER_ID}"
 				                 -D MFC_CASE_OPTIMIZATION=False
                                  -D chemistry=False
+                                 -D eos_state_dependent=False
                                  --line-numbering
                                  --no-folding
 								 --line-length=999

cmake/GPU.cmake

@@ -90,9 +90,15 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
     add_link_options("SHELL:-hkeepfiles")
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        # -h bounds: array-bounds and pointer checking, the Cray equivalent of gfortran's
+        # -fcheck=bounds,pointer / Intel's -check bounds / NVHPC's -Mbounds, all of which the
+        # debug branches above already set. Cray was the ONLY compiler whose debug build had no
+        # bounds checking, so an out-of-bounds write showed up here only as a later, unrelated
+        # allocation failing with an uninitialised descriptor.
         add_compile_options(
                 "SHELL:-h acc_model=auto_async_none"
                 "SHELL: -h acc_model=no_fast_addr"
+                "SHELL: -h bounds"
                 "SHELL: -K trap=fp" "SHELL: -g" "SHELL: -O0"
         )
         add_link_options("SHELL: -K trap=fp" "SHELL: -g" "SHELL: -O0")