browser-slot: a disk floor, and an infrastructure crash is NOT MEASURED, not a red gate

internal/browser-slot.sh

@@ -18,7 +18,8 @@
 # Usage:  internal/browser-slot.sh <command> [args...]
 # Env:    BROWSER_SLOTS (default 2), BROWSER_SLOT_DIR (default /tmp),
 #         BROWSER_MEM_FLOOR_MB (default 2500), BROWSER_LOCK_WAIT (default 900),
-#         BROWSER_MEM_WAIT (default 300), BROWSER_MAX_HOLD (default 900 s)
+#         BROWSER_MEM_WAIT (default 300), BROWSER_MAX_HOLD (default 900 s),
+#         BROWSER_DISK_FLOOR_MB (default 1500; checked on / and on $TMPDIR or /tmp)
 set -u
 
 SLOTS="${BROWSER_SLOTS:-2}"
@@ -33,6 +34,7 @@ MEM_WAIT_S="${BROWSER_MEM_WAIT:-300}"
 BUSY=75
 MAX_HOLD="${BROWSER_MAX_HOLD:-900}"
 KILL_AFTER="${BROWSER_KILL_AFTER:-15}"
+DISK_FLOOR_MB="${BROWSER_DISK_FLOOR_MB:-1500}"
 
 # ★ A GATE THAT IS LEGITIMATELY LONG DECLARES ITS OWN LIMIT, in its header:  @browser-slot-max-hold 2700
 # check:host-locale-contract loads every leaf twice and hit the 900 s default three times on full
@@ -68,21 +70,67 @@ fi
 # the command named so its owner can find it. A long gate that legitimately needs more says so:
 #   BROWSER_MAX_HOLD=1800 internal/browser-slot.sh node internal/<long-gate>.mjs
 run_held() {
-  timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&-
-  local rc=$?
+  # The gate's output still flows to the caller untouched (stdout and stderr kept apart); a COPY goes to
+  # RAM (/dev/shm, never the disk that may be full) so a failure can be classified afterwards.
+  local log="" rc
+  if [ -d /dev/shm ] && [ -w /dev/shm ]; then log=$(mktemp /dev/shm/browser-slot.XXXXXX 2>/dev/null) || log=""; fi
+  if [ -n "$log" ]; then
+    { timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&- 2> >(tee -a "$log" >&2); } | tee -a "$log"
+    rc=${PIPESTATUS[0]}
+    sleep 0.2   # let the stderr tee finish writing its copy
+  else
+    timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&-
+    rc=$?
+  fi
   if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
     echo "browser-slot: '$*' held a browser slot past BROWSER_MAX_HOLD=${MAX_HOLD}s and was killed (timeout rc $rc) — NOT MEASURED. Give it a timeout of its own, or set BROWSER_MAX_HOLD if it is legitimately that long." >&2
-    exit "$BUSY"
+    rm -f "$log"; exit "$BUSY"
   fi
+  # ★ AN INFRASTRUCTURE FAILURE IS NOT A PRODUCT VERDICT (measured 2026-09-25: with the disk at 48 MB free,
+  # three browser gates went RED with Chromium "Page crashed" — an out-of-disk box wearing a product
+  # gate's exit code). ENOSPC is always infrastructure. A browser crash is classified as such ONLY when
+  # the box is below a floor at that moment: a renderer crash on a healthy box can be a real product
+  # bug (a leak, a runaway loop), and blanket-excusing it would hide exactly that.
+  if [ "$rc" -ne 0 ] && [ "$rc" -ne "$BUSY" ] && [ -n "$log" ]; then
+    local infra=""
+    if grep -qiE 'ENOSPC|No space left on device' "$log"; then
+      infra="the disk is full (ENOSPC)"
+    elif grep -qE 'Page crashed|Target (page, context or browser )?has been closed|Target closed|Browser (has been )?closed|browser has disconnected' "$log"; then
+      local d m; d=$(disk_available_mb); m=$(mem_available_mb)
+      if [ "$d" -lt "$DISK_FLOOR_MB" ] || [ "$m" -lt "$FLOOR_MB" ]; then
+        infra="the browser crashed while the box was below a floor (disk ${d}MB / floor ${DISK_FLOOR_MB}MB, memory ${m}MB / floor ${FLOOR_MB}MB)"
+      fi
+    fi
+    if [ -n "$infra" ]; then
+      echo "browser-slot: '$*' failed from INFRASTRUCTURE, not the product — ${infra}. NOT MEASURED (the command exited $rc)." >&2
+      rm -f "$log"; exit "$BUSY"
+    fi
+  fi
+  rm -f "$log"
   exit "$rc"
 }
 
 slot_path() { [ "$1" = "1" ] && echo "$SLOT_BASE" || echo "$SLOT_BASE$1"; }
 
+# Free disk, the lower of / and the temp dir (they may be different filesystems). BROWSER_FAKE_DISK_FILE is
+# a TEST HOOK: when set, the reading comes from that file, so a test can "fill the disk" mid-run.
+disk_available_mb() {
+  if [ -n "${BROWSER_FAKE_DISK_FILE:-}" ] && [ -f "$BROWSER_FAKE_DISK_FILE" ]; then cat "$BROWSER_FAKE_DISK_FILE"; return; fi
+  local a b; a=$(df --output=avail -m / 2>/dev/null | tail -1 | tr -d ' '); b=$(df --output=avail -m "${TMPDIR:-/tmp}" 2>/dev/null | tail -1 | tr -d ' ')
+  a=${a:-999999}; b=${b:-999999}; [ "$a" -lt "$b" ] && echo "$a" || echo "$b"
+}
+
 mem_available_mb() {
   awk '/^MemAvailable:/ { printf "%d", $2 / 1024 }' /proc/meminfo 2>/dev/null || echo 999999
 }
 
+# ★ REFUSE ON A FULL DISK, before taking a slot: Chromium needs disk for its profile and cache, and a
+# measurement started with none ends as a "Page crashed" that reads like a product failure.
+if [ "$(disk_available_mb)" -lt "$DISK_FLOOR_MB" ]; then
+  echo "browser-slot: only $(disk_available_mb)MB free on disk (floor ${DISK_FLOOR_MB}MB) — not measuring" >&2
+  exit "$BUSY"
+fi
+
 # Hold off while the box is below the floor. Proceeding anyway is not an option here the way it is
 # for a load average: load makes a measurement slow, memory makes it get killed halfway through.
 waited=0