diff --git a/internal/browser-slot.sh b/internal/browser-slot.sh index 3f9738ec3..7f47e52f8 100755 --- a/internal/browser-slot.sh +++ b/internal/browser-slot.sh @@ -18,7 +18,8 @@ # Usage: internal/browser-slot.sh [args...] # Env: BROWSER_SLOTS (default 2), BROWSER_SLOT_DIR (default /tmp), # BROWSER_MEM_FLOOR_MB (default 2500), BROWSER_LOCK_WAIT (default 900), -# BROWSER_MEM_WAIT (default 300), BROWSER_MAX_HOLD (default 900 s) +# BROWSER_MEM_WAIT (default 300), BROWSER_MAX_HOLD (default 900 s), +# BROWSER_DISK_FLOOR_MB (default 1500; checked on / and on $TMPDIR or /tmp) set -u SLOTS="${BROWSER_SLOTS:-2}" @@ -33,6 +34,7 @@ MEM_WAIT_S="${BROWSER_MEM_WAIT:-300}" BUSY=75 MAX_HOLD="${BROWSER_MAX_HOLD:-900}" KILL_AFTER="${BROWSER_KILL_AFTER:-15}" +DISK_FLOOR_MB="${BROWSER_DISK_FLOOR_MB:-1500}" # ★ A GATE THAT IS LEGITIMATELY LONG DECLARES ITS OWN LIMIT, in its header: @browser-slot-max-hold 2700 # check:host-locale-contract loads every leaf twice and hit the 900 s default three times on full @@ -68,21 +70,67 @@ fi # the command named so its owner can find it. A long gate that legitimately needs more says so: # BROWSER_MAX_HOLD=1800 internal/browser-slot.sh node internal/.mjs run_held() { - timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&- - local rc=$? + # The gate's output still flows to the caller untouched (stdout and stderr kept apart); a COPY goes to + # RAM (/dev/shm, never the disk that may be full) so a failure can be classified afterwards. + local log="" rc + if [ -d /dev/shm ] && [ -w /dev/shm ]; then log=$(mktemp /dev/shm/browser-slot.XXXXXX 2>/dev/null) || log=""; fi + if [ -n "$log" ]; then + { timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&- 2> >(tee -a "$log" >&2); } | tee -a "$log" + rc=${PIPESTATUS[0]} + sleep 0.2 # let the stderr tee finish writing its copy + else + timeout --kill-after="$KILL_AFTER" "$MAX_HOLD" "$@" 9>&- + rc=$? + fi if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then echo "browser-slot: '$*' held a browser slot past BROWSER_MAX_HOLD=${MAX_HOLD}s and was killed (timeout rc $rc) — NOT MEASURED. Give it a timeout of its own, or set BROWSER_MAX_HOLD if it is legitimately that long." >&2 - exit "$BUSY" + rm -f "$log"; exit "$BUSY" fi + # ★ AN INFRASTRUCTURE FAILURE IS NOT A PRODUCT VERDICT (measured 2026-09-25: with the disk at 48 MB free, + # three browser gates went RED with Chromium "Page crashed" — an out-of-disk box wearing a product + # gate's exit code). ENOSPC is always infrastructure. A browser crash is classified as such ONLY when + # the box is below a floor at that moment: a renderer crash on a healthy box can be a real product + # bug (a leak, a runaway loop), and blanket-excusing it would hide exactly that. + if [ "$rc" -ne 0 ] && [ "$rc" -ne "$BUSY" ] && [ -n "$log" ]; then + local infra="" + if grep -qiE 'ENOSPC|No space left on device' "$log"; then + infra="the disk is full (ENOSPC)" + elif grep -qE 'Page crashed|Target (page, context or browser )?has been closed|Target closed|Browser (has been )?closed|browser has disconnected' "$log"; then + local d m; d=$(disk_available_mb); m=$(mem_available_mb) + if [ "$d" -lt "$DISK_FLOOR_MB" ] || [ "$m" -lt "$FLOOR_MB" ]; then + infra="the browser crashed while the box was below a floor (disk ${d}MB / floor ${DISK_FLOOR_MB}MB, memory ${m}MB / floor ${FLOOR_MB}MB)" + fi + fi + if [ -n "$infra" ]; then + echo "browser-slot: '$*' failed from INFRASTRUCTURE, not the product — ${infra}. NOT MEASURED (the command exited $rc)." >&2 + rm -f "$log"; exit "$BUSY" + fi + fi + rm -f "$log" exit "$rc" } slot_path() { [ "$1" = "1" ] && echo "$SLOT_BASE" || echo "$SLOT_BASE$1"; } +# Free disk, the lower of / and the temp dir (they may be different filesystems). BROWSER_FAKE_DISK_FILE is +# a TEST HOOK: when set, the reading comes from that file, so a test can "fill the disk" mid-run. +disk_available_mb() { + if [ -n "${BROWSER_FAKE_DISK_FILE:-}" ] && [ -f "$BROWSER_FAKE_DISK_FILE" ]; then cat "$BROWSER_FAKE_DISK_FILE"; return; fi + local a b; a=$(df --output=avail -m / 2>/dev/null | tail -1 | tr -d ' '); b=$(df --output=avail -m "${TMPDIR:-/tmp}" 2>/dev/null | tail -1 | tr -d ' ') + a=${a:-999999}; b=${b:-999999}; [ "$a" -lt "$b" ] && echo "$a" || echo "$b" +} + mem_available_mb() { awk '/^MemAvailable:/ { printf "%d", $2 / 1024 }' /proc/meminfo 2>/dev/null || echo 999999 } +# ★ REFUSE ON A FULL DISK, before taking a slot: Chromium needs disk for its profile and cache, and a +# measurement started with none ends as a "Page crashed" that reads like a product failure. +if [ "$(disk_available_mb)" -lt "$DISK_FLOOR_MB" ]; then + echo "browser-slot: only $(disk_available_mb)MB free on disk (floor ${DISK_FLOOR_MB}MB) — not measuring" >&2 + exit "$BUSY" +fi + # Hold off while the box is below the floor. Proceeding anyway is not an option here the way it is # for a load average: load makes a measurement slow, memory makes it get killed halfway through. waited=0