Cloud container: the site serves at once while Rudder sets up, and its model is in the image

Rudder's setup ran before Vite and the sign-in proxy, so the whole site answered 502 for minutes on every
start while the engine built and ~742 MB downloaded; on the first start the download was reset mid-file and
Rudder stayed offline. Now the runtime and pinned model are baked into an early image layer (SHA-256
checked, resumed and retried on a stall), setup runs in the background, /api/rudder answers "starting"
meanwhile, and the dev server restarts once when setup ends so the engine starts.

deployment/cloud/Dockerfile

@@ -13,6 +13,23 @@ COPY --from=node:22-bookworm-slim /usr/local/bin/node /usr/local/bin/node
 # Go, to build Rudder's local engine (backend/rudder-engine, go 1.22) when RUDDER_ENGINE=1.
 COPY --from=golang:1.22-bookworm /usr/local/go /usr/local/go
 ENV PATH="/usr/local/go/bin:${PATH}"
+# Rudder's speech runtime and model (~742 MB), baked into an early layer so a restart downloads nothing.
+# Same pins and SHA-256 as backend/rudder-engine/setup-runtime.sh; if those move on, setup-runtime.sh fetches
+# the new files at start (checksum mismatch) until these lines are bumped too. Never in git: the repo stays small.
+ARG RUDDER_RT=nemo-speech-0.1.0-linux-x86_64-cpu
+ARG RUDDER_RT_SHA=0f74131d631ad2c694cf0ec53490866bb6461147959589a69fb6fc231944065b
+ARG RUDDER_MODEL_REV=ea30d66debe3
+ARG RUDDER_MODEL=nemotron-3.5-asr-streaming-0.6b.q8_0.gguf
+ARG RUDDER_MODEL_SHA=3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1
+# get: resume + retry, and treat a stalled transfer (<100 KB/s for 30 s) as a failure; a mid-file reset
+# from the model host was measured on the first cloud start.
+RUN get() { for n in 1 2 3 4 5 6; do curl -fsSL -C - --speed-limit 100000 --speed-time 30 -o "$2" "$1" && return 0; sleep 5; done; return 1; } \
+ && mkdir -p /opt/rudder/bin /opt/rudder/models \
+ && get "https://github.com/NVIDIA/NeMo-Speech.cpp/releases/download/v0.1.0/${RUDDER_RT}.tar.gz" "/opt/rudder/bin/${RUDDER_RT}.tar.gz" \
+ && echo "${RUDDER_RT_SHA}  /opt/rudder/bin/${RUDDER_RT}.tar.gz" | sha256sum -c - \
+ && tar -xzf "/opt/rudder/bin/${RUDDER_RT}.tar.gz" -C /opt/rudder/bin \
+ && get "https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${RUDDER_MODEL_REV}/${RUDDER_MODEL}" "/opt/rudder/models/${RUDDER_MODEL}" \
+ && echo "${RUDDER_MODEL_SHA}  /opt/rudder/models/${RUDDER_MODEL}" | sha256sum -c -
 # PocketBase, pinned and checked: the backend for plugins (a dev and a prod instance, see entry.sh).
 ARG PB_VERSION=0.40.4
 ARG PB_SHA256=9042ec818570e79c3628dadcd0a756c1496d9e1173918ec409d133c02f82e5fa

deployment/cloud/README.md

@@ -67,7 +67,7 @@ there. Only then does it swap the new build in.
 | `PB_SUPERUSER_EMAIL`, `PB_SUPERUSER_PASSWORD` | PocketBase's superuser, set on start in both instances |
 | `PUBLIC_API_BASE` | the public API URL written into the build's `runtime-config.js` (e.g. `https://<host>/api-prod`) |
 | `BUILD_MAX_OLD_SPACE_MB` | Node heap cap for a build (default `1536`) |
-| `RUDDER_ENGINE` | `1` runs Rudder's voice-to-text engine here, fed by the browser's microphone (about 1 GB more; its model, ~742 MB, downloads once to `RUDDER_HOME`, default `/data/rudder`). Give the container at least 2 vCPUs, or transcription falls behind speech |
+| `RUDDER_ENGINE` | `1` runs Rudder's voice-to-text engine here, fed by the browser's microphone (about 1 GB more; its runtime and model, ~742 MB, are baked into the image at `/opt/rudder`, and `/data/rudder` wins when it holds a model). The site serves at once; Rudder shows "starting" until its engine is built, then the dev server restarts once to start it. Give the container at least 2 vCPUs, or transcription falls behind speech |
 | `RUDDER_ALLOW_HOST` | the public host the engine accepts (it refuses others); defaults to the host in `OIDC_REDIRECT_URL` |
 
 Mount a persistent volume at **`/data`** (PocketBase data and published builds). Memory measured on

deployment/cloud/entry.sh

@@ -31,19 +31,29 @@ bun install --frozen-lockfile || bun install
 CONFIG=/app/repo/deployment/cloud/vite.cloud.config.mjs
 
 # Rudder (optional, RUDDER_ENGINE=1): its voice-to-text engine runs here, fed by the BROWSER's
-# microphone (no server mic). The runtime and the pinned model (~742 MB) go to RUDDER_HOME, which should
-# be on the persistent volume so they download once. Vite's backend plugin starts the engine and serves
-# /api/rudder, which stays behind the sign-in (it is not a public route). About 1 GB resident.
+# microphone (no server mic). The image carries the runtime and the pinned model (/opt/rudder); a copy on
+# the persistent volume (/data/rudder) wins when it exists. Setup (checksums + building the engine) runs
+# in the BACKGROUND so the site serves at once: until it finishes, /api/rudder answers "starting", then
+# the cloud Vite config restarts the dev server once, which starts the engine. /api/rudder stays behind
+# the sign-in (it is not a public route). About 1 GB resident.
 if [ "${RUDDER_ENGINE:-0}" = 1 ]; then
-  export RUDDER_HOME="${RUDDER_HOME:-/data/rudder}" RUDDER_AUDIO=browser
+  if [ -z "${RUDDER_HOME:-}" ]; then
+    if ls /data/rudder/models/*.gguf >/dev/null 2>&1; then RUDDER_HOME=/data/rudder; else RUDDER_HOME=/opt/rudder; fi
+  fi
+  export RUDDER_HOME RUDDER_AUDIO=browser
   # The engine refuses a non-loopback Host header, and Vite's proxy passes the public host through:
   # allow the public host, defaulting to the one in OIDC_REDIRECT_URL.
   if [ -z "${RUDDER_ALLOW_HOST:-}" ] && [ -n "${OIDC_REDIRECT_URL:-}" ]; then
     RUDDER_ALLOW_HOST=$(printf '%s' "$OIDC_REDIRECT_URL" | sed -E 's#^[a-z]+://([^/:]+).*#\1#')
   fi
   export RUDDER_ALLOW_HOST="${RUDDER_ALLOW_HOST:-}"
-  bash /app/repo/backend/rudder-engine/setup-runtime.sh \
-    || echo "[rudder] setup failed: the widget will report the engine offline"
+  # The state file is read by vite.cloud.config.mjs: running -> done | failed.
+  echo running > /tmp/rudder-setup.state
+  # disown: the `wait -n` at the end must not treat this job finishing as the container's exit.
+  ( if bash /app/repo/backend/rudder-engine/setup-runtime.sh >&2; then echo done; else
+      echo "[rudder] setup failed: the widget will report the engine offline" >&2; echo failed; fi \
+    > /tmp/rudder-setup.state.new && mv /tmp/rudder-setup.state.new /tmp/rudder-setup.state ) &
+  disown
 fi
 
 if [ -z "${OIDC_ISSUER:-}" ]; then

deployment/cloud/vite.cloud.config.mjs

@@ -2,14 +2,51 @@
 // host needs. Every value comes from the environment (see deployment/cloud/README.md).
 import base from '/app/repo/vite.config.js'
 import { mergeConfig } from 'vite'
+import { readFileSync } from 'node:fs'
 
 const env = process.env
+const SETUP_STATE = '/tmp/rudder-setup.state'   // written by entry.sh: running -> done | failed
+const setupState = () => { try { return readFileSync(SETUP_STATE, 'utf8').trim() } catch { return '' } }
+
+// Rudder's setup runs in the background (entry.sh), so the site serves before the engine exists. While it
+// runs, /api/rudder answers "starting" (the widget shows "Loading speech model…"; /events is held open so it
+// does not retry in a loop). When setup ends, the dev server restarts ONCE: the repo's backend plugin only
+// looks for the engine binary at start, and the restart is what starts it. 'pre' puts this ahead of that
+// plugin's own "engine offline" answer.
+const rudderStarting = {
+  name: 'cloud-rudder-starting',
+  enforce: 'pre',
+  apply: 'serve',
+  configureServer(server) {
+    if (setupState() !== 'running') return
+    server.middlewares.use('/api/rudder', (req, res, next) => {
+      if (setupState() !== 'running') return next()
+      const status = { engine: 'starting', error: '', audio: 'browser' }
+      if ((req.url || '').startsWith('/events')) {
+        res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' })
+        res.write(`data: ${JSON.stringify({ type: 'status', status })}\n\n`)
+        return
+      }
+      res.writeHead(200, { 'Content-Type': 'application/json' })
+      res.end(JSON.stringify(status))
+    })
+    const poll = setInterval(() => {
+      if (setupState() === 'running') return
+      clearInterval(poll)
+      server.config.logger.info(`[rudder] setup ${setupState() || 'ended'}: restarting the dev server to start the engine`)
+      server.restart()
+    }, 2000)
+    server.httpServer?.once('close', () => clearInterval(poll))
+  },
+}
+
 export default async (e) => {
   const b = typeof base === 'function' ? await base(e) : base
   const hmr = env.HMR_CLIENT_PORT
     ? { clientPort: Number(env.HMR_CLIENT_PORT), protocol: env.HMR_PROTOCOL || 'wss' }
     : undefined
   return mergeConfig(b, {
+    plugins: [rudderStarting],
     cacheDir: '/tmp/vite-cache',
     server: {
       host: env.VITE_HOST || '0.0.0.0',