From 18152f0387382011165fae1064a95d1f2bdac005 Mon Sep 17 00:00:00 2001 From: Doxa Maps Date: Sun, 27 Sep 2026 22:32:22 +0000 Subject: [PATCH] Cloud container: the site serves at once while Rudder sets up, and its model is in the image Rudder's setup ran before Vite and the sign-in proxy, so the whole site answered 502 for minutes on every start while the engine built and ~742 MB downloaded; on the first start the download was reset mid-file and Rudder stayed offline. Now the runtime and pinned model are baked into an early image layer (SHA-256 checked, resumed and retried on a stall), setup runs in the background, /api/rudder answers "starting" meanwhile, and the dev server restarts once when setup ends so the engine starts. --- deployment/cloud/Dockerfile | 17 ++++++++++++ deployment/cloud/README.md | 2 +- deployment/cloud/entry.sh | 22 ++++++++++----- deployment/cloud/vite.cloud.config.mjs | 37 ++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/deployment/cloud/Dockerfile b/deployment/cloud/Dockerfile index cc44d3fcf..39ff6e22f 100644 --- a/deployment/cloud/Dockerfile +++ b/deployment/cloud/Dockerfile @@ -13,6 +13,23 @@ COPY --from=node:22-bookworm-slim /usr/local/bin/node /usr/local/bin/node # Go, to build Rudder's local engine (backend/rudder-engine, go 1.22) when RUDDER_ENGINE=1. COPY --from=golang:1.22-bookworm /usr/local/go /usr/local/go ENV PATH="/usr/local/go/bin:${PATH}" +# Rudder's speech runtime and model (~742 MB), baked into an early layer so a restart downloads nothing. +# Same pins and SHA-256 as backend/rudder-engine/setup-runtime.sh; if those move on, setup-runtime.sh fetches +# the new files at start (checksum mismatch) until these lines are bumped too. Never in git: the repo stays small. +ARG RUDDER_RT=nemo-speech-0.1.0-linux-x86_64-cpu +ARG RUDDER_RT_SHA=0f74131d631ad2c694cf0ec53490866bb6461147959589a69fb6fc231944065b +ARG RUDDER_MODEL_REV=ea30d66debe3 +ARG RUDDER_MODEL=nemotron-3.5-asr-streaming-0.6b.q8_0.gguf +ARG RUDDER_MODEL_SHA=3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1 +# get: resume + retry, and treat a stalled transfer (<100 KB/s for 30 s) as a failure; a mid-file reset +# from the model host was measured on the first cloud start. +RUN get() { for n in 1 2 3 4 5 6; do curl -fsSL -C - --speed-limit 100000 --speed-time 30 -o "$2" "$1" && return 0; sleep 5; done; return 1; } \ + && mkdir -p /opt/rudder/bin /opt/rudder/models \ + && get "https://github.com/NVIDIA/NeMo-Speech.cpp/releases/download/v0.1.0/${RUDDER_RT}.tar.gz" "/opt/rudder/bin/${RUDDER_RT}.tar.gz" \ + && echo "${RUDDER_RT_SHA} /opt/rudder/bin/${RUDDER_RT}.tar.gz" | sha256sum -c - \ + && tar -xzf "/opt/rudder/bin/${RUDDER_RT}.tar.gz" -C /opt/rudder/bin \ + && get "https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${RUDDER_MODEL_REV}/${RUDDER_MODEL}" "/opt/rudder/models/${RUDDER_MODEL}" \ + && echo "${RUDDER_MODEL_SHA} /opt/rudder/models/${RUDDER_MODEL}" | sha256sum -c - # PocketBase, pinned and checked: the backend for plugins (a dev and a prod instance, see entry.sh). ARG PB_VERSION=0.40.4 ARG PB_SHA256=9042ec818570e79c3628dadcd0a756c1496d9e1173918ec409d133c02f82e5fa diff --git a/deployment/cloud/README.md b/deployment/cloud/README.md index e5f0c1d3d..31a1ec61f 100644 --- a/deployment/cloud/README.md +++ b/deployment/cloud/README.md @@ -67,7 +67,7 @@ there. Only then does it swap the new build in. | `PB_SUPERUSER_EMAIL`, `PB_SUPERUSER_PASSWORD` | PocketBase's superuser, set on start in both instances | | `PUBLIC_API_BASE` | the public API URL written into the build's `runtime-config.js` (e.g. `https:///api-prod`) | | `BUILD_MAX_OLD_SPACE_MB` | Node heap cap for a build (default `1536`) | -| `RUDDER_ENGINE` | `1` runs Rudder's voice-to-text engine here, fed by the browser's microphone (about 1 GB more; its model, ~742 MB, downloads once to `RUDDER_HOME`, default `/data/rudder`). Give the container at least 2 vCPUs, or transcription falls behind speech | +| `RUDDER_ENGINE` | `1` runs Rudder's voice-to-text engine here, fed by the browser's microphone (about 1 GB more; its runtime and model, ~742 MB, are baked into the image at `/opt/rudder`, and `/data/rudder` wins when it holds a model). The site serves at once; Rudder shows "starting" until its engine is built, then the dev server restarts once to start it. Give the container at least 2 vCPUs, or transcription falls behind speech | | `RUDDER_ALLOW_HOST` | the public host the engine accepts (it refuses others); defaults to the host in `OIDC_REDIRECT_URL` | Mount a persistent volume at **`/data`** (PocketBase data and published builds). Memory measured on diff --git a/deployment/cloud/entry.sh b/deployment/cloud/entry.sh index 7f2e64ec3..986e86dd2 100755 --- a/deployment/cloud/entry.sh +++ b/deployment/cloud/entry.sh @@ -31,19 +31,29 @@ bun install --frozen-lockfile || bun install CONFIG=/app/repo/deployment/cloud/vite.cloud.config.mjs # Rudder (optional, RUDDER_ENGINE=1): its voice-to-text engine runs here, fed by the BROWSER's -# microphone (no server mic). The runtime and the pinned model (~742 MB) go to RUDDER_HOME, which should -# be on the persistent volume so they download once. Vite's backend plugin starts the engine and serves -# /api/rudder, which stays behind the sign-in (it is not a public route). About 1 GB resident. +# microphone (no server mic). The image carries the runtime and the pinned model (/opt/rudder); a copy on +# the persistent volume (/data/rudder) wins when it exists. Setup (checksums + building the engine) runs +# in the BACKGROUND so the site serves at once: until it finishes, /api/rudder answers "starting", then +# the cloud Vite config restarts the dev server once, which starts the engine. /api/rudder stays behind +# the sign-in (it is not a public route). About 1 GB resident. if [ "${RUDDER_ENGINE:-0}" = 1 ]; then - export RUDDER_HOME="${RUDDER_HOME:-/data/rudder}" RUDDER_AUDIO=browser + if [ -z "${RUDDER_HOME:-}" ]; then + if ls /data/rudder/models/*.gguf >/dev/null 2>&1; then RUDDER_HOME=/data/rudder; else RUDDER_HOME=/opt/rudder; fi + fi + export RUDDER_HOME RUDDER_AUDIO=browser # The engine refuses a non-loopback Host header, and Vite's proxy passes the public host through: # allow the public host, defaulting to the one in OIDC_REDIRECT_URL. if [ -z "${RUDDER_ALLOW_HOST:-}" ] && [ -n "${OIDC_REDIRECT_URL:-}" ]; then RUDDER_ALLOW_HOST=$(printf '%s' "$OIDC_REDIRECT_URL" | sed -E 's#^[a-z]+://([^/:]+).*#\1#') fi export RUDDER_ALLOW_HOST="${RUDDER_ALLOW_HOST:-}" - bash /app/repo/backend/rudder-engine/setup-runtime.sh \ - || echo "[rudder] setup failed: the widget will report the engine offline" + # The state file is read by vite.cloud.config.mjs: running -> done | failed. + echo running > /tmp/rudder-setup.state + # disown: the `wait -n` at the end must not treat this job finishing as the container's exit. + ( if bash /app/repo/backend/rudder-engine/setup-runtime.sh >&2; then echo done; else + echo "[rudder] setup failed: the widget will report the engine offline" >&2; echo failed; fi \ + > /tmp/rudder-setup.state.new && mv /tmp/rudder-setup.state.new /tmp/rudder-setup.state ) & + disown fi if [ -z "${OIDC_ISSUER:-}" ]; then diff --git a/deployment/cloud/vite.cloud.config.mjs b/deployment/cloud/vite.cloud.config.mjs index d43d9c7d2..76a501db7 100644 --- a/deployment/cloud/vite.cloud.config.mjs +++ b/deployment/cloud/vite.cloud.config.mjs @@ -2,14 +2,51 @@ // host needs. Every value comes from the environment (see deployment/cloud/README.md). import base from '/app/repo/vite.config.js' import { mergeConfig } from 'vite' +import { readFileSync } from 'node:fs' const env = process.env +const SETUP_STATE = '/tmp/rudder-setup.state' // written by entry.sh: running -> done | failed +const setupState = () => { try { return readFileSync(SETUP_STATE, 'utf8').trim() } catch { return '' } } + +// Rudder's setup runs in the background (entry.sh), so the site serves before the engine exists. While it +// runs, /api/rudder answers "starting" (the widget shows "Loading speech model…"; /events is held open so it +// does not retry in a loop). When setup ends, the dev server restarts ONCE: the repo's backend plugin only +// looks for the engine binary at start, and the restart is what starts it. 'pre' puts this ahead of that +// plugin's own "engine offline" answer. +const rudderStarting = { + name: 'cloud-rudder-starting', + enforce: 'pre', + apply: 'serve', + configureServer(server) { + if (setupState() !== 'running') return + server.middlewares.use('/api/rudder', (req, res, next) => { + if (setupState() !== 'running') return next() + const status = { engine: 'starting', error: '', audio: 'browser' } + if ((req.url || '').startsWith('/events')) { + res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' }) + res.write(`data: ${JSON.stringify({ type: 'status', status })}\n\n`) + return + } + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify(status)) + }) + const poll = setInterval(() => { + if (setupState() === 'running') return + clearInterval(poll) + server.config.logger.info(`[rudder] setup ${setupState() || 'ended'}: restarting the dev server to start the engine`) + server.restart() + }, 2000) + server.httpServer?.once('close', () => clearInterval(poll)) + }, +} + export default async (e) => { const b = typeof base === 'function' ? await base(e) : base const hmr = env.HMR_CLIENT_PORT ? { clientPort: Number(env.HMR_CLIENT_PORT), protocol: env.HMR_PROTOCOL || 'wss' } : undefined return mergeConfig(b, { + plugins: [rudderStarting], cacheDir: '/tmp/vite-cache', server: { host: env.VITE_HOST || '0.0.0.0',