From 1c3c5830a114ee73d2ff2872e256e1ba4d92e77f Mon Sep 17 00:00:00 2001 From: Doxa Maps Date: Sun, 27 Sep 2026 22:32:49 +0000 Subject: [PATCH] Update from source f3d89f8c --- backend/rudder-engine/main.go | 178 ++++++++++++++---- deployment/cloud/Dockerfile | 3 + deployment/cloud/build-cdn.sh | 17 +- internal/backend-dev-plugin.mjs | 3 +- internal/check-mjs-parses.mjs | 13 +- library/i18n/locales/ar/studio.json | 2 +- library/i18n/locales/de/studio.json | 2 +- library/i18n/locales/en/studio.json | 2 +- library/i18n/locales/es/studio.json | 2 +- library/i18n/locales/fr/studio.json | 2 +- library/i18n/locales/hi/studio.json | 2 +- library/i18n/locales/it/studio.json | 2 +- library/i18n/locales/pt/studio.json | 2 +- library/i18n/locales/ro/studio.json | 2 +- library/i18n/locales/ru/studio.json | 2 +- library/i18n/locales/zh/studio.json | 2 +- .../plugin-studio/plugin-studio.vue | 42 ++++- plugin-designs/rudder-voice/rudder-voice.vue | 153 ++++++++++++--- public/dx-maps/doxa-maps-spa/manifest.json | 8 +- public/dx-maps/index.html | 2 +- public/dx-maps/manifest.json | 8 +- public/dx-maps/plugin-studio/plugin-studio.js | 45 +++-- .../plugin-studio/plugin-studio/index.html | 2 +- .../plugin-studio/studio/index.html | 2 +- public/dx-maps/rudder-voice/rudder-voice.js | 32 +++- .../rudder-voice/rudder-voice/index.html | 2 +- 26 files changed, 409 insertions(+), 123 deletions(-) diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go index 83ed10b5..6a44defb 100644 --- a/backend/rudder-engine/main.go +++ b/backend/rudder-engine/main.go @@ -63,7 +63,8 @@ func envOr(k, d string) string { func loadConfig() config { home, _ := os.UserHomeDir() - base := filepath.Join(home, ".local", "share", "rudder") + // RUDDER_HOME is where setup-runtime.sh installed the runtime and model (a server keeps it on a volume). + base := envOr("RUDDER_HOME", filepath.Join(home, ".local", "share", "rudder")) return config{ Port: envOr("RUDDER_PORT", "5193"), NemoPort: envOr("RUDDER_NEMO_PORT", "5194"), @@ -181,9 +182,11 @@ func (e *engine) status() event { e.mu.Lock() defer e.mu.Unlock() return event{ - "engine": e.state, "error": e.stateErr, "recording": e.recording, "recorder": e.recorder, + // A browser session belongs to one reader; everyone else sees only that the engine is in use. + "engine": e.state, "error": e.stateErr, "recording": e.recording && !browserAudio(), "inUse": e.recording, "recorder": e.recorder, "language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device, "runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model), + "audio": map[bool]string{true: "browser", false: "local"}[browserAudio()], } } @@ -284,7 +287,25 @@ func recorderCmds(device string) [][]string { return cmds } -func (e *engine) start(lang, device string) error { +// browserAudio: RUDDER_AUDIO=browser — the engine runs on a server (the cloud dev server), where there is +// no microphone of the reader's to record. The page captures its own mic and streams 16 kHz s16le PCM to +// /api/rudder/stream; that stream feeds the same pump and speech session as a local recorder. +func browserAudio() bool { return os.Getenv("RUDDER_AUDIO") == "browser" } + +func (e *engine) start(lang, device string) error { return e.startWith(lang, device, nil, nil) } + +// startWith starts a session. src, when set, is the audio source (a browser stream); nil spawns a local recorder. +func (e *engine) startWith(lang, device string, src *recorderProc, sink func(event)) error { + browser := src != nil + // A session's events go to ITS reader (a browser session's own socket), captured here, so a session + // still draining its last words after Stop never writes into the next reader's session. + emit := func(ev event) { + if sink != nil { + sink(ev) + return + } + e.hub.send(ev) + } e.mu.Lock() if e.state != "ready" { e.mu.Unlock() @@ -292,6 +313,9 @@ func (e *engine) start(lang, device string) error { } if e.recording { e.mu.Unlock() + if browser { + return errors.New("busy — someone else is recording on this server; try again when they stop") + } return nil } if lang == "" { @@ -316,22 +340,25 @@ func (e *engine) start(lang, device string) error { // mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after // switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an // 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out. - if card := btCard(device); card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" { + if card := btCard(device); !browser && card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" { if p := currentProfile(card); p == "headset-head-unit-msbc" { e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)") - e.hub.send(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"}) + emit(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"}) if err := switchProfile(card, e.logs); err != nil { e.logs.add("pre-switch to CVSD failed: %v", err) } else { - e.hub.send(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."}) + emit(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."}) } } } - first, err := spawnRecorder(device, e.logs) - if err != nil { - ws.Close() - return err + first := src + if first == nil { + first, err = spawnRecorder(device, e.logs) + if err != nil { + ws.Close() + return err + } } recName := first.name var recMu sync.Mutex @@ -351,19 +378,30 @@ func (e *engine) start(lang, device string) error { writeMu.Lock() _ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`)) writeMu.Unlock() - select { // let the final words arrive + // Let the final words arrive. A shared server's CPU can run behind real time (measured: a loaded + // 8-core box finished an 11 s clip ~15 s after it ended), so a browser session waits longer. + wait := 4 * time.Second + if browser { + wait = 30 * time.Second + } + select { case <-done: - case <-time.After(4 * time.Second): + case <-time.After(wait): } ws.Close() + emit(event{"type": "recording", "recording": false}) } e.mu.Unlock() - e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device))) - go watchKernelBluetooth(ctx, e.logs) - e.hub.send(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device}) + if browser { + e.logs.add("recording started: recorder=browser language=%s", lang) + } else { + e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device))) + go watchKernelBluetooth(ctx, e.logs) + } + emit(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device}) for _, d := range e.devs.current().Devices { if (d.ID == device || (device == "" && d.Default)) && d.Muted { - e.hub.send(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."}) + emit(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."}) } } @@ -400,6 +438,12 @@ func (e *engine) start(lang, device string) error { } continue } + if browser { + e.logs.add("browser audio stream ended: %v", err) + emit(event{"type": "error", "message": "The browser stopped sending microphone audio (the page was closed or lost its connection)."}) + go e.stop() + return + } // The recorder died on its own (bad device, sound server unreachable, device unplugged). // End the session so the UI stops showing "Listening", and say WHY in plain words. _ = r.cmd.Wait() @@ -415,7 +459,7 @@ func (e *engine) start(lang, device string) error { if why != "" { msg += " (" + r.name + ": " + why + ")" } - e.hub.send(event{"type": "error", "message": msg}) + emit(event{"type": "error", "message": msg}) go e.stop() return } @@ -434,7 +478,7 @@ func (e *engine) start(lang, device string) error { // transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1, // the headset losing the mic side of the link. Re-opening the source restarts the audio // stream; the speech session and the transcript carry on untouched. - if peak == 0 { + if peak == 0 && !browser { if silentSince.IsZero() { silentSince = time.Now() } else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond { @@ -443,7 +487,7 @@ func (e *engine) start(lang, device string) error { case recoveries == 1: // Step 1: reopen the recorder (cheap; fixes a stalled stream). e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone") - e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"}) + emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"}) if next, err := spawnRecorder(device, e.logs); err == nil { recMu.Lock() old := cur @@ -458,12 +502,12 @@ func (e *engine) start(lang, device string) error { // "detect first", so it can't decide a dead link is fine. card := btCard(device) e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card) - e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"}) + emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"}) resetting.Store(true) go func() { defer resetting.Store(false) if err := switchProfile(card, e.logs); err != nil { - e.hub.send(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()}) + emit(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()}) return } if next, err := spawnRecorder(device, e.logs); err == nil { @@ -478,7 +522,7 @@ func (e *engine) start(lang, device string) error { case recoveries == 3 && btResetPath() != "": // Step 3: full headset reset (disconnect → restart PipeWire → reconnect). e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET") - e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"}) + emit(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"}) resetting.Store(true) go func() { defer resetting.Store(false) @@ -487,7 +531,7 @@ func (e *engine) start(lang, device string) error { return } if !ok { - e.hub.send(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"}) + emit(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"}) } if next, err := spawnRecorder(device, e.logs); err == nil { recMu.Lock() @@ -505,7 +549,7 @@ func (e *engine) start(lang, device string) error { if btResetPath() == "" { msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)" } - e.hub.send(event{"type": "warning", "message": msg}) + emit(event{"type": "warning", "message": msg}) } silentSince = time.Now() } @@ -514,7 +558,7 @@ func (e *engine) start(lang, device string) error { recovering.Store(false) healthySince = time.Now() e.logs.add("mic recovered after %d reconnect(s)", recoveries) - e.hub.send(event{"type": "recovered", "message": "Microphone is back — keep talking."}) + emit(event{"type": "recovered", "message": "Microphone is back — keep talking."}) } if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second { recoveries, gaveUp, healthySince = 0, false, time.Time{} @@ -530,14 +574,14 @@ func (e *engine) start(lang, device string) error { } if time.Since(lastLevel) > 150*time.Millisecond { lastLevel = time.Now() - e.hub.send(event{"type": "level", "peak": float64(peak) / 32768}) + emit(event{"type": "level", "peak": float64(peak) / 32768}) } writeMu.Lock() err := ws.WriteMessage(websocket.BinaryMessage, frame) writeMu.Unlock() if err != nil { if ctx.Err() == nil { - e.hub.send(event{"type": "error", "message": "speech server connection lost: " + err.Error()}) + emit(event{"type": "error", "message": "speech server connection lost: " + err.Error()}) } return } @@ -562,13 +606,13 @@ func (e *engine) start(lang, device string) error { case strings.HasSuffix(t, "input_audio_transcription.delta"): // Deltas APPEND to the utterance in progress; the server sends many empty ones. if text != "" { - e.hub.send(event{"type": "partial", "text": text}) + emit(event{"type": "partial", "text": text}) } case strings.HasSuffix(t, "input_audio_transcription.completed"): - e.hub.send(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)}) + emit(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)}) case t == "error": e.logs.add("speech server error: %s", string(msg)) - e.hub.send(event{"type": "error", "message": string(msg)}) + emit(event{"type": "error", "message": string(msg)}) case t == "input_audio_buffer.committed": return default: @@ -585,9 +629,13 @@ type recorderProc struct { out io.ReadCloser name string errBuf *bytes.Buffer + closer io.Closer // a browser source: closing it ends the stream } func (r *recorderProc) kill() { + if r != nil && r.closer != nil { + _ = r.closer.Close() + } if r != nil && r.cmd != nil && r.cmd.Process != nil { _ = r.cmd.Process.Kill() go func() { _ = r.cmd.Wait() }() @@ -640,9 +688,8 @@ func (e *engine) stop() { if !was || stop == nil { return } - stop() - e.logs.add("recording stopped") - e.hub.send(event{"type": "recording", "recording": false}) + e.logs.add("recording stopping") + stop() // sends "recording false" to the session's own reader once its last words are in } func (e *engine) shutdown() { @@ -670,6 +717,18 @@ func writeJSON(w http.ResponseWriter, code int, v any) { // - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows) // may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the // dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests). +// +// allowedHost: RUDDER_ALLOW_HOST (comma-separated) names the public host of a server deployment, whose own +// sign-in (oauth2-proxy) sits in front of the dev server that proxies /api/rudder here. Unset = loopback only. +func allowedHost(host string) bool { + for _, h := range strings.Split(os.Getenv("RUDDER_ALLOW_HOST"), ",") { + if h = strings.TrimSpace(h); h != "" && h == host { + return true + } + } + return false +} + func guard(next http.Handler) http.Handler { extra := os.Getenv("RUDDER_ALLOW_ORIGIN") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -677,7 +736,7 @@ func guard(next http.Handler) http.Handler { if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") { host = host[:i] } - if host != "127.0.0.1" && host != "localhost" && host != "[::1]" { + if host != "127.0.0.1" && host != "localhost" && host != "[::1]" && !allowedHost(host) { http.Error(w, "forbidden host", http.StatusForbidden) return } @@ -721,6 +780,10 @@ func main() { Device string `json:"device"` } _ = json.NewDecoder(r.Body).Decode(&body) + if browserAudio() { + writeJSON(w, 409, event{"error": "this engine takes the browser's microphone: stream to /api/rudder/stream"}) + return + } if err := e.start(body.Language, body.Device); err != nil { e.logs.add("start failed: %v", err) writeJSON(w, 409, event{"error": err.Error()}) @@ -728,6 +791,53 @@ func main() { } writeJSON(w, 200, e.status()) }) + // /api/rudder/stream (RUDDER_AUDIO=browser): a WebSocket of binary 16 kHz mono s16le PCM frames from the + // page's microphone. Opening it starts a session, closing it stops one; the text comes back on /events. + // The upgrader's default origin check (Origin must match Host) keeps other sites out. + upgrader := websocket.Upgrader{ReadBufferSize: 8192} + mux.HandleFunc("/api/rudder/stream", func(w http.ResponseWriter, r *http.Request) { + if !browserAudio() { + writeJSON(w, 409, event{"error": "this engine records a local microphone (RUDDER_AUDIO is not browser)"}) + return + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() + pr, pw := io.Pipe() + src := &recorderProc{out: pr, name: "browser", errBuf: &bytes.Buffer{}, closer: pr} + var wmu sync.Mutex + sink := func(ev event) { + b, _ := json.Marshal(ev) + wmu.Lock() + _ = conn.WriteMessage(websocket.TextMessage, b) + wmu.Unlock() + } + if err := e.startWith(r.URL.Query().Get("language"), "", src, sink); err != nil { + e.logs.add("stream start failed: %v", err) + _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(4409, err.Error())) + return + } + for { + t, b, err := conn.ReadMessage() + if err != nil { + break + } + if t != websocket.BinaryMessage { + // {"type":"stop"}: end the session but keep the socket open, so the last words (sent after the + // speech server's commit) still reach this reader; the page closes the socket after "recording false". + if bytes.Contains(b, []byte(`"stop"`)) { + go e.stop() + } + continue + } + if _, err := pw.Write(b); err != nil { + break // the session stopped (its pipe was closed) + } + } + e.stop() + }) mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) { e.stop() writeJSON(w, 200, e.status()) diff --git a/deployment/cloud/Dockerfile b/deployment/cloud/Dockerfile index 519b6744..cc44d3fc 100644 --- a/deployment/cloud/Dockerfile +++ b/deployment/cloud/Dockerfile @@ -7,6 +7,9 @@ FROM oven/bun:1-debian RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates curl unzip \ && rm -rf /var/lib/apt/lists/* COPY --from=oauth2proxy /bin/oauth2-proxy /usr/local/bin/oauth2-proxy +# Real Node for the repo's `node internal/...` scripts: bun's node shim runs `node --check` files +# instead of only parsing them, which hung the production build. +COPY --from=node:22-bookworm-slim /usr/local/bin/node /usr/local/bin/node # Go, to build Rudder's local engine (backend/rudder-engine, go 1.22) when RUDDER_ENGINE=1. COPY --from=golang:1.22-bookworm /usr/local/go /usr/local/go ENV PATH="/usr/local/go/bin:${PATH}" diff --git a/deployment/cloud/build-cdn.sh b/deployment/cloud/build-cdn.sh index 7eb3c694..7a0d9de0 100755 --- a/deployment/cloud/build-cdn.sh +++ b/deployment/cloud/build-cdn.sh @@ -1,11 +1,16 @@ #!/bin/bash # build-cdn.sh — one production build of the pulled commit into /data/cdn, one at a time. # snapshot (so the pull loop can't change files mid-build) -> apply the public exclusions -> -# BUILD_ALLOW=1 bun run build (memory-capped) -> check the output -> atomic swap of /data/cdn/current. -# Exit 75 = another build holds the lock. +# the build steps WITHOUT the gates (memory-capped) -> check the output -> the exclusion check again +# (refuses the swap) -> atomic swap of /data/cdn/current. +# No gates here: they already passed before the commit reached main, and a full gate run on a +# 2-vCPU container takes most of an hour. The public exclusions are NOT a gate and always run. +# Exit 75 = another build holds the lock. To stop a build, kill the PID in /tmp/build-cdn.pid +# (never a process group: inside a container the group can be PID 1, i.e. the whole container). set -u exec 9>/tmp/build-cdn.lock flock -n 9 || { echo "another build is running"; exit 75; } +echo $$ > /tmp/build-cdn.pid cd /app/repo SHA=$(git rev-parse --short HEAD); SNAP=/tmp/build-$SHA rm -rf "$SNAP"; mkdir -p "$SNAP" @@ -14,11 +19,13 @@ bash /app/repo/deployment/cloud/apply-exclusions.sh "$SNAP" ln -s /app/repo/node_modules "$SNAP/node_modules" cd "$SNAP" echo "$(date -u +%FT%TZ) building $SHA" -NODE_OPTIONS="--max-old-space-size=${BUILD_MAX_OLD_SPACE_MB:-1536}" BUILD_ALLOW=1 nice -n 10 bun run build \ - || { echo "build failed"; rm -rf "$SNAP"; exit 1; } +export NODE_OPTIONS="--max-old-space-size=${BUILD_MAX_OLD_SPACE_MB:-1536}" +for step in clean token build:bundles tree check:sri; do + nice -n 10 bun run "$step" || { echo "build failed at $step"; rm -rf "$SNAP"; exit 1; } +done OUT="$SNAP/public/dx-maps" [ -f "$OUT/manifest.json" ] || { echo "no manifest.json in the output"; rm -rf "$SNAP"; exit 1; } -printf 'window.MAP_API_BASE=%s;\n' "$(printf '%s' "${PUBLIC_API_BASE:-}" | python3 -c 'import json,sys;print(json.dumps(sys.stdin.read()))')" > "$OUT/runtime-config.js" +bun -e 'console.log("window.MAP_API_BASE=" + JSON.stringify(process.env.PUBLIC_API_BASE || "") + ";")' > "$OUT/runtime-config.js" NEW=/data/cdn/build-$SHA rm -rf "$NEW"; mkdir -p /data/cdn; mv "$SNAP/public" "$NEW" bash /app/repo/deployment/cloud/apply-exclusions.sh --check "$NEW" || { echo "REFUSED: excluded content in the build; not published"; rm -rf "$NEW" "$SNAP"; exit 1; } diff --git a/internal/backend-dev-plugin.mjs b/internal/backend-dev-plugin.mjs index 9a81be79..2bf49074 100644 --- a/internal/backend-dev-plugin.mjs +++ b/internal/backend-dev-plugin.mjs @@ -35,7 +35,8 @@ export const DEV_BACKENDS = [ // The proxy table for vite's `server.proxy`. SSE (`/events`) streams through unbuffered. export function devBackendProxy() { - return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false }])) + // ws: a backend may take a WebSocket (rudder's /stream carries the browser's microphone on a server deployment). + return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false, ws: true }])) } const REG = (globalThis.__dxDevBackends ||= { procs: new Map(), exitHooked: false, stopping: false }) diff --git a/internal/check-mjs-parses.mjs b/internal/check-mjs-parses.mjs index e8ed5591..be0a854c 100644 --- a/internal/check-mjs-parses.mjs +++ b/internal/check-mjs-parses.mjs @@ -57,13 +57,24 @@ if (!files.length) { process.exit(1) } +// Under bun, process.execPath is bun, and `bun --check` RUNS the script instead of parsing it (a cloud +// build hung 18+ min inside check-link-rot-sample.mjs). Parse with a real node, or refuse to guess. +let NODE = process.execPath +if (process.versions.bun) { + NODE = 'node' + try { execFileSync(NODE, ['--version'], { stdio: 'pipe' }) } catch { + console.error('MJS-PARSE: FAIL — running under bun and no `node` on PATH; this gate needs node --check (bun --check executes).') + process.exit(1) + } +} + const broken = [] for (const f of files) { const p = join(DIR, f) try { // `node --check` is the same parser the build uses, so a pass here means the build will at // least get past parsing. It does not execute the module. - execFileSync(process.execPath, ['--check', p], { stdio: 'pipe' }) + execFileSync(NODE, ['--check', p], { stdio: 'pipe' }) } catch (e) { const out = String(e.stderr || e.stdout || e.message) // node --check prints: \n\n\n\nSyntaxError: diff --git a/library/i18n/locales/ar/studio.json b/library/i18n/locales/ar/studio.json index 5de82d8d..a34f1256 100644 --- a/library/i18n/locales/ar/studio.json +++ b/library/i18n/locales/ar/studio.json @@ -22,7 +22,7 @@ "cap2": "إعداد مسبق", "cap1": "مفرد", "stageExperimental": "تجريبي", - "widget:rudder-voice/rudder-voice": "Rudder — تحويل الصوت إلى نص", + "widget:rudder-voice/rudder-voice": "Rudder · تحويل الصوت إلى نص", "toggleList": "إظهار القائمة أو إخفاؤها", "openViews": "فتح قائمة العروض", "expandList": "توسيع القائمة", diff --git a/library/i18n/locales/de/studio.json b/library/i18n/locales/de/studio.json index 84a1e30e..e9bbcfc1 100644 --- a/library/i18n/locales/de/studio.json +++ b/library/i18n/locales/de/studio.json @@ -22,7 +22,7 @@ "cap2": "Voreinstellung", "cap1": "Einzeln", "stageExperimental": "Experimentell", - "widget:rudder-voice/rudder-voice": "Rudder — Sprache zu Text", + "widget:rudder-voice/rudder-voice": "Rudder · Sprache zu Text", "toggleList": "Liste ein-/ausblenden", "openViews": "Ansichtsliste öffnen", "expandList": "Liste ausklappen", diff --git a/library/i18n/locales/en/studio.json b/library/i18n/locales/en/studio.json index ba5bfba1..f68c30ae 100644 --- a/library/i18n/locales/en/studio.json +++ b/library/i18n/locales/en/studio.json @@ -22,7 +22,7 @@ "cap2": "Preset", "cap1": "Single", "stageExperimental": "Experimental", - "widget:rudder-voice/rudder-voice": "Rudder — voice to text", + "widget:rudder-voice/rudder-voice": "Rudder · voice to text", "toggleList": "Toggle list", "openViews": "Open the list of views", "expandList": "Expand list", diff --git a/library/i18n/locales/es/studio.json b/library/i18n/locales/es/studio.json index 0c401724..dcd04820 100644 --- a/library/i18n/locales/es/studio.json +++ b/library/i18n/locales/es/studio.json @@ -22,7 +22,7 @@ "cap2": "Preajuste", "cap1": "Individual", "stageExperimental": "Experimental", - "widget:rudder-voice/rudder-voice": "Rudder — voz a texto", + "widget:rudder-voice/rudder-voice": "Rudder · voz a texto", "toggleList": "Mostrar u ocultar la lista", "openViews": "Abrir la lista de vistas", "expandList": "Expandir la lista", diff --git a/library/i18n/locales/fr/studio.json b/library/i18n/locales/fr/studio.json index 89f113fa..09f04574 100644 --- a/library/i18n/locales/fr/studio.json +++ b/library/i18n/locales/fr/studio.json @@ -22,7 +22,7 @@ "cap2": "Préréglage", "cap1": "Simple", "stageExperimental": "Expérimental", - "widget:rudder-voice/rudder-voice": "Rudder — voix vers texte", + "widget:rudder-voice/rudder-voice": "Rudder · voix vers texte", "toggleList": "Afficher ou masquer la liste", "openViews": "Ouvrir la liste des vues", "expandList": "Déplier la liste", diff --git a/library/i18n/locales/hi/studio.json b/library/i18n/locales/hi/studio.json index 6fdc6357..80d90d1a 100644 --- a/library/i18n/locales/hi/studio.json +++ b/library/i18n/locales/hi/studio.json @@ -22,7 +22,7 @@ "cap2": "प्रीसेट", "cap1": "एकल", "stageExperimental": "प्रायोगिक", - "widget:rudder-voice/rudder-voice": "Rudder — आवाज़ से टेक्स्ट", + "widget:rudder-voice/rudder-voice": "Rudder · आवाज़ से टेक्स्ट", "toggleList": "सूची दिखाएँ या छिपाएँ", "openViews": "दृश्यों की सूची खोलें", "expandList": "सूची फैलाएँ", diff --git a/library/i18n/locales/it/studio.json b/library/i18n/locales/it/studio.json index 4345b51e..c4bcb5b2 100644 --- a/library/i18n/locales/it/studio.json +++ b/library/i18n/locales/it/studio.json @@ -22,7 +22,7 @@ "cap2": "Preimpostato", "cap1": "Singolo", "stageExperimental": "Sperimentale", - "widget:rudder-voice/rudder-voice": "Rudder — voce in testo", + "widget:rudder-voice/rudder-voice": "Rudder · voce in testo", "toggleList": "Mostra o nascondi l’elenco", "openViews": "Apri l’elenco delle viste", "expandList": "Espandi l’elenco", diff --git a/library/i18n/locales/pt/studio.json b/library/i18n/locales/pt/studio.json index e9eb1246..fe732e43 100644 --- a/library/i18n/locales/pt/studio.json +++ b/library/i18n/locales/pt/studio.json @@ -22,7 +22,7 @@ "cap2": "Predefinição", "cap1": "Único", "stageExperimental": "Experimental", - "widget:rudder-voice/rudder-voice": "Rudder — voz para texto", + "widget:rudder-voice/rudder-voice": "Rudder · voz para texto", "toggleList": "Mostrar ou ocultar a lista", "openViews": "Abrir a lista de visualizações", "expandList": "Expandir a lista", diff --git a/library/i18n/locales/ro/studio.json b/library/i18n/locales/ro/studio.json index df880e8f..8b62dd91 100644 --- a/library/i18n/locales/ro/studio.json +++ b/library/i18n/locales/ro/studio.json @@ -22,7 +22,7 @@ "cap2": "Presetat", "cap1": "Simplu", "stageExperimental": "Experimental", - "widget:rudder-voice/rudder-voice": "Rudder — voce în text", + "widget:rudder-voice/rudder-voice": "Rudder · voce în text", "toggleList": "Afișează sau ascunde lista", "openViews": "Deschide lista de vizualizări", "expandList": "Extinde lista", diff --git a/library/i18n/locales/ru/studio.json b/library/i18n/locales/ru/studio.json index 7501f7a2..9dca6503 100644 --- a/library/i18n/locales/ru/studio.json +++ b/library/i18n/locales/ru/studio.json @@ -22,7 +22,7 @@ "cap2": "Предустановка", "cap1": "Одиночный", "stageExperimental": "Экспериментальные", - "widget:rudder-voice/rudder-voice": "Rudder — голос в текст", + "widget:rudder-voice/rudder-voice": "Rudder · голос в текст", "toggleList": "Показать или скрыть список", "openViews": "Открыть список представлений", "expandList": "Развернуть список", diff --git a/library/i18n/locales/zh/studio.json b/library/i18n/locales/zh/studio.json index 6a95ed0e..00094262 100644 --- a/library/i18n/locales/zh/studio.json +++ b/library/i18n/locales/zh/studio.json @@ -22,7 +22,7 @@ "cap2": "预设", "cap1": "单一", "stageExperimental": "实验性", - "widget:rudder-voice/rudder-voice": "Rudder — 语音转文字", + "widget:rudder-voice/rudder-voice": "Rudder · 语音转文字", "toggleList": "显示或隐藏列表", "openViews": "打开视图列表", "expandList": "展开列表", diff --git a/plugin-designs/plugin-studio/plugin-studio.vue b/plugin-designs/plugin-studio/plugin-studio.vue index 13e02af8..41c3fc3a 100644 --- a/plugin-designs/plugin-studio/plugin-studio.vue +++ b/plugin-designs/plugin-studio/plugin-studio.vue @@ -471,21 +471,32 @@ const WIDGET_CSS = ` [data-studio-widgets^="top"] [data-widget]{flex-direction:column-reverse} [data-widget-panel]{display:none;position:relative;width:360px;max-width:calc(100vw - 32px);height:min(480px,calc(100vh - 110px));height:min(480px,calc(100dvh - 110px - env(safe-area-inset-top) - env(safe-area-inset-bottom)));border-radius:16px;overflow:hidden;background:#0b0f14;box-shadow:0 12px 40px rgba(0,0,0,.35)} [data-widget-panel][data-open]{display:block} -[data-widget-bubble]{position:relative;width:56px;height:56px;padding:0;border:0;border-radius:50%;background:#0284c7;color:#fff;cursor:pointer;display:grid;place-items:center;box-shadow:0 4px 16px rgba(0,0,0,.3)} +[data-widget-bubble]{position:relative;display:inline-flex;align-items:center;gap:8px;height:48px;max-width:min(360px,calc(100vw - 32px));padding:0 18px 0 14px;border:1px solid #334155;border-radius:999px;background:#0f172a;color:#e5e7eb;font:600 14px/1 system-ui,-apple-system,"Segoe UI",sans-serif;cursor:pointer;box-shadow:0 4px 16px rgba(0,0,0,.3)} [data-widget-bubble]:focus-visible{outline:3px solid #7dd3fc;outline-offset:3px} -[data-widget-bubble] svg{width:26px;height:26px;display:block} -[data-widget-bubble][data-listening="true"]{background:#dc2626;animation:studio-widget-pulse 1.4s ease-out infinite} -[data-widget-bubble][data-listening="true"]::after{content:"";position:absolute;top:3px;right:3px;width:12px;height:12px;border-radius:50%;background:#fff;border:2px solid #dc2626} -@keyframes studio-widget-pulse{0%{box-shadow:0 0 0 0 rgba(220,38,38,.6)}70%{box-shadow:0 0 0 14px rgba(220,38,38,0)}100%{box-shadow:0 0 0 0 rgba(220,38,38,0)}} -@media (prefers-reduced-motion:reduce){[data-widget-bubble][data-listening="true"]{animation:none}} +[data-wb-icon]{display:grid;place-items:center;flex:none;color:#38bdf8} +[data-wb-icon] svg,[data-wb-chev] svg{width:20px;height:20px;display:block} +[data-wb-text]{min-width:0;overflow:hidden;white-space:nowrap;text-overflow:ellipsis} +[data-wb-dot],[data-wb-chev]{display:none;flex:none} +[data-wb-dot]{width:10px;height:10px;border-radius:50%;background:#ef4444} +[data-widget-bubble][aria-expanded="true"] [data-wb-chev]{display:block} +[data-widget-bubble][data-listening="true"]{border-color:#ef4444;background:#1f0f12;animation:studio-widget-pulse 1.4s ease-out infinite} +[data-widget-bubble][data-listening="true"] [data-wb-dot]{display:block} +[data-widget-bubble][data-listening="true"] [data-wb-icon]{color:#6b7280} +[data-widget-bubble][data-speaking="true"] [data-wb-icon]{color:#4ade80;animation:studio-widget-speak .5s ease-in-out infinite alternate} +@keyframes studio-widget-pulse{0%{box-shadow:0 0 0 0 rgba(239,68,68,.55)}70%{box-shadow:0 0 0 12px rgba(239,68,68,0)}100%{box-shadow:0 0 0 0 rgba(239,68,68,0)}} +@keyframes studio-widget-speak{from{transform:scale(1)}to{transform:scale(1.18)}} +@media (prefers-reduced-motion:reduce){[data-widget-bubble][data-listening="true"],[data-widget-bubble][data-speaking="true"] [data-wb-icon]{animation:none}} @media (max-width:480px){ [data-studio-widgets^="bottom"]{bottom:max(8px,env(safe-area-inset-bottom))} [data-studio-widgets$="right"]{right:max(8px,env(safe-area-inset-right))} [data-studio-widgets$="left"]{left:max(8px,env(safe-area-inset-left))} [data-widget-panel]{width:calc(100vw - 16px);max-width:none;height:60vh;height:60dvh} + [data-widget-bubble]{max-width:calc(100vw - 16px)} }` const WIDGET_ICON_OPEN = '' const WIDGET_ICON_CLOSE = '' +// The pill: [recording dot] [mic] [label, or the last words while recording] [chevron when open] +const WIDGET_PILL = '' + WIDGET_ICON_OPEN + '' + WIDGET_ICON_CLOSE + '' export default defineComponent({ name: 'PluginStudio', @@ -1131,7 +1142,8 @@ export default defineComponent({ btn.setAttribute('aria-label', wl !== wk ? wl : row.label) btn.title = btn.getAttribute('aria-label') btn.setAttribute('aria-expanded', 'false') - btn.innerHTML = WIDGET_ICON_OPEN + btn.innerHTML = WIDGET_PILL + btn.querySelector('[data-wb-text]').textContent = btn.getAttribute('aria-label') btn.addEventListener('click', () => this.openWidget(row, !panel.hasAttribute('data-open'))) box.appendChild(panel); box.appendChild(btn); host.appendChild(box) } @@ -1151,7 +1163,6 @@ export default defineComponent({ // transcript stays where it was. if (open) panel.setAttribute('data-open', ''); else panel.removeAttribute('data-open') btn.setAttribute('aria-expanded', open ? 'true' : 'false') - btn.innerHTML = open ? WIDGET_ICON_CLOSE : WIDGET_ICON_OPEN try { localStorage.setItem('plugin-studio:widget-open:' + row.key, open ? '1' : '0') } catch (_) {} const W = this.widgetStore() if (!open || W.els[row.key]) return @@ -1165,10 +1176,21 @@ export default defineComponent({ el.style.cssText = 'display:block;position:absolute;inset:0;width:100%;height:100%;overflow-y:auto;overscroll-behavior:contain;' // LIVE MIC, VISIBLE WHEN COLLAPSED: a widget that emits `recording` (Rudder does) turns its bubble // red, pulsing, with a dot while it listens, so closing the panel never hides that the mic is on. + // While recording the pill drops its label for the last words heard, and its mic lights only while + // speech is detected (`activity`), so a collapsed Rudder still shows it is hearing the reader. + const pill = box.querySelector('[data-widget-bubble]') + const text = pill.querySelector('[data-wb-text]') el.addEventListener('recording', (e) => { const on = !!(Array.isArray(e.detail) ? e.detail[0] : e.detail) - const b = box.querySelector('[data-widget-bubble]') - if (b) b.setAttribute('data-listening', on ? 'true' : 'false') + pill.setAttribute('data-listening', on ? 'true' : 'false') + if (!on) { pill.setAttribute('data-speaking', 'false'); text.textContent = pill.getAttribute('aria-label') } + else text.textContent = '' + }) + el.addEventListener('activity', (e) => { + const a = (Array.isArray(e.detail) ? e.detail[0] : e.detail) || {} + if (pill.getAttribute('data-listening') !== 'true') return + pill.setAttribute('data-speaking', a.speaking ? 'true' : 'false') + text.textContent = a.words || '' }) W.els[row.key] = el panel.appendChild(el) diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue index 7f2bfb7b..41f1538f 100644 --- a/plugin-designs/rudder-voice/rudder-voice.vue +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -22,7 +22,7 @@ const props = defineProps({ profileConfig: { type: String, default: '' }, varian const compact = computed(() => props.variant === 'widget') // `recording` (true/false) tells a host that Rudder started or stopped listening — the Plugin Studio's // corner button turns red with it, so a collapsed corner still shows the mic is live. -const emit = defineEmits(['recording']) +const emit = defineEmits(['recording', 'activity']) const cfg = computed(() => { try { return props.profileConfig ? JSON.parse(props.profileConfig) : {} } catch { return {} } @@ -286,7 +286,9 @@ function applyStatus(s) { engineError.value = s.error || '' if (s.runtime) runtime.value = s.runtime if (s.model) model.value = s.model - if (typeof s.recording === 'boolean') recording.value = s.recording + if (s.audio) audioMode.value = s.audio + // A browser session's state is this page's own; the server's shared status never reports it. + if (typeof s.recording === 'boolean' && audioMode.value !== 'browser') recording.value = s.recording } function connect() { @@ -300,9 +302,20 @@ function connect() { } es = new window.EventSource(apiBase.value + '/events') es.onopen = () => logc('events connected') - es.onmessage = (m) => { + es.onmessage = (m) => onEvent(m.data) + es.onerror = () => { + engine.value = 'offline' + if (audioMode.value !== 'browser') recording.value = false + logc('events disconnected; retrying in 3s') + try { es.close() } catch {} + retry = setTimeout(connect, 3000) + } +} + +// One vocabulary for both channels: the shared /events stream, and a browser session's own socket. +function onEvent(data) { let e - try { e = JSON.parse(m.data) } catch { return } + try { e = JSON.parse(data) } catch { return } switch (e.type) { case 'status': applyStatus(e.status || e) @@ -311,7 +324,7 @@ function connect() { break case 'recording': recording.value = !!e.recording - if (!e.recording) { level.value = 0; flushPartial() } + if (!e.recording) { level.value = 0; flushPartial(); closeStream() } break case 'partial': partial.value += e.text @@ -342,14 +355,82 @@ function connect() { logc('engine error: ' + e.message) break } +} + +// ─── BROWSER MICROPHONE (engine on a server, status.audio === 'browser') ───────────────────────────── +// The engine cannot hear the reader's microphone from a server, so this page captures it, turns it into +// 16 kHz mono 16-bit PCM in an AudioWorklet, and streams 100 ms frames to /api/rudder/stream. The words +// come back on that same socket, to this reader only. +const WORKLET = `class RudderPcm16k extends AudioWorkletProcessor { + constructor () { super(); this.step = sampleRate / 16000; this.pos = 0; this.out = new Int16Array(1600); this.n = 0 } + process (inputs) { + const ch = inputs[0] && inputs[0][0] + if (!ch) return true + for (; this.pos < ch.length; this.pos += this.step) { + const v = Math.max(-1, Math.min(1, ch[Math.floor(this.pos)])) + this.out[this.n++] = v < 0 ? v * 32768 : v * 32767 + if (this.n === this.out.length) { this.port.postMessage(this.out.buffer, [this.out.buffer]); this.out = new Int16Array(1600); this.n = 0 } + } + this.pos -= ch.length + return true } - es.onerror = () => { - engine.value = 'offline' - recording.value = false - logc('events disconnected; retrying in 3s') - try { es.close() } catch {} - retry = setTimeout(connect, 3000) +} +registerProcessor('rudder-pcm16k', RudderPcm16k)` +const audioMode = ref('local') +let sock = null, media = null, actx = null + +async function startBrowser() { + if (sock) { closeStream(); flushPartial() } // a previous session still draining its last words + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + throw new Error('This browser cannot share its microphone here (it needs a secure https:// page).') } + media = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }) + actx = new window.AudioContext() + const url = URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' })) + await actx.audioWorklet.addModule(url) + URL.revokeObjectURL(url) + const node = new window.AudioWorkletNode(actx, 'rudder-pcm16k') + const mute = actx.createGain() + mute.gain.value = 0 // keeps the graph pulled without playing the mic back + actx.createMediaStreamSource(media).connect(node) + node.connect(mute).connect(actx.destination) + const u = new URL(apiBase.value + '/stream', location.href) + u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:' + u.searchParams.set('language', language.value) + const ws = new window.WebSocket(u) + sock = ws + ws.binaryType = 'arraybuffer' + ws.onmessage = (m) => { if (sock === ws) onEvent(m.data) } + ws.onclose = (c) => { + if (sock !== ws) return // an older, draining session: the new one owns the mic and the box + if (c.code === 4409) say(c.reason || 'Someone else is recording on this server.', 'error') + sock = null + releaseMic() + if (recording.value) { recording.value = false; level.value = 0 } + flushPartial() + } + node.port.onmessage = (e) => { if (sock && sock.readyState === 1) sock.send(e.data) } +} +function releaseMic() { + try { media && media.getTracks().forEach((t) => t.stop()) } catch {} + try { actx && actx.close() } catch {} + media = null + actx = null +} +// Stop: the mic goes off and the widget is idle at once (rega: it read "Stop" for 12 s+ while a loaded +// server caught up). The socket stays open so the last words still land in the box; the engine closes the +// session with "recording false". Record again meanwhile simply lets go of that tail. +function stopBrowser() { + releaseMic() + recording.value = false + level.value = 0 + if (sock && sock.readyState === 1) sock.send(JSON.stringify({ type: 'stop' })) + else { closeStream(); flushPartial() } +} +function closeStream() { + const ws = sock + sock = null + if (ws) { try { ws.close() } catch {} } } // Recording stopped mid-sentence: the streamed words are already in the box, so just let go of them. @@ -379,7 +460,10 @@ async function toggle() { if (busy.value) return busy.value = true try { - if (recording.value) { + if (audioMode.value === 'browser') { + if (recording.value) stopBrowser() + else { say(''); await startBrowser() } + } else if (recording.value) { await post('/stop') } else { say('') @@ -387,7 +471,8 @@ async function toggle() { applyStatus(await post('/start', { language: language.value, device: device.value })) } } catch (err) { - say(String(err.message || err), 'error') + if (audioMode.value === 'browser') { releaseMic(); closeStream() } + say(err && err.name === 'NotAllowedError' ? 'Microphone permission was refused — allow it for this site to record.' : String(err.message || err), 'error') logc('toggle failed: ' + err) } finally { busy.value = false @@ -463,11 +548,30 @@ onMounted(() => { onBeforeUnmount(() => { clearTimeout(retry) try { es?.close() } catch {} + releaseMic() + closeStream() window.removeEventListener('keydown', onKey) }) watch(recording, (v) => emit('recording', v)) +// `activity` feeds a host's collapsed pill while recording: whether speech is heard right now (the level, +// held briefly so the icon doesn't flicker between words) and the last few words, so a reader with the +// panel closed can see they are being heard. +let speakingUntil = 0, activityTimer = null, lastActivity = '' +function sendActivity() { + activityTimer = null + const speaking = recording.value && Date.now() < speakingUntil + const tail = recording.value ? draft.value.trim().split(/\s+/).slice(-5).join(' ') : '' + const key = speaking + '|' + tail + if (key === lastActivity) return + lastActivity = key + emit('activity', { speaking, words: tail }) +} +function queueActivity() { if (!activityTimer) activityTimer = setTimeout(sendActivity, 150) } +watch(level, (v) => { if (v > 0.12) speakingUntil = Date.now() + 400; queueActivity(); setTimeout(queueActivity, 450) }) +watch([draft, recording], queueActivity) + const statusText = computed(() => ({ connecting: 'Connecting…', starting: 'Loading speech model… (about 20 s the first time)', @@ -479,7 +583,7 @@ const statusText = computed(() => ({