// rudder-engine — the local speech backend for the rudder-voice plugin. // // WHY THIS EXISTS: rudder-voice is a DESKTOP-target plugin. Speech-to-text runs on the user's own // machine, never in the browser and never on a server. In dev, `bun run dev` starts this process and // Vite forwards /api/rudder/* to it; in the desktop app, Tauri runs it as a sidecar. Same binary, // same API, so "works in the Plugin Studio" means "works on desktop". // // browser / webview ──HTTP+SSE──► rudder-engine (127.0.0.1:RUDDER_PORT) // ├─ recorder: parec → pw-record → arecord (mic open ONLY while recording) // └─ WebSocket ──► nemo-speech serve (127.0.0.1:RUDDER_NEMO_PORT) // NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF // // NO TELEMETRY, NO NETWORK: the runtime is NVIDIA NeMo-Speech.cpp (ggml). ONNX-based runtimes were // REJECTED: measured 2026-09-26, the tested ONNX GenAI runtime looked up a vendor telemetry host even // with telemetry "disabled". NeMo-Speech.cpp was traced with strace: zero sockets, zero DNS. // Its only network path is an optional model download that shells out to `curl` when the model is not // a local file — so this engine REFUSES to start unless RUDDER_MODEL is an existing local file. package main import ( "bytes" "context" "crypto/rand" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log" "net/http" "net/url" "os" "os/exec" "os/signal" "path/filepath" "regexp" "runtime" "strings" "sync" "sync/atomic" "syscall" "time" "github.com/gorilla/websocket" ) // ─── Config (env, with local defaults) ────────────────────────────────────────────────────────── type config struct { Port string // this engine's HTTP port NemoPort string // the warm NeMo-Speech server's port (loopback only) NemoBin string // path to the nemo-speech executable Model string // local GGUF path — MUST exist (see the no-network note above) Language string // default language code EndpointMs int // silence that ends an utterance } func envOr(k, d string) string { if v := os.Getenv(k); v != "" { return v } return d } // defaultNemoBin: where setup puts NVIDIA's NeMo-Speech.cpp 0.1.0 CPU release for THIS platform — the // release folders are named nemo-speech-0.1.0---cpu. func defaultNemoBin(base string) string { osName := map[string]string{"darwin": "macos"}[runtime.GOOS] if osName == "" { osName = runtime.GOOS } arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH] exe := "nemo-speech" if runtime.GOOS == "windows" { exe += ".exe" } return filepath.Join(base, "bin", "nemo-speech-0.1.0-"+osName+"-"+arch+"-cpu", "bin", exe) } func loadConfig() config { home, _ := os.UserHomeDir() // RUDDER_HOME is where setup-runtime.sh installed the runtime and model (a server keeps it on a volume). base := envOr("RUDDER_HOME", filepath.Join(home, ".local", "share", "rudder")) return config{ Port: envOr("RUDDER_PORT", "5193"), NemoPort: envOr("RUDDER_NEMO_PORT", "5194"), NemoBin: envOr("RUDDER_NEMO_BIN", defaultNemoBin(base)), Model: envOr("RUDDER_MODEL", filepath.Join(base, "models", "nemotron-3.5-asr-streaming-0.6b.q8_0.gguf")), Language: envOr("RUDDER_LANG", "en-US"), EndpointMs: 800, } } // ─── Log ring: every line is also served at /api/rudder/log so the Debug panel can copy it ────── type ring struct { mu sync.Mutex lines []string } // ★ PRIVACY: the log is what people paste to an agent (🐞 Debug → copy report), so it must never carry // anything that identifies them. Every line is redacted as it is written: device MAC addresses (inside // Bluetooth device names like bluez_input.AA_BB_…) and the home folder (which contains the username). // Transcribed words are never logged at all. var ( macRe = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:_]){5}[0-9a-f]{2}\b`) homeDir = func() string { h, _ := os.UserHomeDir(); return h }() ) func redact(s string) string { s = macRe.ReplaceAllString(s, "XX:XX:XX:XX:XX:XX") if homeDir != "" && homeDir != "/" { s = strings.ReplaceAll(s, homeDir, "~") } return s } func (r *ring) add(format string, a ...any) { line := redact(time.Now().Format("15:04:05.000") + " " + fmt.Sprintf(format, a...)) log.Println(line) r.mu.Lock() r.lines = append(r.lines, line) if len(r.lines) > 3000 { r.lines = r.lines[len(r.lines)-3000:] } r.mu.Unlock() } func (r *ring) text() string { r.mu.Lock() defer r.mu.Unlock() return strings.Join(r.lines, "\n") } // ─── Event hub: Server-Sent Events to every open plugin ───────────────────────────────────────── type event map[string]any type hub struct { mu sync.Mutex clients map[chan []byte]struct{} } func (h *hub) subscribe() chan []byte { c := make(chan []byte, 64) h.mu.Lock() h.clients[c] = struct{}{} h.mu.Unlock() return c } func (h *hub) unsubscribe(c chan []byte) { h.mu.Lock() delete(h.clients, c) h.mu.Unlock() } func (h *hub) send(e event) { b, _ := json.Marshal(e) h.mu.Lock() defer h.mu.Unlock() for c := range h.clients { select { case c <- b: default: // a slow client drops an event rather than stalling the audio path } } } // ─── Engine: supervises nemo-speech serve and one recording session at a time ─────────────────── type engine struct { cfg config logs *ring hub *hub mu sync.Mutex state string // starting | ready | error stateErr string nemo *exec.Cmd recording bool recorder string language string stopSession func() device string devs *deviceWatcher nemoKey string } func (e *engine) setState(s, errText string) { e.mu.Lock() e.state, e.stateErr = s, errText e.mu.Unlock() e.hub.send(event{"type": "status", "engine": s, "error": errText}) } func (e *engine) status() event { e.mu.Lock() defer e.mu.Unlock() return event{ // A browser session belongs to one reader; everyone else sees only that the engine is in use. "engine": e.state, "error": e.stateErr, "recording": e.recording && !browserAudio(), "inUse": e.recording, "recorder": e.recorder, "language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device, "runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model), "audio": map[bool]string{true: "browser", false: "local"}[browserAudio()], } } // startNemo launches the warm speech server. The model loads and warms up once (~20 s on CPU), // so starting and stopping a recording is instant afterwards. func (e *engine) startNemo() { if st, err := os.Stat(e.cfg.Model); err != nil || st.IsDir() { e.setState("error", "model file not found: "+e.cfg.Model+" (refusing to start: a non-local model would make the runtime download it)") return } if _, err := os.Stat(e.cfg.NemoBin); err != nil { e.setState("error", "nemo-speech runtime not found: "+e.cfg.NemoBin) return } e.setState("starting", "") // A fresh random key each start: the speech server answers only this engine, not any other local // program or web page that finds its port. cmd := exec.Command(e.cfg.NemoBin, "serve", "--host", "127.0.0.1", "--port", e.cfg.NemoPort, "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey) cmd.SysProcAttr = childAttr() // dies with us out, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout if err := cmd.Start(); err != nil { e.setState("error", "could not start nemo-speech: "+err.Error()) return } e.mu.Lock() e.nemo = cmd e.mu.Unlock() e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort) go func() { buf := make([]byte, 4096) for { n, err := out.Read(buf) for _, l := range strings.Split(strings.TrimSpace(string(buf[:n])), "\n") { if l != "" { e.logs.add("[nemo] %s", l) } } if err != nil { return } } }() go func() { err := cmd.Wait() e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err)) }() // Ready = the model list answers. go func() { url := "http://127.0.0.1:" + e.cfg.NemoPort + "/v1/models" for i := 0; i < 600; i++ { time.Sleep(500 * time.Millisecond) req, _ := http.NewRequest(http.MethodGet, url, nil) req.Header.Set("Authorization", "Bearer "+e.nemoKey) resp, err := http.DefaultClient.Do(req) if err == nil { resp.Body.Close() if resp.StatusCode == 200 { e.logs.add("nemo-speech ready after %.1fs", float64(i+1)*0.5) e.setState("ready", "") return } } e.mu.Lock() s := e.state e.mu.Unlock() if s == "error" { return } } e.setState("error", "nemo-speech did not become ready within 5 minutes") }() } // recorderCmds: raw s16le 16 kHz mono PCM on stdout. parec first — on PipeWire/Bluetooth desktops it // talks to the sound server directly (the old Rudder's hardest-won lesson: PortAudio/ALSA-only // capture stalls on Bluetooth headsets). // RUDDER_RECORDER overrides the list with one space-separated command (e.g. a test that replays a // PCM file), so the engine can be tested end to end without a microphone. // device is a sound-server source name from /api/rudder/devices; empty = the system default input. func recorderCmds(device string) [][]string { if v := os.Getenv("RUDDER_RECORDER"); v != "" { return [][]string{strings.Fields(v)} } parec := []string{"parec", "--format=s16le", "--rate=16000", "--channels=1", "--raw", "--latency-msec=100"} pw := []string{"pw-record", "--format=s16", "--rate=16000", "--channels=1"} if device != "" { parec = append(parec, "--device="+device) pw = append(pw, "--target="+device) } pw = append(pw, "-") cmds := [][]string{parec, pw} if device == "" { // arecord can't address a sound-server source by name cmds = append(cmds, []string{"arecord", "-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"}) } return cmds } // browserAudio: RUDDER_AUDIO=browser — the engine runs on a server (the cloud dev server), where there is // no microphone of the reader's to record. The page captures its own mic and streams 16 kHz s16le PCM to // /api/rudder/stream; that stream feeds the same pump and speech session as a local recorder. func browserAudio() bool { return os.Getenv("RUDDER_AUDIO") == "browser" } func (e *engine) start(lang, device string) error { return e.startWith(lang, device, nil, nil) } // startWith starts a session. src, when set, is the audio source (a browser stream); nil spawns a local recorder. func (e *engine) startWith(lang, device string, src *recorderProc, sink func(event)) error { browser := src != nil // A session's events go to ITS reader (a browser session's own socket), captured here, so a session // still draining its last words after Stop never writes into the next reader's session. emit := func(ev event) { if sink != nil { sink(ev) return } e.hub.send(ev) } e.mu.Lock() if e.state != "ready" { e.mu.Unlock() return fmt.Errorf("engine is %s — wait for it to be ready", e.state) } if e.recording { e.mu.Unlock() if browser { return errors.New("busy — someone else is recording on this server; try again when they stop") } return nil } if lang == "" { lang = e.cfg.Language } e.mu.Unlock() ws, _, err := websocket.DefaultDialer.Dial("ws://127.0.0.1:"+e.cfg.NemoPort+"/v1/audio/transcriptions/realtime", http.Header{"Authorization": {"Bearer " + e.nemoKey}}) if err != nil { return fmt.Errorf("connect to speech server: %w", err) } upd, _ := json.Marshal(event{"type": "session.update", "session": event{ "sample_rate": 16000, "language": lang, "automatic_punctuation": true, "endpointing_ms": e.cfg.EndpointMs, }}) if err := ws.WriteMessage(websocket.TextMessage, upd); err != nil { ws.Close() return fmt.Errorf("session.update: %w", err) } // Bluetooth headsets start on CVSD, not mSBC. Measured 2026-09-27 on a Bluetooth headset: on mSBC the // mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after // switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an // 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out. if card := btCard(device); !browser && card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" { if p := currentProfile(card); p == "headset-head-unit-msbc" { e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)") emit(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"}) if err := switchProfile(card, e.logs); err != nil { e.logs.add("pre-switch to CVSD failed: %v", err) } else { emit(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."}) } } } first := src if first == nil { first, err = spawnRecorder(device, e.logs) if err != nil { ws.Close() return err } } recName := first.name var recMu sync.Mutex cur := first // the live recorder; replaced when a dead mic link is recovered ctx, cancel := context.WithCancel(context.Background()) var writeMu sync.Mutex done := make(chan struct{}) e.mu.Lock() e.recording, e.recorder, e.language, e.device = true, recName, lang, device e.stopSession = func() { cancel() recMu.Lock() cur.kill() recMu.Unlock() writeMu.Lock() // The streaming model holds back the last fraction of a second (its right context) until more audio // or a pause arrives, and a commit alone does not flush it (rega: a Stop mid-sentence lost "what // your country…"). 1.2 s of silence first lets it finish the words already spoken. silence := make([]byte, 3200) for i := 0; i < 12; i++ { _ = ws.WriteMessage(websocket.BinaryMessage, silence) } _ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`)) writeMu.Unlock() // Let the final words arrive. A shared server's CPU can run behind real time (measured: a loaded // 8-core box finished an 11 s clip ~15 s after it ended), so a browser session waits longer. wait := 4 * time.Second if browser { wait = 30 * time.Second } select { case <-done: case <-time.After(wait): } ws.Close() emit(event{"type": "recording", "recording": false}) } e.mu.Unlock() if browser { e.logs.add("recording started: recorder=browser language=%s", lang) } else { e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device))) go watchKernelBluetooth(ctx, e.logs) } emit(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device}) for _, d := range e.devs.current().Devices { if (d.ID == device || (device == "" && d.Default)) && d.Muted { emit(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."}) } } // Audio pump: 100 ms frames → speech server, plus a level meter and dead-mic RECOVERY. go func() { frame := make([]byte, 3200) var silentSince time.Time recoveries := 0 var recovering atomic.Bool var resetting atomic.Bool tracePeak, lastTrace := 0, time.Now() var healthySince time.Time // a mic that stays healthy 30 s earns back its 3 reconnects gaveUp := false lastLevel := time.Time{} for { recMu.Lock() r := cur recMu.Unlock() if _, err := io.ReadFull(r.out, frame); err != nil { if ctx.Err() != nil { return } recMu.Lock() swapped := cur != r recMu.Unlock() if swapped { // we killed it ourselves during a recovery: read from the new one continue } // A headset reset restarts PipeWire, which kills parec mid-read. That is expected: // wait for the reset to hand over a fresh recorder instead of ending the session. if resetting.Load() { for resetting.Load() && ctx.Err() == nil { time.Sleep(200 * time.Millisecond) } continue } if browser { e.logs.add("browser audio stream ended: %v", err) emit(event{"type": "error", "message": "The browser stopped sending microphone audio (the page was closed or lost its connection)."}) go e.stop() return } // The recorder died on its own (bad device, sound server unreachable, device unplugged). // End the session so the UI stops showing "Listening", and say WHY in plain words. _ = r.cmd.Wait() why := strings.TrimSpace(r.errBuf.String()) e.logs.add("recorder %s ended unexpectedly: %v | stderr: %s", r.name, err, why) msg := "The microphone stopped delivering audio." switch { case strings.Contains(why, "Connection refused") || strings.Contains(why, "pa_context_connect"): msg = "Can't reach the computer's sound system (PipeWire/PulseAudio). Is it running?" case strings.Contains(why, "No such entity") || strings.Contains(why, "not found"): msg = "That microphone isn't available any more — it may have been unplugged or disconnected. Pick another one." } if why != "" { msg += " (" + r.name + ": " + why + ")" } emit(event{"type": "error", "message": msg}) go e.stop() return } peak := 0 for i := 0; i+1 < len(frame); i += 2 { v := int(int16(binary.LittleEndian.Uint16(frame[i:]))) if v < 0 { v = -v } if v > peak { peak = v } } // ★ DEAD MIC LINK. Exact-zero samples are not quiet speech (a live mic always has a noise // floor). Measured 2026-09-27 on a Bluetooth headset (HFP, mSBC): four words // transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1, // the headset losing the mic side of the link. Re-opening the source restarts the audio // stream; the speech session and the transcript carry on untouched. if peak == 0 && !browser { if silentSince.IsZero() { silentSince = time.Now() } else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond { recoveries++ switch { case recoveries == 1: // Step 1: reopen the recorder (cheap; fixes a stalled stream). e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone") emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"}) if next, err := spawnRecorder(device, e.logs); err == nil { recMu.Lock() old := cur cur = next recMu.Unlock() old.kill() recovering.Store(true) } case recoveries == 2 && btCard(device) != "": // Step 2: force a NEW Bluetooth voice link by switching the headset's voice profile to // CVSD (the old Rudder's proven-stabler choice). Unlike a detect-first reset script this does not // "detect first", so it can't decide a dead link is fine. card := btCard(device) e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card) emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"}) resetting.Store(true) go func() { defer resetting.Store(false) if err := switchProfile(card, e.logs); err != nil { emit(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()}) return } if next, err := spawnRecorder(device, e.logs); err == nil { recMu.Lock() old := cur cur = next recMu.Unlock() old.kill() recovering.Store(true) } }() case recoveries == 3 && btResetPath() != "": // Step 3: full headset reset (disconnect → restart PipeWire → reconnect). e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET") emit(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"}) resetting.Store(true) go func() { defer resetting.Store(false) ok, ran := runHeadsetReset(e.logs) if !ran { return } if !ok { emit(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"}) } if next, err := spawnRecorder(device, e.logs); err == nil { recMu.Lock() old := cur cur = next recMu.Unlock() old.kill() recovering.Store(true) } }() default: gaveUp = true e.logs.add("mic still dead after reopen + headset resets — giving up for this session") msg := "Your microphone keeps going silent. This is the Bluetooth headset dropping its mic link (the kernel reports corrupted voice packets), not Rudder. Try a wired or built-in mic, or turn off the headset's noise cancellation / adaptive audio in its app." if btResetPath() == "" { msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)" } emit(event{"type": "warning", "message": msg}) } silentSince = time.Now() } } else { if recovering.Load() { recovering.Store(false) healthySince = time.Now() e.logs.add("mic recovered after %d reconnect(s)", recoveries) emit(event{"type": "recovered", "message": "Microphone is back — keep talking."}) } if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second { recoveries, gaveUp, healthySince = 0, false, time.Time{} } silentSince = time.Time{} } // Flight recorder: the mic level twice a second, so a drop can be lined up against kernel and // sound-server events in the same log (/api/rudder/log). tracePeak = max(tracePeak, peak) if time.Since(lastTrace) >= 500*time.Millisecond { e.logs.add("trace level peak=%d", tracePeak) tracePeak, lastTrace = 0, time.Now() } if time.Since(lastLevel) > 150*time.Millisecond { lastLevel = time.Now() emit(event{"type": "level", "peak": float64(peak) / 32768}) } writeMu.Lock() err := ws.WriteMessage(websocket.BinaryMessage, frame) writeMu.Unlock() if err != nil { if ctx.Err() == nil { emit(event{"type": "error", "message": "speech server connection lost: " + err.Error()}) } return } } }() // Result reader: map the server's events onto the plugin's small vocabulary. go func() { defer close(done) committed := false for { _, msg, err := ws.ReadMessage() if err != nil { return } var m map[string]any if json.Unmarshal(msg, &m) != nil { continue } t, _ := m["type"].(string) text := firstString(m, "transcript", "delta", "text") switch { case strings.HasSuffix(t, "input_audio_transcription.delta"): if committed { _ = ws.SetReadDeadline(time.Now().Add(6 * time.Second)) } // Deltas APPEND to the utterance in progress; the server sends many empty ones. if text != "" { emit(event{"type": "partial", "text": text}) } case strings.HasSuffix(t, "input_audio_transcription.completed"): emit(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)}) if committed { return // the last utterance after Stop is in } case t == "error": e.logs.add("speech server error: %s", string(msg)) emit(event{"type": "error", "message": string(msg)}) case t == "input_audio_buffer.committed": // The commit acknowledges Stop; the words for the audio before it can still follow (rega: a // Stop mid-word lost "what your country…"). Keep reading until the final text or 6 s of quiet (a loaded server can take seconds per backlog chunk). committed = true e.logs.add("speech server committed the stream; reading its last words") _ = ws.SetReadDeadline(time.Now().Add(6 * time.Second)) continue default: e.logs.add("speech event: %s", t) } } }() return nil } // recorderProc is one running recorder process. type recorderProc struct { cmd *exec.Cmd out io.ReadCloser name string errBuf *bytes.Buffer closer io.Closer // a browser source: closing it ends the stream } func (r *recorderProc) kill() { if r != nil && r.closer != nil { _ = r.closer.Close() } if r != nil && r.cmd != nil && r.cmd.Process != nil { _ = r.cmd.Process.Kill() go func() { _ = r.cmd.Wait() }() } } // spawnRecorder starts the first recorder that works for this device. func spawnRecorder(device string, logs *ring) (*recorderProc, error) { for _, c := range recorderCmds(device) { if _, err := exec.LookPath(c[0]); err != nil { continue } cmd := exec.Command(c[0], c[1:]...) cmd.Env = audioEnv() cmd.SysProcAttr = childAttr() errBuf := &bytes.Buffer{} cmd.Stderr = errBuf out, err := cmd.StdoutPipe() if err != nil { continue } if err := cmd.Start(); err != nil { logs.add("recorder %s failed to start: %v", c[0], err) continue } return &recorderProc{cmd: cmd, out: out, name: c[0], errBuf: errBuf}, nil } return nil, errors.New("no microphone recorder found (need parec, pw-record or arecord)") } func firstString(m map[string]any, keys ...string) string { for _, k := range keys { if s, ok := m[k].(string); ok { return s } } if item, ok := m["item"].(map[string]any); ok { return firstString(item, keys...) } return "" } func (e *engine) stop() { e.mu.Lock() stop := e.stopSession e.stopSession = nil was := e.recording e.recording = false e.mu.Unlock() if !was || stop == nil { return } e.logs.add("recording stopping") stop() // sends "recording false" to the session's own reader once its last words are in } func (e *engine) shutdown() { e.stop() e.mu.Lock() n := e.nemo e.mu.Unlock() if n != nil && n.Process != nil { _ = n.Process.Kill() } } // ─── HTTP API ──────────────────────────────────────────────────────────────────────────────────── func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) _ = json.NewEncoder(w).Encode(v) } // guard: two protections for a server that streams what you say. // - Host check (DNS rebinding): a web page on any domain can make the browser resolve its own hostname // to 127.0.0.1 and then read this API as "same-origin". Only requests addressed to 127.0.0.1 or // localhost are served. // - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows) // may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the // dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests). // // allowedHost: RUDDER_ALLOW_HOST (comma-separated) names the public host of a server deployment, whose own // sign-in (oauth2-proxy) sits in front of the dev server that proxies /api/rudder here. Unset = loopback only. func allowedHost(host string) bool { for _, h := range strings.Split(os.Getenv("RUDDER_ALLOW_HOST"), ",") { if h = strings.TrimSpace(h); h != "" && h == host { return true } } return false } func guard(next http.Handler) http.Handler { extra := os.Getenv("RUDDER_ALLOW_ORIGIN") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { host := r.Host if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") { host = host[:i] } if host != "127.0.0.1" && host != "localhost" && host != "[::1]" && !allowedHost(host) { http.Error(w, "forbidden host", http.StatusForbidden) return } o := r.Header.Get("Origin") if o == "tauri://localhost" || o == "http://tauri.localhost" || (extra != "" && o == extra) { w.Header().Set("Access-Control-Allow-Origin", o) w.Header().Set("Access-Control-Allow-Headers", "Content-Type") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") w.Header().Set("Vary", "Origin") } if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) return } next.ServeHTTP(w, r) }) } func main() { cfg := loadConfig() logs := &ring{} h := &hub{clients: map[chan []byte]struct{}{}} key := make([]byte, 24) if _, err := rand.Read(key); err != nil { log.Fatal("no randomness for the speech-server key: ", err) } e := &engine{cfg: cfg, logs: logs, hub: h, state: "starting", devs: &deviceWatcher{hub: h, logs: logs}, nemoKey: hex.EncodeToString(key)} e.devs.run() mux := http.NewServeMux() mux.HandleFunc("/api/rudder/status", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, e.status()) }) mux.HandleFunc("/api/rudder/start", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeJSON(w, 405, event{"error": "POST only"}) return } var body struct { Language string `json:"language"` Device string `json:"device"` } _ = json.NewDecoder(r.Body).Decode(&body) if browserAudio() { writeJSON(w, 409, event{"error": "this engine takes the browser's microphone: stream to /api/rudder/stream"}) return } if err := e.start(body.Language, body.Device); err != nil { e.logs.add("start failed: %v", err) writeJSON(w, 409, event{"error": err.Error()}) return } writeJSON(w, 200, e.status()) }) // /api/rudder/stream (RUDDER_AUDIO=browser): a WebSocket of binary 16 kHz mono s16le PCM frames from the // page's microphone. Opening it starts a session, closing it stops one; the text comes back on /events. // The upgrader's default origin check (Origin must match Host) keeps other sites out. // Origin: the page's own host (the dev server proxy), a desktop app's webview (tauri://localhost, or // http://tauri.localhost on Windows — the same origins guard() answers CORS for), or RUDDER_ALLOW_ORIGIN. // Anything else is another site and is refused. upgrader := websocket.Upgrader{ReadBufferSize: 8192, CheckOrigin: func(r *http.Request) bool { o := r.Header.Get("Origin") if o == "" || o == "tauri://localhost" || o == "http://tauri.localhost" || (os.Getenv("RUDDER_ALLOW_ORIGIN") != "" && o == os.Getenv("RUDDER_ALLOW_ORIGIN")) { return true } u, err := url.Parse(o) return err == nil && strings.EqualFold(u.Host, r.Host) }} mux.HandleFunc("/api/rudder/stream", func(w http.ResponseWriter, r *http.Request) { if !browserAudio() { writeJSON(w, 409, event{"error": "this engine records a local microphone (RUDDER_AUDIO is not browser)"}) return } conn, err := upgrader.Upgrade(w, r, nil) if err != nil { return } defer conn.Close() pr, pw := io.Pipe() src := &recorderProc{out: pr, name: "browser", errBuf: &bytes.Buffer{}, closer: pr} var wmu sync.Mutex sink := func(ev event) { b, _ := json.Marshal(ev) wmu.Lock() _ = conn.WriteMessage(websocket.TextMessage, b) wmu.Unlock() } if err := e.startWith(r.URL.Query().Get("language"), "", src, sink); err != nil { e.logs.add("stream start failed: %v", err) _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(4409, err.Error())) return } for { t, b, err := conn.ReadMessage() if err != nil { break } if t != websocket.BinaryMessage { // {"type":"stop"}: end the session but keep the socket open, so the last words (sent after the // speech server's commit) still reach this reader; the page closes the socket after "recording false". if bytes.Contains(b, []byte(`"stop"`)) { go e.stop() } continue } if _, err := pw.Write(b); err != nil { break // the session stopped (its pipe was closed) } } e.stop() }) mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) { e.stop() writeJSON(w, 200, e.status()) }) mux.HandleFunc("/api/rudder/devices", func(w http.ResponseWriter, r *http.Request) { e.devs.refresh("request") writeJSON(w, 200, e.devs.current()) }) mux.HandleFunc("/api/rudder/test", func(w http.ResponseWriter, r *http.Request) { var body struct { Device string `json:"device"` } _ = json.NewDecoder(r.Body).Decode(&body) writeJSON(w, 200, e.micTest(body.Device)) }) mux.HandleFunc("/api/rudder/log", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") _, _ = io.WriteString(w, logs.text()) }) mux.HandleFunc("/api/rudder/events", func(w http.ResponseWriter, r *http.Request) { fl, ok := w.(http.Flusher) if !ok { http.Error(w, "streaming unsupported", 500) return } w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("X-Accel-Buffering", "no") c := e.hub.subscribe() defer e.hub.unsubscribe(c) first, _ := json.Marshal(event{"type": "status", "status": e.status()}) fmt.Fprintf(w, "data: %s\n\n", first) devs, _ := json.Marshal(event{"type": "devices", "devices": e.devs.current()}) fmt.Fprintf(w, "data: %s\n\n", devs) fl.Flush() ping := time.NewTicker(15 * time.Second) defer ping.Stop() for { select { case <-r.Context().Done(): return case b := <-c: fmt.Fprintf(w, "data: %s\n\n", b) fl.Flush() case <-ping.C: fmt.Fprint(w, ": ping\n\n") fl.Flush() } } }) srv := &http.Server{Addr: "127.0.0.1:" + cfg.Port, Handler: guard(mux)} logs.add("rudder-engine listening on http://127.0.0.1:%s (runtime=%s model=%s)", cfg.Port, cfg.NemoBin, cfg.Model) e.startNemo() sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) go func() { <-sig e.shutdown() os.Exit(0) }() if err := srv.ListenAndServe(); err != nil { e.shutdown() log.Fatal(err) } }