From 4d6aeb2949277f2de6e8092532b718823ac05569 Mon Sep 17 00:00:00 2001 From: Doxa Maps Date: Sun, 27 Sep 2026 01:48:38 -0500 Subject: [PATCH] Add Rudder (rudder-voice): offline streaming voice-to-text, a desktop-target plugin Rudder is the first plugin that isn't a map: it shows that one plugin-designs/ folder, developed live in the Plugin Studio, can ship as a desktop app with its own local backend. - plugin-designs/rudder-voice: element. Words stream into the end of a text area while you type/paste anywhere (typing holds the stream; nothing duplicated or lost). Mic picker with hot-plug, 3-second mic test, plain-language errors, info panel, privacy-safe debug report. meta.json declares "targets": ["desktop"]. - backend/rudder-engine (Go): supervises NVIDIA NeMo-Speech.cpp (ggml, CPU) with the Nemotron 3.5 ASR Streaming GGUF over a loopback WebSocket; records with parec/pw-record only while recording; recovers dead Bluetooth mic links (reopen -> CVSD voice profile -> optional RUDDER_BT_RESET) and starts headsets on CVSD. No telemetry: traced with strace, zero sockets/DNS while transcribing. - Privacy: log lines redact MAC addresses and the home folder, transcripts are never logged, Host check against DNS rebinding, CORS only for the desktop webview, random API key for the speech server. - internal/backend-dev-plugin.mjs + vite.config.js: `bun run dev` starts built plugin backends and proxies /api/ same-origin, so a plugin calls the same API in the studio and on desktop. - backend/rudder-engine/setup-runtime.sh: fetches the runtime and model pinned by SHA-256. Not yet: the Tauri desktop build (sidecar) and the PocketBase word-correction table. --- .gitignore | 3 + backend/rudder-engine/devices.go | 456 ++++++++++ backend/rudder-engine/go.mod | 5 + backend/rudder-engine/go.sum | 2 + backend/rudder-engine/main.go | 797 ++++++++++++++++++ backend/rudder-engine/setup-runtime.sh | 59 ++ internal/backend-dev-plugin.mjs | 61 ++ plugin-designs/rudder-voice/README.md | 55 ++ plugin-designs/rudder-voice/index.html | 32 + plugin-designs/rudder-voice/index.js | 23 + .../rudder-voice/rudder-voice.meta.json | 7 + plugin-designs/rudder-voice/rudder-voice.vue | 650 ++++++++++++++ vite.config.js | 6 +- 13 files changed, 2155 insertions(+), 1 deletion(-) create mode 100644 backend/rudder-engine/devices.go create mode 100644 backend/rudder-engine/go.mod create mode 100644 backend/rudder-engine/go.sum create mode 100644 backend/rudder-engine/main.go create mode 100755 backend/rudder-engine/setup-runtime.sh create mode 100644 internal/backend-dev-plugin.mjs create mode 100644 plugin-designs/rudder-voice/README.md create mode 100644 plugin-designs/rudder-voice/index.html create mode 100644 plugin-designs/rudder-voice/index.js create mode 100644 plugin-designs/rudder-voice/rudder-voice.meta.json create mode 100644 plugin-designs/rudder-voice/rudder-voice.vue diff --git a/.gitignore b/.gitignore index 21dac278..3e0c1fc7 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,6 @@ data-private/ /public/assets/ /public/__gallery/ /public/index.html + +# Local plugin backends (backend//) — the compiled binaries are rebuilt from source, never committed. +/backend/bin/ diff --git a/backend/rudder-engine/devices.go b/backend/rudder-engine/devices.go new file mode 100644 index 00000000..be2be71c --- /dev/null +++ b/backend/rudder-engine/devices.go @@ -0,0 +1,456 @@ +// devices.go — microphone discovery, hot-plug, and the sound-server environment. +// +// The Jitsi Meet pattern (lib-jitsi-meet modules/RTC/RTCUtils.js), done natively because Rudder records +// outside the browser: +// +// Jitsi (browser) Rudder (native, PipeWire/PulseAudio) +// navigator.mediaDevices.enumerateDevices() → pactl -f json list sources +// 'devicechange' event + polling fallback → pactl subscribe (+ re-list on every source/card event) +// compareAvailableMediaDevices(new) → sameDevices(old, new) — only emit on a real change +// getUserMedia({audio:{deviceId:{exact:id}}}) → parec --device= +// +// Jitsi also reports the device list to analytics (sendDeviceListToAnalytics). Rudder never does: +// the list goes to the local plugin over SSE and nowhere else. +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "math" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "time" +) + +// audioEnv gives child processes a path to the user's sound server even when the engine was launched +// from a context without a desktop session (a detached shell, a service, an agent's terminal). Measured +// 2026-09-27: started with no XDG_RUNTIME_DIR, parec exited instantly with "Connection refused" and every +// recording captured nothing, although the socket sat at /run/user//pulse/native all along. +func audioEnv() []string { + env := os.Environ() + rt := os.Getenv("XDG_RUNTIME_DIR") + if rt == "" { + rt = fmt.Sprintf("/run/user/%d", os.Getuid()) + if st, err := os.Stat(rt); err == nil && st.IsDir() { + env = append(env, "XDG_RUNTIME_DIR="+rt) + } + } + // systemctl --user (a headset-reset command may restart PipeWire) needs the session bus. + if os.Getenv("DBUS_SESSION_BUS_ADDRESS") == "" { + if _, err := os.Stat(rt + "/bus"); err == nil { + env = append(env, "DBUS_SESSION_BUS_ADDRESS=unix:path="+rt+"/bus") + } + } + if os.Getenv("PULSE_SERVER") == "" { + if _, err := os.Stat(rt + "/pulse/native"); err == nil { + env = append(env, "PULSE_SERVER=unix:"+rt+"/pulse/native") + } + } + return env +} + +// Device is one microphone as the plugin shows it. +type Device struct { + ID string `json:"id"` // the sound server's source name: what parec --device takes + Label string `json:"label"` // human name ("My Headset", "Built-in Audio Analog Stereo") + Bus string `json:"bus"` // bluetooth | usb | pci | ... + Default bool `json:"default"` // the system's default input + Muted bool `json:"muted"` + Volume string `json:"volume"` // e.g. "52%" + State string `json:"state"` // RUNNING | IDLE | SUSPENDED +} + +// DeviceList is the whole picture: the mics, plus sound cards that have a mic but are switched off +// (the laptop's built-in card, profile "off", is the common case — its mic simply doesn't exist until +// the card is turned on in Sound settings). +type DeviceList struct { + Devices []Device `json:"devices"` + Default string `json:"default"` + OffCards []string `json:"offCards"` + Error string `json:"error,omitempty"` +} + +func runPactl(args ...string) ([]byte, error) { + cmd := exec.Command("pactl", args...) + cmd.Env = audioEnv() + return cmd.Output() +} + +func listDevices() DeviceList { + out := DeviceList{Devices: []Device{}, OffCards: []string{}} + if _, err := exec.LookPath("pactl"); err != nil { + out.Error = "pactl not found: install pipewire-pulse or pulseaudio-utils to list microphones" + return out + } + if b, err := runPactl("get-default-source"); err == nil { + out.Default = strings.TrimSpace(string(b)) + } + b, err := runPactl("-f", "json", "list", "sources") + if err != nil { + out.Error = "cannot reach the sound server (pactl list sources failed: " + err.Error() + ")" + return out + } + var raw []struct { + Name string `json:"name"` + Description string `json:"description"` + State string `json:"state"` + Mute bool `json:"mute"` + Volume map[string]any `json:"volume"` + Properties map[string]any `json:"properties"` + } + if err := json.Unmarshal(b, &raw); err != nil { + out.Error = "unexpected pactl output: " + err.Error() + return out + } + for _, s := range raw { + // A ".monitor" source is the loopback of a SPEAKER, not a microphone: listing it would let the + // user "record" their own music. Hidden, as Jitsi lists only audioinput kinds. + if strings.HasSuffix(s.Name, ".monitor") || s.Properties["device.class"] == "monitor" { + continue + } + d := Device{ID: s.Name, Label: s.Description, State: s.State, Muted: s.Mute, Default: s.Name == out.Default} + if bus, ok := s.Properties["device.bus"].(string); ok { + d.Bus = bus + } + for _, ch := range s.Volume { // first channel is representative + if m, ok := ch.(map[string]any); ok { + if p, ok := m["value_percent"].(string); ok { + d.Volume = p + break + } + } + } + out.Devices = append(out.Devices, d) + } + // Cards whose active profile is "off" but which offer an input profile. + if b, err := runPactl("-f", "json", "list", "cards"); err == nil { + var cards []struct { + Name string `json:"name"` + ActiveProfile string `json:"active_profile"` + Properties map[string]any `json:"properties"` + Profiles map[string]struct { + Sources int `json:"sources"` + Available bool `json:"available"` + } `json:"profiles"` + } + if json.Unmarshal(b, &cards) == nil { + for _, c := range cards { + if c.ActiveProfile != "off" { + continue + } + for _, p := range c.Profiles { + if p.Sources > 0 && p.Available { + label, _ := c.Properties["device.description"].(string) + if label == "" { + label = c.Name + } + out.OffCards = append(out.OffCards, label) + break + } + } + } + } + } + return out +} + +// sameDevices mirrors Jitsi's compareAvailableMediaDevices: emit only when something a user would +// notice changed (a mic came, went, was muted, or the default moved), not on every state flicker. +func sameDevices(a, b DeviceList) bool { + if a.Default != b.Default || len(a.Devices) != len(b.Devices) || a.Error != b.Error || len(a.OffCards) != len(b.OffCards) { + return false + } + for i := range a.Devices { + x, y := a.Devices[i], b.Devices[i] + if x.ID != y.ID || x.Label != y.Label || x.Muted != y.Muted || x.Volume != y.Volume { + return false + } + } + return true +} + +// deviceWatcher keeps the current list and pushes a "devices" event on every real change. It listens +// to `pactl subscribe` (the native 'devicechange') and, like Jitsi, also polls as a fallback in case +// the subscription dies. +type deviceWatcher struct { + mu sync.Mutex + last DeviceList + hub *hub + logs *ring +} + +func (w *deviceWatcher) current() DeviceList { + w.mu.Lock() + defer w.mu.Unlock() + return w.last +} + +func (w *deviceWatcher) refresh(reason string) { + now := listDevices() + w.mu.Lock() + changed := !sameDevices(w.last, now) + w.last = now + w.mu.Unlock() + if changed { + w.logs.add("microphones (%s): %d found, default=%s %s", reason, len(now.Devices), now.Default, now.Error) + w.hub.send(event{"type": "devices", "devices": now}) + } +} + +func (w *deviceWatcher) run() { + w.refresh("startup") + // Poll fallback (Jitsi polls enumerateDevices when devicechange is unavailable). + go func() { + for range time.Tick(5 * time.Second) { + w.refresh("poll") + } + }() + go func() { + for { + cmd := exec.Command("pactl", "subscribe") + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + out, err := cmd.StdoutPipe() + if err == nil && cmd.Start() == nil { + sc := bufio.NewScanner(out) + var debounce *time.Timer + for sc.Scan() { + l := sc.Text() + if strings.Contains(l, " card ") || (strings.Contains(l, " source ") && !strings.Contains(l, "source-output")) { + w.logs.add("sound-server event: %s", l) + } + if strings.Contains(l, " source ") || strings.Contains(l, " card ") || strings.Contains(l, " server ") { + if debounce != nil { + debounce.Stop() + } + debounce = time.AfterFunc(300*time.Millisecond, func() { w.refresh("hot-plug") }) + } + } + _ = cmd.Wait() + } + time.Sleep(3 * time.Second) // sound server restarted or unreachable: retry + } + }() +} + +// micTest records ~3 s from one device WITHOUT transcribing and says, in plain words, whether the mic +// works — Rudder's version of Jitsi's pre-join audio preview. Level events stream to the plugin while +// it runs so the meter moves. +func (e *engine) micTest(device string) event { + cmds := recorderCmds(device) + var cmd *exec.Cmd + for _, c := range cmds { + if _, err := exec.LookPath(c[0]); err == nil { + cmd = exec.Command(c[0], c[1:]...) + break + } + } + if cmd == nil { + return event{"ok": false, "verdict": "No recorder found (need parec, pw-record or arecord)."} + } + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + var errBuf strings.Builder + cmd.Stderr = &errBuf + out, err := cmd.StdoutPipe() + if err != nil || cmd.Start() != nil { + return event{"ok": false, "verdict": "Could not start the recorder."} + } + defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }() + + frame := make([]byte, 3200) // 100 ms + peak, frames, zeroFrames := 0, 0, 0 + var sumSq float64 + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + n, err := io.ReadFull(out, frame) + if err != nil { + why := strings.TrimSpace(errBuf.String()) + e.logs.add("mic test on %q failed after %d frames: %v | %s", device, frames, err, why) + return event{"ok": false, "verdict": "The microphone produced no audio. " + why, "recorder": cmd.Path} + } + fp := 0 + for i := 0; i+1 < n; i += 2 { + v := int(int16(uint16(frame[i]) | uint16(frame[i+1])<<8)) + sumSq += float64(v * v) + if v < 0 { + v = -v + } + if v > fp { + fp = v + } + } + if fp == 0 { + zeroFrames++ + } + if fp > peak { + peak = fp + } + frames++ + e.hub.send(event{"type": "level", "peak": float64(fp) / 32768, "test": true}) + } + rms := 0.0 + if frames > 0 { + rms = math.Sqrt(sumSq/float64(frames*1600)) / 32768 + } + res := event{"ok": true, "peak": float64(peak) / 32768, "rms": rms, "frames": frames} + switch { + case zeroFrames == frames: + res["ok"] = false + res["verdict"] = "Pure silence — the mic is connected but sends nothing. It may be muted, or a Bluetooth headset's mic link may have dropped (try reconnecting the headset or pick another mic)." + case peak < 330: // below about -40 dBFS + res["verdict"] = "Very quiet — Rudder can hear the mic, but barely. Speak closer or raise its input volume." + default: + res["verdict"] = "Working — Rudder can hear you." + } + e.logs.add("mic test on %q: peak=%d rms=%.4f frames=%d verdict=%s", device, peak, rms, frames, res["verdict"]) + return res +} + +// ─── Optional Bluetooth headset reset ───────────────────────────────────────────────────────────── +// +// Some Bluetooth headsets drop the mic side of the HFP link (exact-zero audio while everything looks +// connected; the kernel logs `hci0: corrupted SCO packet`). Re-opening the recorder does not revive +// it; a full reset (disconnect → restart PipeWire/WirePlumber → reconnect) can. That is machine-specific, +// so Rudder only runs one if RUDDER_BT_RESET names an executable that does it (exit 0 = headset works +// again). Rate-limited: never more than once per 20 s. +func btResetPath() string { + p := os.Getenv("RUDDER_BT_RESET") + if p == "" { + return "" + } + if st, err := os.Stat(p); err == nil && !st.IsDir() && st.Mode()&0o111 != 0 { + return p + } + return "" +} + +var ( + resetMu sync.Mutex + lastReset time.Time +) + +// runHeadsetReset runs the RUDDER_BT_RESET command and reports whether it said the headset works again. +func runHeadsetReset(logs *ring) (ok bool, ran bool) { + path := btResetPath() + if path == "" { + return false, false + } + resetMu.Lock() + if time.Since(lastReset) < 20*time.Second { + resetMu.Unlock() + logs.add("headset reset skipped: last one was %.0fs ago (rate limit 20s)", time.Since(lastReset).Seconds()) + return false, false + } + lastReset = time.Now() + resetMu.Unlock() + + logs.add("running %s (headset reset + CVSD)", path) + cmd := exec.Command(path) + cmd.Env = audioEnv() + done := make(chan struct{}) + var out []byte + var err error + go func() { out, err = cmd.CombinedOutput(); close(done) }() + select { + case <-done: + case <-time.After(60 * time.Second): + _ = cmd.Process.Kill() + logs.add("headset reset timed out after 60s") + return false, true + } + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if l != "" { + logs.add("[headset reset] %s", l) + } + } + if err != nil { + logs.add("headset reset failed: %v", err) + return false, true + } + return true, true +} + +// ─── Bluetooth voice-profile helpers ──────────────────────────────────────────────────────────── + +// btCard maps a Bluetooth mic (bluez_input.., or "" = the default input) to its card. +func btCard(device string) string { + if device == "" { + if b, err := runPactl("get-default-source"); err == nil { + device = strings.TrimSpace(string(b)) + } + } + if !strings.HasPrefix(device, "bluez_input.") { + return "" + } + mac := strings.TrimPrefix(device, "bluez_input.") + if i := strings.LastIndex(mac, "."); i > 0 { + mac = mac[:i] + } + return "bluez_card." + mac +} + +func currentProfile(card string) string { + if card == "" { + return "(not bluetooth)" + } + b, err := runPactl("-f", "json", "list", "cards") + if err != nil { + return "?" + } + var cards []struct { + Name string `json:"name"` + ActiveProfile string `json:"active_profile"` + } + if json.Unmarshal(b, &cards) != nil { + return "?" + } + for _, c := range cards { + if c.Name == card { + return c.ActiveProfile + } + } + return "?" +} + +// switchProfile forces a fresh voice link: CVSD if the card is on anything else, otherwise a bounce +// through mSBC and back to CVSD (a same-profile set is a no-op and would not renegotiate the link). +func switchProfile(card string, logs *ring) error { + before := currentProfile(card) + target := "headset-head-unit-cvsd" + if before == target { + if _, err := runPactl("set-card-profile", card, "headset-head-unit-msbc"); err != nil { + logs.add("profile bounce via msbc failed: %v", err) + } + time.Sleep(700 * time.Millisecond) + } + if out, err := runPactl("set-card-profile", card, target); err != nil { + return fmt.Errorf("pactl set-card-profile %s %s: %v %s", card, target, err, strings.TrimSpace(string(out))) + } + time.Sleep(1500 * time.Millisecond) // let the new SCO link and the source come up + logs.add("voice profile %s → %s", before, currentProfile(card)) + return nil +} + +// watchKernelBluetooth copies kernel Bluetooth messages into the log while a session runs +// (`journalctl -k` is readable without root on this system). +func watchKernelBluetooth(ctx context.Context, logs *ring) { + cmd := exec.CommandContext(ctx, "journalctl", "-k", "-f", "-n", "0", "-o", "cat") + out, err := cmd.StdoutPipe() + if err != nil || cmd.Start() != nil { + return + } + sc := bufio.NewScanner(out) + for sc.Scan() { + if l := sc.Text(); strings.Contains(strings.ToLower(l), "bluetooth") { + logs.add("kernel: %s", l) + } + } + _ = cmd.Wait() +} diff --git a/backend/rudder-engine/go.mod b/backend/rudder-engine/go.mod new file mode 100644 index 00000000..917075ed --- /dev/null +++ b/backend/rudder-engine/go.mod @@ -0,0 +1,5 @@ +module dx-maps/backend/rudder-engine + +go 1.22.2 + +require github.com/gorilla/websocket v1.5.3 diff --git a/backend/rudder-engine/go.sum b/backend/rudder-engine/go.sum new file mode 100644 index 00000000..25a9fc4b --- /dev/null +++ b/backend/rudder-engine/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go new file mode 100644 index 00000000..83ed10b5 --- /dev/null +++ b/backend/rudder-engine/main.go @@ -0,0 +1,797 @@ +// rudder-engine — the local speech backend for the rudder-voice plugin. +// +// WHY THIS EXISTS: rudder-voice is a DESKTOP-target plugin. Speech-to-text runs on the user's own +// machine, never in the browser and never on a server. In dev, `bun run dev` starts this process and +// Vite forwards /api/rudder/* to it; in the desktop app, Tauri runs it as a sidecar. Same binary, +// same API, so "works in the Plugin Studio" means "works on desktop". +// +// browser / webview ──HTTP+SSE──► rudder-engine (127.0.0.1:RUDDER_PORT) +// ├─ recorder: parec → pw-record → arecord (mic open ONLY while recording) +// └─ WebSocket ──► nemo-speech serve (127.0.0.1:RUDDER_NEMO_PORT) +// NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF +// +// NO TELEMETRY, NO NETWORK: the runtime is NVIDIA NeMo-Speech.cpp (ggml). ONNX-based runtimes were +// REJECTED: measured 2026-09-26, the tested ONNX GenAI runtime looked up a vendor telemetry host even +// with telemetry "disabled". NeMo-Speech.cpp was traced with strace: zero sockets, zero DNS. +// Its only network path is an optional model download that shells out to `curl` when the model is not +// a local file — so this engine REFUSES to start unless RUDDER_MODEL is an existing local file. +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/gorilla/websocket" +) + +// ─── Config (env, with local defaults) ────────────────────────────────────────────────────────── + +type config struct { + Port string // this engine's HTTP port + NemoPort string // the warm NeMo-Speech server's port (loopback only) + NemoBin string // path to the nemo-speech executable + Model string // local GGUF path — MUST exist (see the no-network note above) + Language string // default language code + EndpointMs int // silence that ends an utterance +} + +func envOr(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} + +func loadConfig() config { + home, _ := os.UserHomeDir() + base := filepath.Join(home, ".local", "share", "rudder") + return config{ + Port: envOr("RUDDER_PORT", "5193"), + NemoPort: envOr("RUDDER_NEMO_PORT", "5194"), + NemoBin: envOr("RUDDER_NEMO_BIN", filepath.Join(base, "bin", "nemo-speech-0.1.0-linux-x86_64-cpu", "bin", "nemo-speech")), + Model: envOr("RUDDER_MODEL", filepath.Join(base, "models", "nemotron-3.5-asr-streaming-0.6b.q8_0.gguf")), + Language: envOr("RUDDER_LANG", "en-US"), + EndpointMs: 800, + } +} + +// ─── Log ring: every line is also served at /api/rudder/log so the Debug panel can copy it ────── + +type ring struct { + mu sync.Mutex + lines []string +} + +// ★ PRIVACY: the log is what people paste to an agent (🐞 Debug → copy report), so it must never carry +// anything that identifies them. Every line is redacted as it is written: device MAC addresses (inside +// Bluetooth device names like bluez_input.AA_BB_…) and the home folder (which contains the username). +// Transcribed words are never logged at all. +var ( + macRe = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:_]){5}[0-9a-f]{2}\b`) + homeDir = func() string { h, _ := os.UserHomeDir(); return h }() +) + +func redact(s string) string { + s = macRe.ReplaceAllString(s, "XX:XX:XX:XX:XX:XX") + if homeDir != "" && homeDir != "/" { + s = strings.ReplaceAll(s, homeDir, "~") + } + return s +} + +func (r *ring) add(format string, a ...any) { + line := redact(time.Now().Format("15:04:05.000") + " " + fmt.Sprintf(format, a...)) + log.Println(line) + r.mu.Lock() + r.lines = append(r.lines, line) + if len(r.lines) > 3000 { + r.lines = r.lines[len(r.lines)-3000:] + } + r.mu.Unlock() +} + +func (r *ring) text() string { + r.mu.Lock() + defer r.mu.Unlock() + return strings.Join(r.lines, "\n") +} + +// ─── Event hub: Server-Sent Events to every open plugin ───────────────────────────────────────── + +type event map[string]any + +type hub struct { + mu sync.Mutex + clients map[chan []byte]struct{} +} + +func (h *hub) subscribe() chan []byte { + c := make(chan []byte, 64) + h.mu.Lock() + h.clients[c] = struct{}{} + h.mu.Unlock() + return c +} + +func (h *hub) unsubscribe(c chan []byte) { + h.mu.Lock() + delete(h.clients, c) + h.mu.Unlock() +} + +func (h *hub) send(e event) { + b, _ := json.Marshal(e) + h.mu.Lock() + defer h.mu.Unlock() + for c := range h.clients { + select { + case c <- b: + default: // a slow client drops an event rather than stalling the audio path + } + } +} + +// ─── Engine: supervises nemo-speech serve and one recording session at a time ─────────────────── + +type engine struct { + cfg config + logs *ring + hub *hub + + mu sync.Mutex + state string // starting | ready | error + stateErr string + nemo *exec.Cmd + recording bool + recorder string + language string + stopSession func() + device string + devs *deviceWatcher + nemoKey string +} + +func (e *engine) setState(s, errText string) { + e.mu.Lock() + e.state, e.stateErr = s, errText + e.mu.Unlock() + e.hub.send(event{"type": "status", "engine": s, "error": errText}) +} + +func (e *engine) status() event { + e.mu.Lock() + defer e.mu.Unlock() + return event{ + "engine": e.state, "error": e.stateErr, "recording": e.recording, "recorder": e.recorder, + "language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device, + "runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model), + } +} + +// startNemo launches the warm speech server. The model loads and warms up once (~20 s on CPU), +// so starting and stopping a recording is instant afterwards. +func (e *engine) startNemo() { + if st, err := os.Stat(e.cfg.Model); err != nil || st.IsDir() { + e.setState("error", "model file not found: "+e.cfg.Model+" (refusing to start: a non-local model would make the runtime download it)") + return + } + if _, err := os.Stat(e.cfg.NemoBin); err != nil { + e.setState("error", "nemo-speech runtime not found: "+e.cfg.NemoBin) + return + } + e.setState("starting", "") + // A fresh random key each start: the speech server answers only this engine, not any other local + // program or web page that finds its port. + cmd := exec.Command(e.cfg.NemoBin, "serve", + "--host", "127.0.0.1", "--port", e.cfg.NemoPort, + "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey) + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} // dies with us + out, _ := cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + if err := cmd.Start(); err != nil { + e.setState("error", "could not start nemo-speech: "+err.Error()) + return + } + e.mu.Lock() + e.nemo = cmd + e.mu.Unlock() + e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort) + go func() { + buf := make([]byte, 4096) + for { + n, err := out.Read(buf) + for _, l := range strings.Split(strings.TrimSpace(string(buf[:n])), "\n") { + if l != "" { + e.logs.add("[nemo] %s", l) + } + } + if err != nil { + return + } + } + }() + go func() { + err := cmd.Wait() + e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err)) + }() + // Ready = the model list answers. + go func() { + url := "http://127.0.0.1:" + e.cfg.NemoPort + "/v1/models" + for i := 0; i < 600; i++ { + time.Sleep(500 * time.Millisecond) + req, _ := http.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer "+e.nemoKey) + resp, err := http.DefaultClient.Do(req) + if err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + e.logs.add("nemo-speech ready after %.1fs", float64(i+1)*0.5) + e.setState("ready", "") + return + } + } + e.mu.Lock() + s := e.state + e.mu.Unlock() + if s == "error" { + return + } + } + e.setState("error", "nemo-speech did not become ready within 5 minutes") + }() +} + +// recorderCmds: raw s16le 16 kHz mono PCM on stdout. parec first — on PipeWire/Bluetooth desktops it +// talks to the sound server directly (the old Rudder's hardest-won lesson: PortAudio/ALSA-only +// capture stalls on Bluetooth headsets). +// RUDDER_RECORDER overrides the list with one space-separated command (e.g. a test that replays a +// PCM file), so the engine can be tested end to end without a microphone. +// device is a sound-server source name from /api/rudder/devices; empty = the system default input. +func recorderCmds(device string) [][]string { + if v := os.Getenv("RUDDER_RECORDER"); v != "" { + return [][]string{strings.Fields(v)} + } + parec := []string{"parec", "--format=s16le", "--rate=16000", "--channels=1", "--raw", "--latency-msec=100"} + pw := []string{"pw-record", "--format=s16", "--rate=16000", "--channels=1"} + if device != "" { + parec = append(parec, "--device="+device) + pw = append(pw, "--target="+device) + } + pw = append(pw, "-") + cmds := [][]string{parec, pw} + if device == "" { // arecord can't address a sound-server source by name + cmds = append(cmds, []string{"arecord", "-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"}) + } + return cmds +} + +func (e *engine) start(lang, device string) error { + e.mu.Lock() + if e.state != "ready" { + e.mu.Unlock() + return fmt.Errorf("engine is %s — wait for it to be ready", e.state) + } + if e.recording { + e.mu.Unlock() + return nil + } + if lang == "" { + lang = e.cfg.Language + } + e.mu.Unlock() + + ws, _, err := websocket.DefaultDialer.Dial("ws://127.0.0.1:"+e.cfg.NemoPort+"/v1/audio/transcriptions/realtime", + http.Header{"Authorization": {"Bearer " + e.nemoKey}}) + if err != nil { + return fmt.Errorf("connect to speech server: %w", err) + } + upd, _ := json.Marshal(event{"type": "session.update", "session": event{ + "sample_rate": 16000, "language": lang, "automatic_punctuation": true, "endpointing_ms": e.cfg.EndpointMs, + }}) + if err := ws.WriteMessage(websocket.TextMessage, upd); err != nil { + ws.Close() + return fmt.Errorf("session.update: %w", err) + } + + // Bluetooth headsets start on CVSD, not mSBC. Measured 2026-09-27 on a Bluetooth headset: on mSBC the + // mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after + // switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an + // 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out. + if card := btCard(device); card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" { + if p := currentProfile(card); p == "headset-head-unit-msbc" { + e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)") + e.hub.send(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"}) + if err := switchProfile(card, e.logs); err != nil { + e.logs.add("pre-switch to CVSD failed: %v", err) + } else { + e.hub.send(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."}) + } + } + } + + first, err := spawnRecorder(device, e.logs) + if err != nil { + ws.Close() + return err + } + recName := first.name + var recMu sync.Mutex + cur := first // the live recorder; replaced when a dead mic link is recovered + + ctx, cancel := context.WithCancel(context.Background()) + var writeMu sync.Mutex + done := make(chan struct{}) + + e.mu.Lock() + e.recording, e.recorder, e.language, e.device = true, recName, lang, device + e.stopSession = func() { + cancel() + recMu.Lock() + cur.kill() + recMu.Unlock() + writeMu.Lock() + _ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`)) + writeMu.Unlock() + select { // let the final words arrive + case <-done: + case <-time.After(4 * time.Second): + } + ws.Close() + } + e.mu.Unlock() + e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device))) + go watchKernelBluetooth(ctx, e.logs) + e.hub.send(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device}) + for _, d := range e.devs.current().Devices { + if (d.ID == device || (device == "" && d.Default)) && d.Muted { + e.hub.send(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."}) + } + } + + // Audio pump: 100 ms frames → speech server, plus a level meter and dead-mic RECOVERY. + go func() { + frame := make([]byte, 3200) + var silentSince time.Time + recoveries := 0 + var recovering atomic.Bool + var resetting atomic.Bool + tracePeak, lastTrace := 0, time.Now() + var healthySince time.Time // a mic that stays healthy 30 s earns back its 3 reconnects + gaveUp := false + lastLevel := time.Time{} + for { + recMu.Lock() + r := cur + recMu.Unlock() + if _, err := io.ReadFull(r.out, frame); err != nil { + if ctx.Err() != nil { + return + } + recMu.Lock() + swapped := cur != r + recMu.Unlock() + if swapped { // we killed it ourselves during a recovery: read from the new one + continue + } + // A headset reset restarts PipeWire, which kills parec mid-read. That is expected: + // wait for the reset to hand over a fresh recorder instead of ending the session. + if resetting.Load() { + for resetting.Load() && ctx.Err() == nil { + time.Sleep(200 * time.Millisecond) + } + continue + } + // The recorder died on its own (bad device, sound server unreachable, device unplugged). + // End the session so the UI stops showing "Listening", and say WHY in plain words. + _ = r.cmd.Wait() + why := strings.TrimSpace(r.errBuf.String()) + e.logs.add("recorder %s ended unexpectedly: %v | stderr: %s", r.name, err, why) + msg := "The microphone stopped delivering audio." + switch { + case strings.Contains(why, "Connection refused") || strings.Contains(why, "pa_context_connect"): + msg = "Can't reach the computer's sound system (PipeWire/PulseAudio). Is it running?" + case strings.Contains(why, "No such entity") || strings.Contains(why, "not found"): + msg = "That microphone isn't available any more — it may have been unplugged or disconnected. Pick another one." + } + if why != "" { + msg += " (" + r.name + ": " + why + ")" + } + e.hub.send(event{"type": "error", "message": msg}) + go e.stop() + return + } + peak := 0 + for i := 0; i+1 < len(frame); i += 2 { + v := int(int16(binary.LittleEndian.Uint16(frame[i:]))) + if v < 0 { + v = -v + } + if v > peak { + peak = v + } + } + // ★ DEAD MIC LINK. Exact-zero samples are not quiet speech (a live mic always has a noise + // floor). Measured 2026-09-27 on a Bluetooth headset (HFP, mSBC): four words + // transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1, + // the headset losing the mic side of the link. Re-opening the source restarts the audio + // stream; the speech session and the transcript carry on untouched. + if peak == 0 { + if silentSince.IsZero() { + silentSince = time.Now() + } else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond { + recoveries++ + switch { + case recoveries == 1: + // Step 1: reopen the recorder (cheap; fixes a stalled stream). + e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone") + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"}) + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + case recoveries == 2 && btCard(device) != "": + // Step 2: force a NEW Bluetooth voice link by switching the headset's voice profile to + // CVSD (the old Rudder's proven-stabler choice). Unlike a detect-first reset script this does not + // "detect first", so it can't decide a dead link is fine. + card := btCard(device) + e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card) + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"}) + resetting.Store(true) + go func() { + defer resetting.Store(false) + if err := switchProfile(card, e.logs); err != nil { + e.hub.send(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()}) + return + } + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + }() + case recoveries == 3 && btResetPath() != "": + // Step 3: full headset reset (disconnect → restart PipeWire → reconnect). + e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET") + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"}) + resetting.Store(true) + go func() { + defer resetting.Store(false) + ok, ran := runHeadsetReset(e.logs) + if !ran { + return + } + if !ok { + e.hub.send(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"}) + } + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + }() + default: + gaveUp = true + e.logs.add("mic still dead after reopen + headset resets — giving up for this session") + msg := "Your microphone keeps going silent. This is the Bluetooth headset dropping its mic link (the kernel reports corrupted voice packets), not Rudder. Try a wired or built-in mic, or turn off the headset's noise cancellation / adaptive audio in its app." + if btResetPath() == "" { + msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)" + } + e.hub.send(event{"type": "warning", "message": msg}) + } + silentSince = time.Now() + } + } else { + if recovering.Load() { + recovering.Store(false) + healthySince = time.Now() + e.logs.add("mic recovered after %d reconnect(s)", recoveries) + e.hub.send(event{"type": "recovered", "message": "Microphone is back — keep talking."}) + } + if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second { + recoveries, gaveUp, healthySince = 0, false, time.Time{} + } + silentSince = time.Time{} + } + // Flight recorder: the mic level twice a second, so a drop can be lined up against kernel and + // sound-server events in the same log (/api/rudder/log). + tracePeak = max(tracePeak, peak) + if time.Since(lastTrace) >= 500*time.Millisecond { + e.logs.add("trace level peak=%d", tracePeak) + tracePeak, lastTrace = 0, time.Now() + } + if time.Since(lastLevel) > 150*time.Millisecond { + lastLevel = time.Now() + e.hub.send(event{"type": "level", "peak": float64(peak) / 32768}) + } + writeMu.Lock() + err := ws.WriteMessage(websocket.BinaryMessage, frame) + writeMu.Unlock() + if err != nil { + if ctx.Err() == nil { + e.hub.send(event{"type": "error", "message": "speech server connection lost: " + err.Error()}) + } + return + } + } + }() + + // Result reader: map the server's events onto the plugin's small vocabulary. + go func() { + defer close(done) + for { + _, msg, err := ws.ReadMessage() + if err != nil { + return + } + var m map[string]any + if json.Unmarshal(msg, &m) != nil { + continue + } + t, _ := m["type"].(string) + text := firstString(m, "transcript", "delta", "text") + switch { + case strings.HasSuffix(t, "input_audio_transcription.delta"): + // Deltas APPEND to the utterance in progress; the server sends many empty ones. + if text != "" { + e.hub.send(event{"type": "partial", "text": text}) + } + case strings.HasSuffix(t, "input_audio_transcription.completed"): + e.hub.send(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)}) + case t == "error": + e.logs.add("speech server error: %s", string(msg)) + e.hub.send(event{"type": "error", "message": string(msg)}) + case t == "input_audio_buffer.committed": + return + default: + e.logs.add("speech event: %s", t) + } + } + }() + return nil +} + +// recorderProc is one running recorder process. +type recorderProc struct { + cmd *exec.Cmd + out io.ReadCloser + name string + errBuf *bytes.Buffer +} + +func (r *recorderProc) kill() { + if r != nil && r.cmd != nil && r.cmd.Process != nil { + _ = r.cmd.Process.Kill() + go func() { _ = r.cmd.Wait() }() + } +} + +// spawnRecorder starts the first recorder that works for this device. +func spawnRecorder(device string, logs *ring) (*recorderProc, error) { + for _, c := range recorderCmds(device) { + if _, err := exec.LookPath(c[0]); err != nil { + continue + } + cmd := exec.Command(c[0], c[1:]...) + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + out, err := cmd.StdoutPipe() + if err != nil { + continue + } + if err := cmd.Start(); err != nil { + logs.add("recorder %s failed to start: %v", c[0], err) + continue + } + return &recorderProc{cmd: cmd, out: out, name: c[0], errBuf: errBuf}, nil + } + return nil, errors.New("no microphone recorder found (need parec, pw-record or arecord)") +} + +func firstString(m map[string]any, keys ...string) string { + for _, k := range keys { + if s, ok := m[k].(string); ok { + return s + } + } + if item, ok := m["item"].(map[string]any); ok { + return firstString(item, keys...) + } + return "" +} + +func (e *engine) stop() { + e.mu.Lock() + stop := e.stopSession + e.stopSession = nil + was := e.recording + e.recording = false + e.mu.Unlock() + if !was || stop == nil { + return + } + stop() + e.logs.add("recording stopped") + e.hub.send(event{"type": "recording", "recording": false}) +} + +func (e *engine) shutdown() { + e.stop() + e.mu.Lock() + n := e.nemo + e.mu.Unlock() + if n != nil && n.Process != nil { + _ = n.Process.Kill() + } +} + +// ─── HTTP API ──────────────────────────────────────────────────────────────────────────────────── + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +// guard: two protections for a server that streams what you say. +// - Host check (DNS rebinding): a web page on any domain can make the browser resolve its own hostname +// to 127.0.0.1 and then read this API as "same-origin". Only requests addressed to 127.0.0.1 or +// localhost are served. +// - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows) +// may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the +// dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests). +func guard(next http.Handler) http.Handler { + extra := os.Getenv("RUDDER_ALLOW_ORIGIN") + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host := r.Host + if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") { + host = host[:i] + } + if host != "127.0.0.1" && host != "localhost" && host != "[::1]" { + http.Error(w, "forbidden host", http.StatusForbidden) + return + } + o := r.Header.Get("Origin") + if o == "tauri://localhost" || o == "http://tauri.localhost" || (extra != "" && o == extra) { + w.Header().Set("Access-Control-Allow-Origin", o) + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Vary", "Origin") + } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) +} + +func main() { + cfg := loadConfig() + logs := &ring{} + h := &hub{clients: map[chan []byte]struct{}{}} + key := make([]byte, 24) + if _, err := rand.Read(key); err != nil { + log.Fatal("no randomness for the speech-server key: ", err) + } + e := &engine{cfg: cfg, logs: logs, hub: h, state: "starting", devs: &deviceWatcher{hub: h, logs: logs}, nemoKey: hex.EncodeToString(key)} + e.devs.run() + + mux := http.NewServeMux() + mux.HandleFunc("/api/rudder/status", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/start", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, 405, event{"error": "POST only"}) + return + } + var body struct { + Language string `json:"language"` + Device string `json:"device"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if err := e.start(body.Language, body.Device); err != nil { + e.logs.add("start failed: %v", err) + writeJSON(w, 409, event{"error": err.Error()}) + return + } + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) { + e.stop() + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/devices", func(w http.ResponseWriter, r *http.Request) { + e.devs.refresh("request") + writeJSON(w, 200, e.devs.current()) + }) + mux.HandleFunc("/api/rudder/test", func(w http.ResponseWriter, r *http.Request) { + var body struct { + Device string `json:"device"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + writeJSON(w, 200, e.micTest(body.Device)) + }) + mux.HandleFunc("/api/rudder/log", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = io.WriteString(w, logs.text()) + }) + mux.HandleFunc("/api/rudder/events", func(w http.ResponseWriter, r *http.Request) { + fl, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", 500) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + c := e.hub.subscribe() + defer e.hub.unsubscribe(c) + first, _ := json.Marshal(event{"type": "status", "status": e.status()}) + fmt.Fprintf(w, "data: %s\n\n", first) + devs, _ := json.Marshal(event{"type": "devices", "devices": e.devs.current()}) + fmt.Fprintf(w, "data: %s\n\n", devs) + fl.Flush() + ping := time.NewTicker(15 * time.Second) + defer ping.Stop() + for { + select { + case <-r.Context().Done(): + return + case b := <-c: + fmt.Fprintf(w, "data: %s\n\n", b) + fl.Flush() + case <-ping.C: + fmt.Fprint(w, ": ping\n\n") + fl.Flush() + } + } + }) + + srv := &http.Server{Addr: "127.0.0.1:" + cfg.Port, Handler: guard(mux)} + logs.add("rudder-engine listening on http://127.0.0.1:%s (runtime=%s model=%s)", cfg.Port, cfg.NemoBin, cfg.Model) + e.startNemo() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sig + e.shutdown() + os.Exit(0) + }() + if err := srv.ListenAndServe(); err != nil { + e.shutdown() + log.Fatal(err) + } +} diff --git a/backend/rudder-engine/setup-runtime.sh b/backend/rudder-engine/setup-runtime.sh new file mode 100755 index 00000000..57c6291b --- /dev/null +++ b/backend/rudder-engine/setup-runtime.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# setup-runtime.sh — fetch Rudder's speech runtime + model, verified by SHA-256, then build the engine. +# +# Rudder (plugin-designs/rudder-voice) transcribes on THIS computer. It needs two things that are too big +# for git: NVIDIA's NeMo-Speech.cpp runtime (~4 MB, CPU build) and the Nemotron 3.5 ASR Streaming GGUF +# (~742 MB). Both are pinned by version and checksum; a mismatch aborts instead of running an unverified +# file. After this, `bun run dev` starts the engine and Rudder appears in the Plugin Studio. +# +# bash backend/rudder-engine/setup-runtime.sh # installs into ~/.local/share/rudder +# RUDDER_HOME=/some/dir bash backend/rudder-engine/setup-runtime.sh +# +# Linux x86_64 only for now (the runtime ships other platforms; add them when a desktop build needs them). +set -euo pipefail + +RUDDER_HOME="${RUDDER_HOME:-$HOME/.local/share/rudder}" +RT_VER="0.1.0" +RT_NAME="nemo-speech-${RT_VER}-linux-x86_64-cpu" +RT_URL="https://github.com/NVIDIA/NeMo-Speech.cpp/releases/download/v${RT_VER}/${RT_NAME}.tar.gz" +RT_SHA="0f74131d631ad2c694cf0ec53490866bb6461147959589a69fb6fc231944065b" +# NVIDIA re-converted the GGUF on 2026-09-10 (embedded tokenizer); this revision is the one Rudder is tested with. +MODEL_REV="ea30d66debe3" +MODEL_FILE="nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +MODEL_URL="https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${MODEL_REV}/${MODEL_FILE}" +MODEL_SHA="3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1" + +say() { printf '\033[1;34m→\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; } + +[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || die "This script currently supports Linux x86_64 only." +command -v curl >/dev/null || die "curl is required." +command -v sha256sum >/dev/null || die "sha256sum is required." + +fetch() { # url dest sha + local url="$1" dest="$2" sha="$3" + if [ -f "$dest" ] && [ "$(sha256sum "$dest" | cut -d' ' -f1)" = "$sha" ]; then + say "already have $(basename "$dest") (checksum OK)"; return + fi + say "downloading $(basename "$dest")" + curl -fL --retry 3 -C - -o "$dest.part" "$url" + [ "$(sha256sum "$dest.part" | cut -d' ' -f1)" = "$sha" ] || { rm -f "$dest.part"; die "checksum mismatch for $(basename "$dest") — refusing to use it"; } + mv "$dest.part" "$dest" +} + +mkdir -p "$RUDDER_HOME/bin" "$RUDDER_HOME/models" +fetch "$RT_URL" "$RUDDER_HOME/bin/$RT_NAME.tar.gz" "$RT_SHA" +tar -xzf "$RUDDER_HOME/bin/$RT_NAME.tar.gz" -C "$RUDDER_HOME/bin" +fetch "$MODEL_URL" "$RUDDER_HOME/models/$MODEL_FILE" "$MODEL_SHA" + +if command -v go >/dev/null; then + say "building backend/bin/rudder-engine" + (cd "$(dirname "$0")" && go build -o ../bin/rudder-engine .) +else + printf '! Go not found — install Go, then: cd backend/rudder-engine && go build -o ../bin/rudder-engine .\n' +fi + +for tool in parec pactl; do command -v "$tool" >/dev/null || printf '! %s not found — install pipewire-pulse or pulseaudio-utils for microphone capture.\n' "$tool"; done + +say "done. Runtime: $RUDDER_HOME/bin/$RT_NAME Model: $RUDDER_HOME/models/$MODEL_FILE" +say "run \`bun run dev\` and open Rudder (rudder-voice) in the Plugin Studio." diff --git a/internal/backend-dev-plugin.mjs b/internal/backend-dev-plugin.mjs new file mode 100644 index 00000000..46993c63 --- /dev/null +++ b/internal/backend-dev-plugin.mjs @@ -0,0 +1,61 @@ +/** + * internal/backend-dev-plugin.mjs — `bun run dev` starts the local plugin backends. + * + * WHY: some plugins ship to DESKTOP with a local backend (rudder-voice → backend/bin/rudder-engine, + * run as a Tauri sidecar). In the Plugin Studio the same backend must be running, or the plugin can't + * be tested in the browser. This plugin starts each built backend with the dev server, forwards its + * /api// routes, and stops it when the dev server stops. A backend that isn't built is skipped + * with a hint: plugins without a backend never notice this file. + * + * DEV ONLY (`apply: 'serve'`). A production build never starts or bundles a backend. + */ +import { spawn } from 'node:child_process' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' + +// name → how to run it. `route` is proxied to `port` on 127.0.0.1. +export const DEV_BACKENDS = [ + { + name: 'rudder-engine', + bin: 'backend/bin/rudder-engine', + build: 'cd backend/rudder-engine && go build -o ../bin/rudder-engine .', + route: '/api/rudder', + port: Number(process.env.RUDDER_PORT || 5193), + env: { RUDDER_PORT: String(process.env.RUDDER_PORT || 5193), RUDDER_NEMO_PORT: String(process.env.RUDDER_NEMO_PORT || 5194) }, + }, +] + +// The proxy table for vite's `server.proxy`. SSE (`/events`) streams through unbuffered. +export function devBackendProxy() { + return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false }])) +} + +export function backendDevPlugin(root) { + const children = [] + return { + name: 'dx-backend-dev', + apply: 'serve', + configureServer(server) { + for (const b of DEV_BACKENDS) { + const bin = resolve(root, b.bin) + if (!existsSync(bin)) { + server.config.logger.warn(`[backend] ${b.name} not built — its plugin will show "engine offline". Build: ${b.build}`) + continue + } + const child = spawn(bin, [], { cwd: root, env: { ...process.env, ...b.env }, stdio: ['ignore', 'pipe', 'pipe'] }) + const tag = `[${b.name}]` + const out = (d) => String(d).trim().split('\n').forEach((l) => l && server.config.logger.info(`${tag} ${l}`)) + child.stdout.on('data', out) + child.stderr.on('data', out) + child.on('exit', (code) => server.config.logger.warn(`${tag} exited (${code})`)) + children.push(child) + server.config.logger.info(`${tag} started → ${b.route} on 127.0.0.1:${b.port}`) + } + const stop = () => children.forEach((c) => { try { c.kill('SIGTERM') } catch {} }) + server.httpServer?.on('close', stop) + process.once('exit', stop) + process.once('SIGINT', () => { stop(); process.exit(0) }) + process.once('SIGTERM', () => { stop(); process.exit(0) }) + }, + } +} diff --git a/plugin-designs/rudder-voice/README.md b/plugin-designs/rudder-voice/README.md new file mode 100644 index 00000000..06c50fe9 --- /dev/null +++ b/plugin-designs/rudder-voice/README.md @@ -0,0 +1,55 @@ +# rudder-voice — Rudder, private voice-to-text (desktop target) + +**Rudder is a plugin that isn't a map.** It shows that the same dx-maps pattern — one folder in +`plugin-designs/`, developed live in the Plugin Studio — can ship a **desktop app with its own local +backend**, not only embeddable maps. It is also a genuinely useful tool for the people who build here: +talk to your agents instead of typing, with nothing leaving your computer. + +- **Streaming.** Words appear in the text box while you speak; talk for as long as you like. +- **Type and talk at once.** Type, paste links or code anywhere in the box; everything you say is + added to the end. While you're typing, Rudder holds its words and adds them when you pause. +- **CPU only.** No graphics card. About 1 GB of memory; keeps up with normal speech on a laptop CPU. +- **Offline, no telemetry.** NVIDIA Nemotron 3.5 ASR Streaming (0.6B) run by NVIDIA's open-source + NeMo-Speech.cpp (ggml). Traced with `strace` while transcribing: zero sockets, zero DNS lookups. +- **Microphones like a video-call app.** Pick the input, hot-plug updates, a 3-second mic test, and + plain-language errors (muted, disconnected, dead Bluetooth mic link — which it tries to recover). + +## Run it in the Plugin Studio + +```bash +bash backend/rudder-engine/setup-runtime.sh # once: runtime + model (verified by SHA-256) + engine build +bun run dev # starts the engine too; open "rudder-voice" in the studio +``` + +Linux with PipeWire or PulseAudio (`parec`, `pactl`). The engine listens on `127.0.0.1` only. + +## How it fits together + +``` + (this folder) ──HTTP + SSE /api/rudder──► backend/rudder-engine (Go) + ├─ parec / pw-record (mic, open only while recording) + └─ WebSocket ► nemo-speech serve (127.0.0.1, model stays loaded) +``` + +`bun run dev` starts `backend/bin/rudder-engine` through `internal/backend-dev-plugin.mjs` and forwards +`/api/rudder` to it, so the plugin calls the same API in the browser as it will inside the desktop app, +where Tauri runs the engine as a sidecar and passes its address in `profile-config` (`apiBase`). + +`rudder-voice.meta.json` declares `"targets": ["desktop"]`: on a plain website there is no local engine, +so this plugin is not meant for web embeds. + +## Settings (environment variables) + +| Variable | Default | What it does | +|---|---|---| +| `RUDDER_PORT` | `5193` | engine HTTP port (loopback) | +| `RUDDER_NEMO_PORT` | `5194` | speech server port (loopback) | +| `RUDDER_MODEL` / `RUDDER_NEMO_BIN` | `~/.local/share/rudder/...` | model and runtime paths; the engine refuses a model that isn't a local file | +| `RUDDER_LANG` | `en-US` | default language (the model supports ~30) | +| `RUDDER_KEEP_MSBC` | unset | keep Bluetooth headsets on mSBC instead of switching them to the more stable CVSD | +| `RUDDER_BT_RESET` | unset | optional command that fully resets a Bluetooth headset, used when its mic link dies | + +## Not yet + +- Desktop build (`desktop:build`) with the engine as a Tauri sidecar. +- A saved word-correction table (PocketBase) for technical terms the model mishears. diff --git a/plugin-designs/rudder-voice/index.html b/plugin-designs/rudder-voice/index.html new file mode 100644 index 00000000..82187f90 --- /dev/null +++ b/plugin-designs/rudder-voice/index.html @@ -0,0 +1,32 @@ + + + + + + doxa-maps · rudder-voice staging + + + + +
+

rudder-voice · staging

+

Desktop-target plugin: private, offline, streaming voice-to-text. The speech model runs in the + local backend backend/rudder-engine (NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR), started by + bun run dev and reached at /api/rudder.

+
+ +
+
+ + + diff --git a/plugin-designs/rudder-voice/index.js b/plugin-designs/rudder-voice/index.js new file mode 100644 index 00000000..579c5acc --- /dev/null +++ b/plugin-designs/rudder-voice/index.js @@ -0,0 +1,23 @@ +/** + * rudder-voice / index.js — Bundle entry (Capability-1 single element, DESKTOP target). + * + * — private, offline, streaming voice-to-text. The UI is this bundle; the speech model + * runs in the local backend (backend/rudder-engine), started by `bun run dev` in the Plugin Studio and + * by Tauri as a sidecar in the desktop app. See rudder-voice.vue for the contract. + * + * Desktop-only: rudder-voice.meta.json declares "targets": ["desktop"]. On a plain website there is no + * local engine, so the element shows "Engine offline" instead of pretending to work. + * + * Usage (desktop shell / Plugin Studio): + * + */ +import { defineCustomElement } from 'vue' +import RudderVoice from './rudder-voice.vue' + +const RudderVoiceElement = defineCustomElement(RudderVoice) + +if (typeof customElements !== 'undefined') { + if (!customElements.get('rudder-voice')) customElements.define('rudder-voice', RudderVoiceElement) +} + +export default RudderVoiceElement diff --git a/plugin-designs/rudder-voice/rudder-voice.meta.json b/plugin-designs/rudder-voice/rudder-voice.meta.json new file mode 100644 index 00000000..316e368c --- /dev/null +++ b/plugin-designs/rudder-voice/rudder-voice.meta.json @@ -0,0 +1,7 @@ +{ + "_comment": "Rudder — private, offline, streaming voice-to-text. DESKTOP target: the speech model runs in the local backend (backend/rudder-engine: NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF), started by `bun run dev` and run by Tauri as a sidecar. Not for public web embeds: a website has no local engine.", + "type": "single", + "label": "Rudder — voice to text (desktop)", + "targets": ["desktop"], + "backend": "backend/rudder-engine" +} diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue new file mode 100644 index 00000000..2b2ff3ee --- /dev/null +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -0,0 +1,650 @@ + + + + + diff --git a/vite.config.js b/vite.config.js index e58edb14..8f1cfac7 100644 --- a/vite.config.js +++ b/vite.config.js @@ -24,6 +24,7 @@ import vue from '@vitejs/plugin-vue' import cssInjectedByJsPlugin from 'vite-plugin-css-injected-by-js' import { autoSpaDevPlugin } from './internal/plugin-gallery/dev-plugin.mjs' import { stripShadowCssCommentsPlugin } from './internal/vite-plugin-strip-shadow-css-comments.mjs' +import { backendDevPlugin, devBackendProxy } from './internal/backend-dev-plugin.mjs' import { fileURLToPath } from 'node:url' import { resolve, dirname } from 'node:path' import { readdirSync, statSync, existsSync } from 'node:fs' @@ -118,7 +119,9 @@ export default defineConfig(({ command, mode }) => { // The dev server listens on THIS MACHINE ONLY. A reachable one serves the source tree, node_modules // and the token baked into /; a remote viewer reaches it through an ssh tunnel (ssh -L), never a public // port. `--host` on the command line still overrides this for a deliberate LAN session. - server: { host: '127.0.0.1' }, + // Local plugin backends (backend/, e.g. rudder-engine) are started by backendDevPlugin and reached + // through these same-origin routes, so a plugin calls /api//… in dev exactly as on desktop. + server: { host: '127.0.0.1', proxy: devBackendProxy() }, define: { // ★ NO 'unsafe-eval' (CRITIC-4, 2026-09-26). vue-i18n's default message compiler builds each message // with `new Function`, so a partner with a strict Content-Security-Policy saw every translated string @@ -159,6 +162,7 @@ export default defineConfig(({ command, mode }) => { vue({ customElement: true }), cssInjectedByJsPlugin(), autoSpaDevPlugin(env.VITE_MAPBOX_TOKEN || process.env.VITE_MAPBOX_TOKEN || ''), + backendDevPlugin(__dirname), ], resolve: { alias: { '@map': resolve(__dirname, 'library') },