diff --git a/.gitignore b/.gitignore index 21dac278..3e0c1fc7 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,6 @@ data-private/ /public/assets/ /public/__gallery/ /public/index.html + +# Local plugin backends (backend//) — the compiled binaries are rebuilt from source, never committed. +/backend/bin/ diff --git a/backend/rudder-engine/devices.go b/backend/rudder-engine/devices.go new file mode 100644 index 00000000..be2be71c --- /dev/null +++ b/backend/rudder-engine/devices.go @@ -0,0 +1,456 @@ +// devices.go — microphone discovery, hot-plug, and the sound-server environment. +// +// The Jitsi Meet pattern (lib-jitsi-meet modules/RTC/RTCUtils.js), done natively because Rudder records +// outside the browser: +// +// Jitsi (browser) Rudder (native, PipeWire/PulseAudio) +// navigator.mediaDevices.enumerateDevices() → pactl -f json list sources +// 'devicechange' event + polling fallback → pactl subscribe (+ re-list on every source/card event) +// compareAvailableMediaDevices(new) → sameDevices(old, new) — only emit on a real change +// getUserMedia({audio:{deviceId:{exact:id}}}) → parec --device= +// +// Jitsi also reports the device list to analytics (sendDeviceListToAnalytics). Rudder never does: +// the list goes to the local plugin over SSE and nowhere else. +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "math" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "time" +) + +// audioEnv gives child processes a path to the user's sound server even when the engine was launched +// from a context without a desktop session (a detached shell, a service, an agent's terminal). Measured +// 2026-09-27: started with no XDG_RUNTIME_DIR, parec exited instantly with "Connection refused" and every +// recording captured nothing, although the socket sat at /run/user//pulse/native all along. +func audioEnv() []string { + env := os.Environ() + rt := os.Getenv("XDG_RUNTIME_DIR") + if rt == "" { + rt = fmt.Sprintf("/run/user/%d", os.Getuid()) + if st, err := os.Stat(rt); err == nil && st.IsDir() { + env = append(env, "XDG_RUNTIME_DIR="+rt) + } + } + // systemctl --user (a headset-reset command may restart PipeWire) needs the session bus. + if os.Getenv("DBUS_SESSION_BUS_ADDRESS") == "" { + if _, err := os.Stat(rt + "/bus"); err == nil { + env = append(env, "DBUS_SESSION_BUS_ADDRESS=unix:path="+rt+"/bus") + } + } + if os.Getenv("PULSE_SERVER") == "" { + if _, err := os.Stat(rt + "/pulse/native"); err == nil { + env = append(env, "PULSE_SERVER=unix:"+rt+"/pulse/native") + } + } + return env +} + +// Device is one microphone as the plugin shows it. +type Device struct { + ID string `json:"id"` // the sound server's source name: what parec --device takes + Label string `json:"label"` // human name ("My Headset", "Built-in Audio Analog Stereo") + Bus string `json:"bus"` // bluetooth | usb | pci | ... + Default bool `json:"default"` // the system's default input + Muted bool `json:"muted"` + Volume string `json:"volume"` // e.g. "52%" + State string `json:"state"` // RUNNING | IDLE | SUSPENDED +} + +// DeviceList is the whole picture: the mics, plus sound cards that have a mic but are switched off +// (the laptop's built-in card, profile "off", is the common case — its mic simply doesn't exist until +// the card is turned on in Sound settings). +type DeviceList struct { + Devices []Device `json:"devices"` + Default string `json:"default"` + OffCards []string `json:"offCards"` + Error string `json:"error,omitempty"` +} + +func runPactl(args ...string) ([]byte, error) { + cmd := exec.Command("pactl", args...) + cmd.Env = audioEnv() + return cmd.Output() +} + +func listDevices() DeviceList { + out := DeviceList{Devices: []Device{}, OffCards: []string{}} + if _, err := exec.LookPath("pactl"); err != nil { + out.Error = "pactl not found: install pipewire-pulse or pulseaudio-utils to list microphones" + return out + } + if b, err := runPactl("get-default-source"); err == nil { + out.Default = strings.TrimSpace(string(b)) + } + b, err := runPactl("-f", "json", "list", "sources") + if err != nil { + out.Error = "cannot reach the sound server (pactl list sources failed: " + err.Error() + ")" + return out + } + var raw []struct { + Name string `json:"name"` + Description string `json:"description"` + State string `json:"state"` + Mute bool `json:"mute"` + Volume map[string]any `json:"volume"` + Properties map[string]any `json:"properties"` + } + if err := json.Unmarshal(b, &raw); err != nil { + out.Error = "unexpected pactl output: " + err.Error() + return out + } + for _, s := range raw { + // A ".monitor" source is the loopback of a SPEAKER, not a microphone: listing it would let the + // user "record" their own music. Hidden, as Jitsi lists only audioinput kinds. + if strings.HasSuffix(s.Name, ".monitor") || s.Properties["device.class"] == "monitor" { + continue + } + d := Device{ID: s.Name, Label: s.Description, State: s.State, Muted: s.Mute, Default: s.Name == out.Default} + if bus, ok := s.Properties["device.bus"].(string); ok { + d.Bus = bus + } + for _, ch := range s.Volume { // first channel is representative + if m, ok := ch.(map[string]any); ok { + if p, ok := m["value_percent"].(string); ok { + d.Volume = p + break + } + } + } + out.Devices = append(out.Devices, d) + } + // Cards whose active profile is "off" but which offer an input profile. + if b, err := runPactl("-f", "json", "list", "cards"); err == nil { + var cards []struct { + Name string `json:"name"` + ActiveProfile string `json:"active_profile"` + Properties map[string]any `json:"properties"` + Profiles map[string]struct { + Sources int `json:"sources"` + Available bool `json:"available"` + } `json:"profiles"` + } + if json.Unmarshal(b, &cards) == nil { + for _, c := range cards { + if c.ActiveProfile != "off" { + continue + } + for _, p := range c.Profiles { + if p.Sources > 0 && p.Available { + label, _ := c.Properties["device.description"].(string) + if label == "" { + label = c.Name + } + out.OffCards = append(out.OffCards, label) + break + } + } + } + } + } + return out +} + +// sameDevices mirrors Jitsi's compareAvailableMediaDevices: emit only when something a user would +// notice changed (a mic came, went, was muted, or the default moved), not on every state flicker. +func sameDevices(a, b DeviceList) bool { + if a.Default != b.Default || len(a.Devices) != len(b.Devices) || a.Error != b.Error || len(a.OffCards) != len(b.OffCards) { + return false + } + for i := range a.Devices { + x, y := a.Devices[i], b.Devices[i] + if x.ID != y.ID || x.Label != y.Label || x.Muted != y.Muted || x.Volume != y.Volume { + return false + } + } + return true +} + +// deviceWatcher keeps the current list and pushes a "devices" event on every real change. It listens +// to `pactl subscribe` (the native 'devicechange') and, like Jitsi, also polls as a fallback in case +// the subscription dies. +type deviceWatcher struct { + mu sync.Mutex + last DeviceList + hub *hub + logs *ring +} + +func (w *deviceWatcher) current() DeviceList { + w.mu.Lock() + defer w.mu.Unlock() + return w.last +} + +func (w *deviceWatcher) refresh(reason string) { + now := listDevices() + w.mu.Lock() + changed := !sameDevices(w.last, now) + w.last = now + w.mu.Unlock() + if changed { + w.logs.add("microphones (%s): %d found, default=%s %s", reason, len(now.Devices), now.Default, now.Error) + w.hub.send(event{"type": "devices", "devices": now}) + } +} + +func (w *deviceWatcher) run() { + w.refresh("startup") + // Poll fallback (Jitsi polls enumerateDevices when devicechange is unavailable). + go func() { + for range time.Tick(5 * time.Second) { + w.refresh("poll") + } + }() + go func() { + for { + cmd := exec.Command("pactl", "subscribe") + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + out, err := cmd.StdoutPipe() + if err == nil && cmd.Start() == nil { + sc := bufio.NewScanner(out) + var debounce *time.Timer + for sc.Scan() { + l := sc.Text() + if strings.Contains(l, " card ") || (strings.Contains(l, " source ") && !strings.Contains(l, "source-output")) { + w.logs.add("sound-server event: %s", l) + } + if strings.Contains(l, " source ") || strings.Contains(l, " card ") || strings.Contains(l, " server ") { + if debounce != nil { + debounce.Stop() + } + debounce = time.AfterFunc(300*time.Millisecond, func() { w.refresh("hot-plug") }) + } + } + _ = cmd.Wait() + } + time.Sleep(3 * time.Second) // sound server restarted or unreachable: retry + } + }() +} + +// micTest records ~3 s from one device WITHOUT transcribing and says, in plain words, whether the mic +// works — Rudder's version of Jitsi's pre-join audio preview. Level events stream to the plugin while +// it runs so the meter moves. +func (e *engine) micTest(device string) event { + cmds := recorderCmds(device) + var cmd *exec.Cmd + for _, c := range cmds { + if _, err := exec.LookPath(c[0]); err == nil { + cmd = exec.Command(c[0], c[1:]...) + break + } + } + if cmd == nil { + return event{"ok": false, "verdict": "No recorder found (need parec, pw-record or arecord)."} + } + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + var errBuf strings.Builder + cmd.Stderr = &errBuf + out, err := cmd.StdoutPipe() + if err != nil || cmd.Start() != nil { + return event{"ok": false, "verdict": "Could not start the recorder."} + } + defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }() + + frame := make([]byte, 3200) // 100 ms + peak, frames, zeroFrames := 0, 0, 0 + var sumSq float64 + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + n, err := io.ReadFull(out, frame) + if err != nil { + why := strings.TrimSpace(errBuf.String()) + e.logs.add("mic test on %q failed after %d frames: %v | %s", device, frames, err, why) + return event{"ok": false, "verdict": "The microphone produced no audio. " + why, "recorder": cmd.Path} + } + fp := 0 + for i := 0; i+1 < n; i += 2 { + v := int(int16(uint16(frame[i]) | uint16(frame[i+1])<<8)) + sumSq += float64(v * v) + if v < 0 { + v = -v + } + if v > fp { + fp = v + } + } + if fp == 0 { + zeroFrames++ + } + if fp > peak { + peak = fp + } + frames++ + e.hub.send(event{"type": "level", "peak": float64(fp) / 32768, "test": true}) + } + rms := 0.0 + if frames > 0 { + rms = math.Sqrt(sumSq/float64(frames*1600)) / 32768 + } + res := event{"ok": true, "peak": float64(peak) / 32768, "rms": rms, "frames": frames} + switch { + case zeroFrames == frames: + res["ok"] = false + res["verdict"] = "Pure silence — the mic is connected but sends nothing. It may be muted, or a Bluetooth headset's mic link may have dropped (try reconnecting the headset or pick another mic)." + case peak < 330: // below about -40 dBFS + res["verdict"] = "Very quiet — Rudder can hear the mic, but barely. Speak closer or raise its input volume." + default: + res["verdict"] = "Working — Rudder can hear you." + } + e.logs.add("mic test on %q: peak=%d rms=%.4f frames=%d verdict=%s", device, peak, rms, frames, res["verdict"]) + return res +} + +// ─── Optional Bluetooth headset reset ───────────────────────────────────────────────────────────── +// +// Some Bluetooth headsets drop the mic side of the HFP link (exact-zero audio while everything looks +// connected; the kernel logs `hci0: corrupted SCO packet`). Re-opening the recorder does not revive +// it; a full reset (disconnect → restart PipeWire/WirePlumber → reconnect) can. That is machine-specific, +// so Rudder only runs one if RUDDER_BT_RESET names an executable that does it (exit 0 = headset works +// again). Rate-limited: never more than once per 20 s. +func btResetPath() string { + p := os.Getenv("RUDDER_BT_RESET") + if p == "" { + return "" + } + if st, err := os.Stat(p); err == nil && !st.IsDir() && st.Mode()&0o111 != 0 { + return p + } + return "" +} + +var ( + resetMu sync.Mutex + lastReset time.Time +) + +// runHeadsetReset runs the RUDDER_BT_RESET command and reports whether it said the headset works again. +func runHeadsetReset(logs *ring) (ok bool, ran bool) { + path := btResetPath() + if path == "" { + return false, false + } + resetMu.Lock() + if time.Since(lastReset) < 20*time.Second { + resetMu.Unlock() + logs.add("headset reset skipped: last one was %.0fs ago (rate limit 20s)", time.Since(lastReset).Seconds()) + return false, false + } + lastReset = time.Now() + resetMu.Unlock() + + logs.add("running %s (headset reset + CVSD)", path) + cmd := exec.Command(path) + cmd.Env = audioEnv() + done := make(chan struct{}) + var out []byte + var err error + go func() { out, err = cmd.CombinedOutput(); close(done) }() + select { + case <-done: + case <-time.After(60 * time.Second): + _ = cmd.Process.Kill() + logs.add("headset reset timed out after 60s") + return false, true + } + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if l != "" { + logs.add("[headset reset] %s", l) + } + } + if err != nil { + logs.add("headset reset failed: %v", err) + return false, true + } + return true, true +} + +// ─── Bluetooth voice-profile helpers ──────────────────────────────────────────────────────────── + +// btCard maps a Bluetooth mic (bluez_input.., or "" = the default input) to its card. +func btCard(device string) string { + if device == "" { + if b, err := runPactl("get-default-source"); err == nil { + device = strings.TrimSpace(string(b)) + } + } + if !strings.HasPrefix(device, "bluez_input.") { + return "" + } + mac := strings.TrimPrefix(device, "bluez_input.") + if i := strings.LastIndex(mac, "."); i > 0 { + mac = mac[:i] + } + return "bluez_card." + mac +} + +func currentProfile(card string) string { + if card == "" { + return "(not bluetooth)" + } + b, err := runPactl("-f", "json", "list", "cards") + if err != nil { + return "?" + } + var cards []struct { + Name string `json:"name"` + ActiveProfile string `json:"active_profile"` + } + if json.Unmarshal(b, &cards) != nil { + return "?" + } + for _, c := range cards { + if c.Name == card { + return c.ActiveProfile + } + } + return "?" +} + +// switchProfile forces a fresh voice link: CVSD if the card is on anything else, otherwise a bounce +// through mSBC and back to CVSD (a same-profile set is a no-op and would not renegotiate the link). +func switchProfile(card string, logs *ring) error { + before := currentProfile(card) + target := "headset-head-unit-cvsd" + if before == target { + if _, err := runPactl("set-card-profile", card, "headset-head-unit-msbc"); err != nil { + logs.add("profile bounce via msbc failed: %v", err) + } + time.Sleep(700 * time.Millisecond) + } + if out, err := runPactl("set-card-profile", card, target); err != nil { + return fmt.Errorf("pactl set-card-profile %s %s: %v %s", card, target, err, strings.TrimSpace(string(out))) + } + time.Sleep(1500 * time.Millisecond) // let the new SCO link and the source come up + logs.add("voice profile %s → %s", before, currentProfile(card)) + return nil +} + +// watchKernelBluetooth copies kernel Bluetooth messages into the log while a session runs +// (`journalctl -k` is readable without root on this system). +func watchKernelBluetooth(ctx context.Context, logs *ring) { + cmd := exec.CommandContext(ctx, "journalctl", "-k", "-f", "-n", "0", "-o", "cat") + out, err := cmd.StdoutPipe() + if err != nil || cmd.Start() != nil { + return + } + sc := bufio.NewScanner(out) + for sc.Scan() { + if l := sc.Text(); strings.Contains(strings.ToLower(l), "bluetooth") { + logs.add("kernel: %s", l) + } + } + _ = cmd.Wait() +} diff --git a/backend/rudder-engine/go.mod b/backend/rudder-engine/go.mod new file mode 100644 index 00000000..917075ed --- /dev/null +++ b/backend/rudder-engine/go.mod @@ -0,0 +1,5 @@ +module dx-maps/backend/rudder-engine + +go 1.22.2 + +require github.com/gorilla/websocket v1.5.3 diff --git a/backend/rudder-engine/go.sum b/backend/rudder-engine/go.sum new file mode 100644 index 00000000..25a9fc4b --- /dev/null +++ b/backend/rudder-engine/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go new file mode 100644 index 00000000..83ed10b5 --- /dev/null +++ b/backend/rudder-engine/main.go @@ -0,0 +1,797 @@ +// rudder-engine — the local speech backend for the rudder-voice plugin. +// +// WHY THIS EXISTS: rudder-voice is a DESKTOP-target plugin. Speech-to-text runs on the user's own +// machine, never in the browser and never on a server. In dev, `bun run dev` starts this process and +// Vite forwards /api/rudder/* to it; in the desktop app, Tauri runs it as a sidecar. Same binary, +// same API, so "works in the Plugin Studio" means "works on desktop". +// +// browser / webview ──HTTP+SSE──► rudder-engine (127.0.0.1:RUDDER_PORT) +// ├─ recorder: parec → pw-record → arecord (mic open ONLY while recording) +// └─ WebSocket ──► nemo-speech serve (127.0.0.1:RUDDER_NEMO_PORT) +// NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF +// +// NO TELEMETRY, NO NETWORK: the runtime is NVIDIA NeMo-Speech.cpp (ggml). ONNX-based runtimes were +// REJECTED: measured 2026-09-26, the tested ONNX GenAI runtime looked up a vendor telemetry host even +// with telemetry "disabled". NeMo-Speech.cpp was traced with strace: zero sockets, zero DNS. +// Its only network path is an optional model download that shells out to `curl` when the model is not +// a local file — so this engine REFUSES to start unless RUDDER_MODEL is an existing local file. +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/gorilla/websocket" +) + +// ─── Config (env, with local defaults) ────────────────────────────────────────────────────────── + +type config struct { + Port string // this engine's HTTP port + NemoPort string // the warm NeMo-Speech server's port (loopback only) + NemoBin string // path to the nemo-speech executable + Model string // local GGUF path — MUST exist (see the no-network note above) + Language string // default language code + EndpointMs int // silence that ends an utterance +} + +func envOr(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} + +func loadConfig() config { + home, _ := os.UserHomeDir() + base := filepath.Join(home, ".local", "share", "rudder") + return config{ + Port: envOr("RUDDER_PORT", "5193"), + NemoPort: envOr("RUDDER_NEMO_PORT", "5194"), + NemoBin: envOr("RUDDER_NEMO_BIN", filepath.Join(base, "bin", "nemo-speech-0.1.0-linux-x86_64-cpu", "bin", "nemo-speech")), + Model: envOr("RUDDER_MODEL", filepath.Join(base, "models", "nemotron-3.5-asr-streaming-0.6b.q8_0.gguf")), + Language: envOr("RUDDER_LANG", "en-US"), + EndpointMs: 800, + } +} + +// ─── Log ring: every line is also served at /api/rudder/log so the Debug panel can copy it ────── + +type ring struct { + mu sync.Mutex + lines []string +} + +// ★ PRIVACY: the log is what people paste to an agent (🐞 Debug → copy report), so it must never carry +// anything that identifies them. Every line is redacted as it is written: device MAC addresses (inside +// Bluetooth device names like bluez_input.AA_BB_…) and the home folder (which contains the username). +// Transcribed words are never logged at all. +var ( + macRe = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:_]){5}[0-9a-f]{2}\b`) + homeDir = func() string { h, _ := os.UserHomeDir(); return h }() +) + +func redact(s string) string { + s = macRe.ReplaceAllString(s, "XX:XX:XX:XX:XX:XX") + if homeDir != "" && homeDir != "/" { + s = strings.ReplaceAll(s, homeDir, "~") + } + return s +} + +func (r *ring) add(format string, a ...any) { + line := redact(time.Now().Format("15:04:05.000") + " " + fmt.Sprintf(format, a...)) + log.Println(line) + r.mu.Lock() + r.lines = append(r.lines, line) + if len(r.lines) > 3000 { + r.lines = r.lines[len(r.lines)-3000:] + } + r.mu.Unlock() +} + +func (r *ring) text() string { + r.mu.Lock() + defer r.mu.Unlock() + return strings.Join(r.lines, "\n") +} + +// ─── Event hub: Server-Sent Events to every open plugin ───────────────────────────────────────── + +type event map[string]any + +type hub struct { + mu sync.Mutex + clients map[chan []byte]struct{} +} + +func (h *hub) subscribe() chan []byte { + c := make(chan []byte, 64) + h.mu.Lock() + h.clients[c] = struct{}{} + h.mu.Unlock() + return c +} + +func (h *hub) unsubscribe(c chan []byte) { + h.mu.Lock() + delete(h.clients, c) + h.mu.Unlock() +} + +func (h *hub) send(e event) { + b, _ := json.Marshal(e) + h.mu.Lock() + defer h.mu.Unlock() + for c := range h.clients { + select { + case c <- b: + default: // a slow client drops an event rather than stalling the audio path + } + } +} + +// ─── Engine: supervises nemo-speech serve and one recording session at a time ─────────────────── + +type engine struct { + cfg config + logs *ring + hub *hub + + mu sync.Mutex + state string // starting | ready | error + stateErr string + nemo *exec.Cmd + recording bool + recorder string + language string + stopSession func() + device string + devs *deviceWatcher + nemoKey string +} + +func (e *engine) setState(s, errText string) { + e.mu.Lock() + e.state, e.stateErr = s, errText + e.mu.Unlock() + e.hub.send(event{"type": "status", "engine": s, "error": errText}) +} + +func (e *engine) status() event { + e.mu.Lock() + defer e.mu.Unlock() + return event{ + "engine": e.state, "error": e.stateErr, "recording": e.recording, "recorder": e.recorder, + "language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device, + "runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model), + } +} + +// startNemo launches the warm speech server. The model loads and warms up once (~20 s on CPU), +// so starting and stopping a recording is instant afterwards. +func (e *engine) startNemo() { + if st, err := os.Stat(e.cfg.Model); err != nil || st.IsDir() { + e.setState("error", "model file not found: "+e.cfg.Model+" (refusing to start: a non-local model would make the runtime download it)") + return + } + if _, err := os.Stat(e.cfg.NemoBin); err != nil { + e.setState("error", "nemo-speech runtime not found: "+e.cfg.NemoBin) + return + } + e.setState("starting", "") + // A fresh random key each start: the speech server answers only this engine, not any other local + // program or web page that finds its port. + cmd := exec.Command(e.cfg.NemoBin, "serve", + "--host", "127.0.0.1", "--port", e.cfg.NemoPort, + "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey) + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} // dies with us + out, _ := cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + if err := cmd.Start(); err != nil { + e.setState("error", "could not start nemo-speech: "+err.Error()) + return + } + e.mu.Lock() + e.nemo = cmd + e.mu.Unlock() + e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort) + go func() { + buf := make([]byte, 4096) + for { + n, err := out.Read(buf) + for _, l := range strings.Split(strings.TrimSpace(string(buf[:n])), "\n") { + if l != "" { + e.logs.add("[nemo] %s", l) + } + } + if err != nil { + return + } + } + }() + go func() { + err := cmd.Wait() + e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err)) + }() + // Ready = the model list answers. + go func() { + url := "http://127.0.0.1:" + e.cfg.NemoPort + "/v1/models" + for i := 0; i < 600; i++ { + time.Sleep(500 * time.Millisecond) + req, _ := http.NewRequest(http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer "+e.nemoKey) + resp, err := http.DefaultClient.Do(req) + if err == nil { + resp.Body.Close() + if resp.StatusCode == 200 { + e.logs.add("nemo-speech ready after %.1fs", float64(i+1)*0.5) + e.setState("ready", "") + return + } + } + e.mu.Lock() + s := e.state + e.mu.Unlock() + if s == "error" { + return + } + } + e.setState("error", "nemo-speech did not become ready within 5 minutes") + }() +} + +// recorderCmds: raw s16le 16 kHz mono PCM on stdout. parec first — on PipeWire/Bluetooth desktops it +// talks to the sound server directly (the old Rudder's hardest-won lesson: PortAudio/ALSA-only +// capture stalls on Bluetooth headsets). +// RUDDER_RECORDER overrides the list with one space-separated command (e.g. a test that replays a +// PCM file), so the engine can be tested end to end without a microphone. +// device is a sound-server source name from /api/rudder/devices; empty = the system default input. +func recorderCmds(device string) [][]string { + if v := os.Getenv("RUDDER_RECORDER"); v != "" { + return [][]string{strings.Fields(v)} + } + parec := []string{"parec", "--format=s16le", "--rate=16000", "--channels=1", "--raw", "--latency-msec=100"} + pw := []string{"pw-record", "--format=s16", "--rate=16000", "--channels=1"} + if device != "" { + parec = append(parec, "--device="+device) + pw = append(pw, "--target="+device) + } + pw = append(pw, "-") + cmds := [][]string{parec, pw} + if device == "" { // arecord can't address a sound-server source by name + cmds = append(cmds, []string{"arecord", "-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"}) + } + return cmds +} + +func (e *engine) start(lang, device string) error { + e.mu.Lock() + if e.state != "ready" { + e.mu.Unlock() + return fmt.Errorf("engine is %s — wait for it to be ready", e.state) + } + if e.recording { + e.mu.Unlock() + return nil + } + if lang == "" { + lang = e.cfg.Language + } + e.mu.Unlock() + + ws, _, err := websocket.DefaultDialer.Dial("ws://127.0.0.1:"+e.cfg.NemoPort+"/v1/audio/transcriptions/realtime", + http.Header{"Authorization": {"Bearer " + e.nemoKey}}) + if err != nil { + return fmt.Errorf("connect to speech server: %w", err) + } + upd, _ := json.Marshal(event{"type": "session.update", "session": event{ + "sample_rate": 16000, "language": lang, "automatic_punctuation": true, "endpointing_ms": e.cfg.EndpointMs, + }}) + if err := ws.WriteMessage(websocket.TextMessage, upd); err != nil { + ws.Close() + return fmt.Errorf("session.update: %w", err) + } + + // Bluetooth headsets start on CVSD, not mSBC. Measured 2026-09-27 on a Bluetooth headset: on mSBC the + // mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after + // switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an + // 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out. + if card := btCard(device); card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" { + if p := currentProfile(card); p == "headset-head-unit-msbc" { + e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)") + e.hub.send(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"}) + if err := switchProfile(card, e.logs); err != nil { + e.logs.add("pre-switch to CVSD failed: %v", err) + } else { + e.hub.send(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."}) + } + } + } + + first, err := spawnRecorder(device, e.logs) + if err != nil { + ws.Close() + return err + } + recName := first.name + var recMu sync.Mutex + cur := first // the live recorder; replaced when a dead mic link is recovered + + ctx, cancel := context.WithCancel(context.Background()) + var writeMu sync.Mutex + done := make(chan struct{}) + + e.mu.Lock() + e.recording, e.recorder, e.language, e.device = true, recName, lang, device + e.stopSession = func() { + cancel() + recMu.Lock() + cur.kill() + recMu.Unlock() + writeMu.Lock() + _ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`)) + writeMu.Unlock() + select { // let the final words arrive + case <-done: + case <-time.After(4 * time.Second): + } + ws.Close() + } + e.mu.Unlock() + e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device))) + go watchKernelBluetooth(ctx, e.logs) + e.hub.send(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device}) + for _, d := range e.devs.current().Devices { + if (d.ID == device || (device == "" && d.Default)) && d.Muted { + e.hub.send(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."}) + } + } + + // Audio pump: 100 ms frames → speech server, plus a level meter and dead-mic RECOVERY. + go func() { + frame := make([]byte, 3200) + var silentSince time.Time + recoveries := 0 + var recovering atomic.Bool + var resetting atomic.Bool + tracePeak, lastTrace := 0, time.Now() + var healthySince time.Time // a mic that stays healthy 30 s earns back its 3 reconnects + gaveUp := false + lastLevel := time.Time{} + for { + recMu.Lock() + r := cur + recMu.Unlock() + if _, err := io.ReadFull(r.out, frame); err != nil { + if ctx.Err() != nil { + return + } + recMu.Lock() + swapped := cur != r + recMu.Unlock() + if swapped { // we killed it ourselves during a recovery: read from the new one + continue + } + // A headset reset restarts PipeWire, which kills parec mid-read. That is expected: + // wait for the reset to hand over a fresh recorder instead of ending the session. + if resetting.Load() { + for resetting.Load() && ctx.Err() == nil { + time.Sleep(200 * time.Millisecond) + } + continue + } + // The recorder died on its own (bad device, sound server unreachable, device unplugged). + // End the session so the UI stops showing "Listening", and say WHY in plain words. + _ = r.cmd.Wait() + why := strings.TrimSpace(r.errBuf.String()) + e.logs.add("recorder %s ended unexpectedly: %v | stderr: %s", r.name, err, why) + msg := "The microphone stopped delivering audio." + switch { + case strings.Contains(why, "Connection refused") || strings.Contains(why, "pa_context_connect"): + msg = "Can't reach the computer's sound system (PipeWire/PulseAudio). Is it running?" + case strings.Contains(why, "No such entity") || strings.Contains(why, "not found"): + msg = "That microphone isn't available any more — it may have been unplugged or disconnected. Pick another one." + } + if why != "" { + msg += " (" + r.name + ": " + why + ")" + } + e.hub.send(event{"type": "error", "message": msg}) + go e.stop() + return + } + peak := 0 + for i := 0; i+1 < len(frame); i += 2 { + v := int(int16(binary.LittleEndian.Uint16(frame[i:]))) + if v < 0 { + v = -v + } + if v > peak { + peak = v + } + } + // ★ DEAD MIC LINK. Exact-zero samples are not quiet speech (a live mic always has a noise + // floor). Measured 2026-09-27 on a Bluetooth headset (HFP, mSBC): four words + // transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1, + // the headset losing the mic side of the link. Re-opening the source restarts the audio + // stream; the speech session and the transcript carry on untouched. + if peak == 0 { + if silentSince.IsZero() { + silentSince = time.Now() + } else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond { + recoveries++ + switch { + case recoveries == 1: + // Step 1: reopen the recorder (cheap; fixes a stalled stream). + e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone") + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"}) + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + case recoveries == 2 && btCard(device) != "": + // Step 2: force a NEW Bluetooth voice link by switching the headset's voice profile to + // CVSD (the old Rudder's proven-stabler choice). Unlike a detect-first reset script this does not + // "detect first", so it can't decide a dead link is fine. + card := btCard(device) + e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card) + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"}) + resetting.Store(true) + go func() { + defer resetting.Store(false) + if err := switchProfile(card, e.logs); err != nil { + e.hub.send(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()}) + return + } + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + }() + case recoveries == 3 && btResetPath() != "": + // Step 3: full headset reset (disconnect → restart PipeWire → reconnect). + e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET") + e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"}) + resetting.Store(true) + go func() { + defer resetting.Store(false) + ok, ran := runHeadsetReset(e.logs) + if !ran { + return + } + if !ok { + e.hub.send(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"}) + } + if next, err := spawnRecorder(device, e.logs); err == nil { + recMu.Lock() + old := cur + cur = next + recMu.Unlock() + old.kill() + recovering.Store(true) + } + }() + default: + gaveUp = true + e.logs.add("mic still dead after reopen + headset resets — giving up for this session") + msg := "Your microphone keeps going silent. This is the Bluetooth headset dropping its mic link (the kernel reports corrupted voice packets), not Rudder. Try a wired or built-in mic, or turn off the headset's noise cancellation / adaptive audio in its app." + if btResetPath() == "" { + msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)" + } + e.hub.send(event{"type": "warning", "message": msg}) + } + silentSince = time.Now() + } + } else { + if recovering.Load() { + recovering.Store(false) + healthySince = time.Now() + e.logs.add("mic recovered after %d reconnect(s)", recoveries) + e.hub.send(event{"type": "recovered", "message": "Microphone is back — keep talking."}) + } + if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second { + recoveries, gaveUp, healthySince = 0, false, time.Time{} + } + silentSince = time.Time{} + } + // Flight recorder: the mic level twice a second, so a drop can be lined up against kernel and + // sound-server events in the same log (/api/rudder/log). + tracePeak = max(tracePeak, peak) + if time.Since(lastTrace) >= 500*time.Millisecond { + e.logs.add("trace level peak=%d", tracePeak) + tracePeak, lastTrace = 0, time.Now() + } + if time.Since(lastLevel) > 150*time.Millisecond { + lastLevel = time.Now() + e.hub.send(event{"type": "level", "peak": float64(peak) / 32768}) + } + writeMu.Lock() + err := ws.WriteMessage(websocket.BinaryMessage, frame) + writeMu.Unlock() + if err != nil { + if ctx.Err() == nil { + e.hub.send(event{"type": "error", "message": "speech server connection lost: " + err.Error()}) + } + return + } + } + }() + + // Result reader: map the server's events onto the plugin's small vocabulary. + go func() { + defer close(done) + for { + _, msg, err := ws.ReadMessage() + if err != nil { + return + } + var m map[string]any + if json.Unmarshal(msg, &m) != nil { + continue + } + t, _ := m["type"].(string) + text := firstString(m, "transcript", "delta", "text") + switch { + case strings.HasSuffix(t, "input_audio_transcription.delta"): + // Deltas APPEND to the utterance in progress; the server sends many empty ones. + if text != "" { + e.hub.send(event{"type": "partial", "text": text}) + } + case strings.HasSuffix(t, "input_audio_transcription.completed"): + e.hub.send(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)}) + case t == "error": + e.logs.add("speech server error: %s", string(msg)) + e.hub.send(event{"type": "error", "message": string(msg)}) + case t == "input_audio_buffer.committed": + return + default: + e.logs.add("speech event: %s", t) + } + } + }() + return nil +} + +// recorderProc is one running recorder process. +type recorderProc struct { + cmd *exec.Cmd + out io.ReadCloser + name string + errBuf *bytes.Buffer +} + +func (r *recorderProc) kill() { + if r != nil && r.cmd != nil && r.cmd.Process != nil { + _ = r.cmd.Process.Kill() + go func() { _ = r.cmd.Wait() }() + } +} + +// spawnRecorder starts the first recorder that works for this device. +func spawnRecorder(device string, logs *ring) (*recorderProc, error) { + for _, c := range recorderCmds(device) { + if _, err := exec.LookPath(c[0]); err != nil { + continue + } + cmd := exec.Command(c[0], c[1:]...) + cmd.Env = audioEnv() + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + out, err := cmd.StdoutPipe() + if err != nil { + continue + } + if err := cmd.Start(); err != nil { + logs.add("recorder %s failed to start: %v", c[0], err) + continue + } + return &recorderProc{cmd: cmd, out: out, name: c[0], errBuf: errBuf}, nil + } + return nil, errors.New("no microphone recorder found (need parec, pw-record or arecord)") +} + +func firstString(m map[string]any, keys ...string) string { + for _, k := range keys { + if s, ok := m[k].(string); ok { + return s + } + } + if item, ok := m["item"].(map[string]any); ok { + return firstString(item, keys...) + } + return "" +} + +func (e *engine) stop() { + e.mu.Lock() + stop := e.stopSession + e.stopSession = nil + was := e.recording + e.recording = false + e.mu.Unlock() + if !was || stop == nil { + return + } + stop() + e.logs.add("recording stopped") + e.hub.send(event{"type": "recording", "recording": false}) +} + +func (e *engine) shutdown() { + e.stop() + e.mu.Lock() + n := e.nemo + e.mu.Unlock() + if n != nil && n.Process != nil { + _ = n.Process.Kill() + } +} + +// ─── HTTP API ──────────────────────────────────────────────────────────────────────────────────── + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +// guard: two protections for a server that streams what you say. +// - Host check (DNS rebinding): a web page on any domain can make the browser resolve its own hostname +// to 127.0.0.1 and then read this API as "same-origin". Only requests addressed to 127.0.0.1 or +// localhost are served. +// - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows) +// may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the +// dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests). +func guard(next http.Handler) http.Handler { + extra := os.Getenv("RUDDER_ALLOW_ORIGIN") + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host := r.Host + if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") { + host = host[:i] + } + if host != "127.0.0.1" && host != "localhost" && host != "[::1]" { + http.Error(w, "forbidden host", http.StatusForbidden) + return + } + o := r.Header.Get("Origin") + if o == "tauri://localhost" || o == "http://tauri.localhost" || (extra != "" && o == extra) { + w.Header().Set("Access-Control-Allow-Origin", o) + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Vary", "Origin") + } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) +} + +func main() { + cfg := loadConfig() + logs := &ring{} + h := &hub{clients: map[chan []byte]struct{}{}} + key := make([]byte, 24) + if _, err := rand.Read(key); err != nil { + log.Fatal("no randomness for the speech-server key: ", err) + } + e := &engine{cfg: cfg, logs: logs, hub: h, state: "starting", devs: &deviceWatcher{hub: h, logs: logs}, nemoKey: hex.EncodeToString(key)} + e.devs.run() + + mux := http.NewServeMux() + mux.HandleFunc("/api/rudder/status", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/start", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, 405, event{"error": "POST only"}) + return + } + var body struct { + Language string `json:"language"` + Device string `json:"device"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if err := e.start(body.Language, body.Device); err != nil { + e.logs.add("start failed: %v", err) + writeJSON(w, 409, event{"error": err.Error()}) + return + } + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) { + e.stop() + writeJSON(w, 200, e.status()) + }) + mux.HandleFunc("/api/rudder/devices", func(w http.ResponseWriter, r *http.Request) { + e.devs.refresh("request") + writeJSON(w, 200, e.devs.current()) + }) + mux.HandleFunc("/api/rudder/test", func(w http.ResponseWriter, r *http.Request) { + var body struct { + Device string `json:"device"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + writeJSON(w, 200, e.micTest(body.Device)) + }) + mux.HandleFunc("/api/rudder/log", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = io.WriteString(w, logs.text()) + }) + mux.HandleFunc("/api/rudder/events", func(w http.ResponseWriter, r *http.Request) { + fl, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", 500) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + c := e.hub.subscribe() + defer e.hub.unsubscribe(c) + first, _ := json.Marshal(event{"type": "status", "status": e.status()}) + fmt.Fprintf(w, "data: %s\n\n", first) + devs, _ := json.Marshal(event{"type": "devices", "devices": e.devs.current()}) + fmt.Fprintf(w, "data: %s\n\n", devs) + fl.Flush() + ping := time.NewTicker(15 * time.Second) + defer ping.Stop() + for { + select { + case <-r.Context().Done(): + return + case b := <-c: + fmt.Fprintf(w, "data: %s\n\n", b) + fl.Flush() + case <-ping.C: + fmt.Fprint(w, ": ping\n\n") + fl.Flush() + } + } + }) + + srv := &http.Server{Addr: "127.0.0.1:" + cfg.Port, Handler: guard(mux)} + logs.add("rudder-engine listening on http://127.0.0.1:%s (runtime=%s model=%s)", cfg.Port, cfg.NemoBin, cfg.Model) + e.startNemo() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sig + e.shutdown() + os.Exit(0) + }() + if err := srv.ListenAndServe(); err != nil { + e.shutdown() + log.Fatal(err) + } +} diff --git a/backend/rudder-engine/setup-runtime.sh b/backend/rudder-engine/setup-runtime.sh new file mode 100755 index 00000000..57c6291b --- /dev/null +++ b/backend/rudder-engine/setup-runtime.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# setup-runtime.sh — fetch Rudder's speech runtime + model, verified by SHA-256, then build the engine. +# +# Rudder (plugin-designs/rudder-voice) transcribes on THIS computer. It needs two things that are too big +# for git: NVIDIA's NeMo-Speech.cpp runtime (~4 MB, CPU build) and the Nemotron 3.5 ASR Streaming GGUF +# (~742 MB). Both are pinned by version and checksum; a mismatch aborts instead of running an unverified +# file. After this, `bun run dev` starts the engine and Rudder appears in the Plugin Studio. +# +# bash backend/rudder-engine/setup-runtime.sh # installs into ~/.local/share/rudder +# RUDDER_HOME=/some/dir bash backend/rudder-engine/setup-runtime.sh +# +# Linux x86_64 only for now (the runtime ships other platforms; add them when a desktop build needs them). +set -euo pipefail + +RUDDER_HOME="${RUDDER_HOME:-$HOME/.local/share/rudder}" +RT_VER="0.1.0" +RT_NAME="nemo-speech-${RT_VER}-linux-x86_64-cpu" +RT_URL="https://github.com/NVIDIA/NeMo-Speech.cpp/releases/download/v${RT_VER}/${RT_NAME}.tar.gz" +RT_SHA="0f74131d631ad2c694cf0ec53490866bb6461147959589a69fb6fc231944065b" +# NVIDIA re-converted the GGUF on 2026-09-10 (embedded tokenizer); this revision is the one Rudder is tested with. +MODEL_REV="ea30d66debe3" +MODEL_FILE="nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" +MODEL_URL="https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${MODEL_REV}/${MODEL_FILE}" +MODEL_SHA="3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1" + +say() { printf '\033[1;34m→\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; } + +[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || die "This script currently supports Linux x86_64 only." +command -v curl >/dev/null || die "curl is required." +command -v sha256sum >/dev/null || die "sha256sum is required." + +fetch() { # url dest sha + local url="$1" dest="$2" sha="$3" + if [ -f "$dest" ] && [ "$(sha256sum "$dest" | cut -d' ' -f1)" = "$sha" ]; then + say "already have $(basename "$dest") (checksum OK)"; return + fi + say "downloading $(basename "$dest")" + curl -fL --retry 3 -C - -o "$dest.part" "$url" + [ "$(sha256sum "$dest.part" | cut -d' ' -f1)" = "$sha" ] || { rm -f "$dest.part"; die "checksum mismatch for $(basename "$dest") — refusing to use it"; } + mv "$dest.part" "$dest" +} + +mkdir -p "$RUDDER_HOME/bin" "$RUDDER_HOME/models" +fetch "$RT_URL" "$RUDDER_HOME/bin/$RT_NAME.tar.gz" "$RT_SHA" +tar -xzf "$RUDDER_HOME/bin/$RT_NAME.tar.gz" -C "$RUDDER_HOME/bin" +fetch "$MODEL_URL" "$RUDDER_HOME/models/$MODEL_FILE" "$MODEL_SHA" + +if command -v go >/dev/null; then + say "building backend/bin/rudder-engine" + (cd "$(dirname "$0")" && go build -o ../bin/rudder-engine .) +else + printf '! Go not found — install Go, then: cd backend/rudder-engine && go build -o ../bin/rudder-engine .\n' +fi + +for tool in parec pactl; do command -v "$tool" >/dev/null || printf '! %s not found — install pipewire-pulse or pulseaudio-utils for microphone capture.\n' "$tool"; done + +say "done. Runtime: $RUDDER_HOME/bin/$RT_NAME Model: $RUDDER_HOME/models/$MODEL_FILE" +say "run \`bun run dev\` and open Rudder (rudder-voice) in the Plugin Studio." diff --git a/internal/backend-dev-plugin.mjs b/internal/backend-dev-plugin.mjs new file mode 100644 index 00000000..46993c63 --- /dev/null +++ b/internal/backend-dev-plugin.mjs @@ -0,0 +1,61 @@ +/** + * internal/backend-dev-plugin.mjs — `bun run dev` starts the local plugin backends. + * + * WHY: some plugins ship to DESKTOP with a local backend (rudder-voice → backend/bin/rudder-engine, + * run as a Tauri sidecar). In the Plugin Studio the same backend must be running, or the plugin can't + * be tested in the browser. This plugin starts each built backend with the dev server, forwards its + * /api// routes, and stops it when the dev server stops. A backend that isn't built is skipped + * with a hint: plugins without a backend never notice this file. + * + * DEV ONLY (`apply: 'serve'`). A production build never starts or bundles a backend. + */ +import { spawn } from 'node:child_process' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' + +// name → how to run it. `route` is proxied to `port` on 127.0.0.1. +export const DEV_BACKENDS = [ + { + name: 'rudder-engine', + bin: 'backend/bin/rudder-engine', + build: 'cd backend/rudder-engine && go build -o ../bin/rudder-engine .', + route: '/api/rudder', + port: Number(process.env.RUDDER_PORT || 5193), + env: { RUDDER_PORT: String(process.env.RUDDER_PORT || 5193), RUDDER_NEMO_PORT: String(process.env.RUDDER_NEMO_PORT || 5194) }, + }, +] + +// The proxy table for vite's `server.proxy`. SSE (`/events`) streams through unbuffered. +export function devBackendProxy() { + return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false }])) +} + +export function backendDevPlugin(root) { + const children = [] + return { + name: 'dx-backend-dev', + apply: 'serve', + configureServer(server) { + for (const b of DEV_BACKENDS) { + const bin = resolve(root, b.bin) + if (!existsSync(bin)) { + server.config.logger.warn(`[backend] ${b.name} not built — its plugin will show "engine offline". Build: ${b.build}`) + continue + } + const child = spawn(bin, [], { cwd: root, env: { ...process.env, ...b.env }, stdio: ['ignore', 'pipe', 'pipe'] }) + const tag = `[${b.name}]` + const out = (d) => String(d).trim().split('\n').forEach((l) => l && server.config.logger.info(`${tag} ${l}`)) + child.stdout.on('data', out) + child.stderr.on('data', out) + child.on('exit', (code) => server.config.logger.warn(`${tag} exited (${code})`)) + children.push(child) + server.config.logger.info(`${tag} started → ${b.route} on 127.0.0.1:${b.port}`) + } + const stop = () => children.forEach((c) => { try { c.kill('SIGTERM') } catch {} }) + server.httpServer?.on('close', stop) + process.once('exit', stop) + process.once('SIGINT', () => { stop(); process.exit(0) }) + process.once('SIGTERM', () => { stop(); process.exit(0) }) + }, + } +} diff --git a/plugin-designs/rudder-voice/README.md b/plugin-designs/rudder-voice/README.md new file mode 100644 index 00000000..06c50fe9 --- /dev/null +++ b/plugin-designs/rudder-voice/README.md @@ -0,0 +1,55 @@ +# rudder-voice — Rudder, private voice-to-text (desktop target) + +**Rudder is a plugin that isn't a map.** It shows that the same dx-maps pattern — one folder in +`plugin-designs/`, developed live in the Plugin Studio — can ship a **desktop app with its own local +backend**, not only embeddable maps. It is also a genuinely useful tool for the people who build here: +talk to your agents instead of typing, with nothing leaving your computer. + +- **Streaming.** Words appear in the text box while you speak; talk for as long as you like. +- **Type and talk at once.** Type, paste links or code anywhere in the box; everything you say is + added to the end. While you're typing, Rudder holds its words and adds them when you pause. +- **CPU only.** No graphics card. About 1 GB of memory; keeps up with normal speech on a laptop CPU. +- **Offline, no telemetry.** NVIDIA Nemotron 3.5 ASR Streaming (0.6B) run by NVIDIA's open-source + NeMo-Speech.cpp (ggml). Traced with `strace` while transcribing: zero sockets, zero DNS lookups. +- **Microphones like a video-call app.** Pick the input, hot-plug updates, a 3-second mic test, and + plain-language errors (muted, disconnected, dead Bluetooth mic link — which it tries to recover). + +## Run it in the Plugin Studio + +```bash +bash backend/rudder-engine/setup-runtime.sh # once: runtime + model (verified by SHA-256) + engine build +bun run dev # starts the engine too; open "rudder-voice" in the studio +``` + +Linux with PipeWire or PulseAudio (`parec`, `pactl`). The engine listens on `127.0.0.1` only. + +## How it fits together + +``` + (this folder) ──HTTP + SSE /api/rudder──► backend/rudder-engine (Go) + ├─ parec / pw-record (mic, open only while recording) + └─ WebSocket ► nemo-speech serve (127.0.0.1, model stays loaded) +``` + +`bun run dev` starts `backend/bin/rudder-engine` through `internal/backend-dev-plugin.mjs` and forwards +`/api/rudder` to it, so the plugin calls the same API in the browser as it will inside the desktop app, +where Tauri runs the engine as a sidecar and passes its address in `profile-config` (`apiBase`). + +`rudder-voice.meta.json` declares `"targets": ["desktop"]`: on a plain website there is no local engine, +so this plugin is not meant for web embeds. + +## Settings (environment variables) + +| Variable | Default | What it does | +|---|---|---| +| `RUDDER_PORT` | `5193` | engine HTTP port (loopback) | +| `RUDDER_NEMO_PORT` | `5194` | speech server port (loopback) | +| `RUDDER_MODEL` / `RUDDER_NEMO_BIN` | `~/.local/share/rudder/...` | model and runtime paths; the engine refuses a model that isn't a local file | +| `RUDDER_LANG` | `en-US` | default language (the model supports ~30) | +| `RUDDER_KEEP_MSBC` | unset | keep Bluetooth headsets on mSBC instead of switching them to the more stable CVSD | +| `RUDDER_BT_RESET` | unset | optional command that fully resets a Bluetooth headset, used when its mic link dies | + +## Not yet + +- Desktop build (`desktop:build`) with the engine as a Tauri sidecar. +- A saved word-correction table (PocketBase) for technical terms the model mishears. diff --git a/plugin-designs/rudder-voice/index.html b/plugin-designs/rudder-voice/index.html new file mode 100644 index 00000000..82187f90 --- /dev/null +++ b/plugin-designs/rudder-voice/index.html @@ -0,0 +1,32 @@ + + + + + + doxa-maps · rudder-voice staging + + + + +
+

rudder-voice · staging

+

Desktop-target plugin: private, offline, streaming voice-to-text. The speech model runs in the + local backend backend/rudder-engine (NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR), started by + bun run dev and reached at /api/rudder.

+
+ +
+
+ + + diff --git a/plugin-designs/rudder-voice/index.js b/plugin-designs/rudder-voice/index.js new file mode 100644 index 00000000..579c5acc --- /dev/null +++ b/plugin-designs/rudder-voice/index.js @@ -0,0 +1,23 @@ +/** + * rudder-voice / index.js — Bundle entry (Capability-1 single element, DESKTOP target). + * + * — private, offline, streaming voice-to-text. The UI is this bundle; the speech model + * runs in the local backend (backend/rudder-engine), started by `bun run dev` in the Plugin Studio and + * by Tauri as a sidecar in the desktop app. See rudder-voice.vue for the contract. + * + * Desktop-only: rudder-voice.meta.json declares "targets": ["desktop"]. On a plain website there is no + * local engine, so the element shows "Engine offline" instead of pretending to work. + * + * Usage (desktop shell / Plugin Studio): + * + */ +import { defineCustomElement } from 'vue' +import RudderVoice from './rudder-voice.vue' + +const RudderVoiceElement = defineCustomElement(RudderVoice) + +if (typeof customElements !== 'undefined') { + if (!customElements.get('rudder-voice')) customElements.define('rudder-voice', RudderVoiceElement) +} + +export default RudderVoiceElement diff --git a/plugin-designs/rudder-voice/rudder-voice.meta.json b/plugin-designs/rudder-voice/rudder-voice.meta.json new file mode 100644 index 00000000..316e368c --- /dev/null +++ b/plugin-designs/rudder-voice/rudder-voice.meta.json @@ -0,0 +1,7 @@ +{ + "_comment": "Rudder — private, offline, streaming voice-to-text. DESKTOP target: the speech model runs in the local backend (backend/rudder-engine: NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF), started by `bun run dev` and run by Tauri as a sidecar. Not for public web embeds: a website has no local engine.", + "type": "single", + "label": "Rudder — voice to text (desktop)", + "targets": ["desktop"], + "backend": "backend/rudder-engine" +} diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue new file mode 100644 index 00000000..2b2ff3ee --- /dev/null +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -0,0 +1,650 @@ + + + + + diff --git a/vite.config.js b/vite.config.js index e58edb14..8f1cfac7 100644 --- a/vite.config.js +++ b/vite.config.js @@ -24,6 +24,7 @@ import vue from '@vitejs/plugin-vue' import cssInjectedByJsPlugin from 'vite-plugin-css-injected-by-js' import { autoSpaDevPlugin } from './internal/plugin-gallery/dev-plugin.mjs' import { stripShadowCssCommentsPlugin } from './internal/vite-plugin-strip-shadow-css-comments.mjs' +import { backendDevPlugin, devBackendProxy } from './internal/backend-dev-plugin.mjs' import { fileURLToPath } from 'node:url' import { resolve, dirname } from 'node:path' import { readdirSync, statSync, existsSync } from 'node:fs' @@ -118,7 +119,9 @@ export default defineConfig(({ command, mode }) => { // The dev server listens on THIS MACHINE ONLY. A reachable one serves the source tree, node_modules // and the token baked into /; a remote viewer reaches it through an ssh tunnel (ssh -L), never a public // port. `--host` on the command line still overrides this for a deliberate LAN session. - server: { host: '127.0.0.1' }, + // Local plugin backends (backend/, e.g. rudder-engine) are started by backendDevPlugin and reached + // through these same-origin routes, so a plugin calls /api//… in dev exactly as on desktop. + server: { host: '127.0.0.1', proxy: devBackendProxy() }, define: { // ★ NO 'unsafe-eval' (CRITIC-4, 2026-09-26). vue-i18n's default message compiler builds each message // with `new Function`, so a partner with a strict Content-Security-Policy saw every translated string @@ -159,6 +162,7 @@ export default defineConfig(({ command, mode }) => { vue({ customElement: true }), cssInjectedByJsPlugin(), autoSpaDevPlugin(env.VITE_MAPBOX_TOKEN || process.env.VITE_MAPBOX_TOKEN || ''), + backendDevPlugin(__dirname), ], resolve: { alias: { '@map': resolve(__dirname, 'library') },