Add Rudder (rudder-voice): offline streaming voice-to-text, a desktop-target plugin
Some checks are pending
Deploy to GitHub Pages / build (push) Waiting to run
Deploy to GitHub Pages / deploy (push) Blocked by required conditions

Rudder is the first plugin that isn't a map: it shows that one plugin-designs/ folder, developed live
in the Plugin Studio, can ship as a desktop app with its own local backend.

- plugin-designs/rudder-voice: <rudder-voice> element. Words stream into the end of a text area while
  you type/paste anywhere (typing holds the stream; nothing duplicated or lost). Mic picker with
  hot-plug, 3-second mic test, plain-language errors, info panel, privacy-safe debug report.
  meta.json declares "targets": ["desktop"].
- backend/rudder-engine (Go): supervises NVIDIA NeMo-Speech.cpp (ggml, CPU) with the Nemotron 3.5 ASR
  Streaming GGUF over a loopback WebSocket; records with parec/pw-record only while recording; recovers
  dead Bluetooth mic links (reopen -> CVSD voice profile -> optional RUDDER_BT_RESET) and starts
  headsets on CVSD. No telemetry: traced with strace, zero sockets/DNS while transcribing.
- Privacy: log lines redact MAC addresses and the home folder, transcripts are never logged, Host check
  against DNS rebinding, CORS only for the desktop webview, random API key for the speech server.
- internal/backend-dev-plugin.mjs + vite.config.js: `bun run dev` starts built plugin backends and
  proxies /api/<name> same-origin, so a plugin calls the same API in the studio and on desktop.
- backend/rudder-engine/setup-runtime.sh: fetches the runtime and model pinned by SHA-256.

Not yet: the Tauri desktop build (sidecar) and the PocketBase word-correction table.
This commit is contained in:
Doxa Maps 2026-09-27 01:48:38 -05:00
parent f294159b64
commit 4d6aeb2949
13 changed files with 2155 additions and 1 deletions

3
.gitignore vendored
View file

@ -145,3 +145,6 @@ data-private/
/public/assets/
/public/__gallery/
/public/index.html
# Local plugin backends (backend/<name>/) — the compiled binaries are rebuilt from source, never committed.
/backend/bin/

View file

@ -0,0 +1,456 @@
// devices.go — microphone discovery, hot-plug, and the sound-server environment.
//
// The Jitsi Meet pattern (lib-jitsi-meet modules/RTC/RTCUtils.js), done natively because Rudder records
// outside the browser:
//
// Jitsi (browser) Rudder (native, PipeWire/PulseAudio)
// navigator.mediaDevices.enumerateDevices() → pactl -f json list sources
// 'devicechange' event + polling fallback → pactl subscribe (+ re-list on every source/card event)
// compareAvailableMediaDevices(new) → sameDevices(old, new) — only emit on a real change
// getUserMedia({audio:{deviceId:{exact:id}}}) → parec --device=<id>
//
// Jitsi also reports the device list to analytics (sendDeviceListToAnalytics). Rudder never does:
// the list goes to the local plugin over SSE and nowhere else.
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"math"
"os"
"os/exec"
"strings"
"sync"
"syscall"
"time"
)
// audioEnv gives child processes a path to the user's sound server even when the engine was launched
// from a context without a desktop session (a detached shell, a service, an agent's terminal). Measured
// 2026-09-27: started with no XDG_RUNTIME_DIR, parec exited instantly with "Connection refused" and every
// recording captured nothing, although the socket sat at /run/user/<uid>/pulse/native all along.
func audioEnv() []string {
env := os.Environ()
rt := os.Getenv("XDG_RUNTIME_DIR")
if rt == "" {
rt = fmt.Sprintf("/run/user/%d", os.Getuid())
if st, err := os.Stat(rt); err == nil && st.IsDir() {
env = append(env, "XDG_RUNTIME_DIR="+rt)
}
}
// systemctl --user (a headset-reset command may restart PipeWire) needs the session bus.
if os.Getenv("DBUS_SESSION_BUS_ADDRESS") == "" {
if _, err := os.Stat(rt + "/bus"); err == nil {
env = append(env, "DBUS_SESSION_BUS_ADDRESS=unix:path="+rt+"/bus")
}
}
if os.Getenv("PULSE_SERVER") == "" {
if _, err := os.Stat(rt + "/pulse/native"); err == nil {
env = append(env, "PULSE_SERVER=unix:"+rt+"/pulse/native")
}
}
return env
}
// Device is one microphone as the plugin shows it.
type Device struct {
ID string `json:"id"` // the sound server's source name: what parec --device takes
Label string `json:"label"` // human name ("My Headset", "Built-in Audio Analog Stereo")
Bus string `json:"bus"` // bluetooth | usb | pci | ...
Default bool `json:"default"` // the system's default input
Muted bool `json:"muted"`
Volume string `json:"volume"` // e.g. "52%"
State string `json:"state"` // RUNNING | IDLE | SUSPENDED
}
// DeviceList is the whole picture: the mics, plus sound cards that have a mic but are switched off
// (the laptop's built-in card, profile "off", is the common case — its mic simply doesn't exist until
// the card is turned on in Sound settings).
type DeviceList struct {
Devices []Device `json:"devices"`
Default string `json:"default"`
OffCards []string `json:"offCards"`
Error string `json:"error,omitempty"`
}
func runPactl(args ...string) ([]byte, error) {
cmd := exec.Command("pactl", args...)
cmd.Env = audioEnv()
return cmd.Output()
}
func listDevices() DeviceList {
out := DeviceList{Devices: []Device{}, OffCards: []string{}}
if _, err := exec.LookPath("pactl"); err != nil {
out.Error = "pactl not found: install pipewire-pulse or pulseaudio-utils to list microphones"
return out
}
if b, err := runPactl("get-default-source"); err == nil {
out.Default = strings.TrimSpace(string(b))
}
b, err := runPactl("-f", "json", "list", "sources")
if err != nil {
out.Error = "cannot reach the sound server (pactl list sources failed: " + err.Error() + ")"
return out
}
var raw []struct {
Name string `json:"name"`
Description string `json:"description"`
State string `json:"state"`
Mute bool `json:"mute"`
Volume map[string]any `json:"volume"`
Properties map[string]any `json:"properties"`
}
if err := json.Unmarshal(b, &raw); err != nil {
out.Error = "unexpected pactl output: " + err.Error()
return out
}
for _, s := range raw {
// A ".monitor" source is the loopback of a SPEAKER, not a microphone: listing it would let the
// user "record" their own music. Hidden, as Jitsi lists only audioinput kinds.
if strings.HasSuffix(s.Name, ".monitor") || s.Properties["device.class"] == "monitor" {
continue
}
d := Device{ID: s.Name, Label: s.Description, State: s.State, Muted: s.Mute, Default: s.Name == out.Default}
if bus, ok := s.Properties["device.bus"].(string); ok {
d.Bus = bus
}
for _, ch := range s.Volume { // first channel is representative
if m, ok := ch.(map[string]any); ok {
if p, ok := m["value_percent"].(string); ok {
d.Volume = p
break
}
}
}
out.Devices = append(out.Devices, d)
}
// Cards whose active profile is "off" but which offer an input profile.
if b, err := runPactl("-f", "json", "list", "cards"); err == nil {
var cards []struct {
Name string `json:"name"`
ActiveProfile string `json:"active_profile"`
Properties map[string]any `json:"properties"`
Profiles map[string]struct {
Sources int `json:"sources"`
Available bool `json:"available"`
} `json:"profiles"`
}
if json.Unmarshal(b, &cards) == nil {
for _, c := range cards {
if c.ActiveProfile != "off" {
continue
}
for _, p := range c.Profiles {
if p.Sources > 0 && p.Available {
label, _ := c.Properties["device.description"].(string)
if label == "" {
label = c.Name
}
out.OffCards = append(out.OffCards, label)
break
}
}
}
}
}
return out
}
// sameDevices mirrors Jitsi's compareAvailableMediaDevices: emit only when something a user would
// notice changed (a mic came, went, was muted, or the default moved), not on every state flicker.
func sameDevices(a, b DeviceList) bool {
if a.Default != b.Default || len(a.Devices) != len(b.Devices) || a.Error != b.Error || len(a.OffCards) != len(b.OffCards) {
return false
}
for i := range a.Devices {
x, y := a.Devices[i], b.Devices[i]
if x.ID != y.ID || x.Label != y.Label || x.Muted != y.Muted || x.Volume != y.Volume {
return false
}
}
return true
}
// deviceWatcher keeps the current list and pushes a "devices" event on every real change. It listens
// to `pactl subscribe` (the native 'devicechange') and, like Jitsi, also polls as a fallback in case
// the subscription dies.
type deviceWatcher struct {
mu sync.Mutex
last DeviceList
hub *hub
logs *ring
}
func (w *deviceWatcher) current() DeviceList {
w.mu.Lock()
defer w.mu.Unlock()
return w.last
}
func (w *deviceWatcher) refresh(reason string) {
now := listDevices()
w.mu.Lock()
changed := !sameDevices(w.last, now)
w.last = now
w.mu.Unlock()
if changed {
w.logs.add("microphones (%s): %d found, default=%s %s", reason, len(now.Devices), now.Default, now.Error)
w.hub.send(event{"type": "devices", "devices": now})
}
}
func (w *deviceWatcher) run() {
w.refresh("startup")
// Poll fallback (Jitsi polls enumerateDevices when devicechange is unavailable).
go func() {
for range time.Tick(5 * time.Second) {
w.refresh("poll")
}
}()
go func() {
for {
cmd := exec.Command("pactl", "subscribe")
cmd.Env = audioEnv()
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
out, err := cmd.StdoutPipe()
if err == nil && cmd.Start() == nil {
sc := bufio.NewScanner(out)
var debounce *time.Timer
for sc.Scan() {
l := sc.Text()
if strings.Contains(l, " card ") || (strings.Contains(l, " source ") && !strings.Contains(l, "source-output")) {
w.logs.add("sound-server event: %s", l)
}
if strings.Contains(l, " source ") || strings.Contains(l, " card ") || strings.Contains(l, " server ") {
if debounce != nil {
debounce.Stop()
}
debounce = time.AfterFunc(300*time.Millisecond, func() { w.refresh("hot-plug") })
}
}
_ = cmd.Wait()
}
time.Sleep(3 * time.Second) // sound server restarted or unreachable: retry
}
}()
}
// micTest records ~3 s from one device WITHOUT transcribing and says, in plain words, whether the mic
// works — Rudder's version of Jitsi's pre-join audio preview. Level events stream to the plugin while
// it runs so the meter moves.
func (e *engine) micTest(device string) event {
cmds := recorderCmds(device)
var cmd *exec.Cmd
for _, c := range cmds {
if _, err := exec.LookPath(c[0]); err == nil {
cmd = exec.Command(c[0], c[1:]...)
break
}
}
if cmd == nil {
return event{"ok": false, "verdict": "No recorder found (need parec, pw-record or arecord)."}
}
cmd.Env = audioEnv()
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
var errBuf strings.Builder
cmd.Stderr = &errBuf
out, err := cmd.StdoutPipe()
if err != nil || cmd.Start() != nil {
return event{"ok": false, "verdict": "Could not start the recorder."}
}
defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
frame := make([]byte, 3200) // 100 ms
peak, frames, zeroFrames := 0, 0, 0
var sumSq float64
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
n, err := io.ReadFull(out, frame)
if err != nil {
why := strings.TrimSpace(errBuf.String())
e.logs.add("mic test on %q failed after %d frames: %v | %s", device, frames, err, why)
return event{"ok": false, "verdict": "The microphone produced no audio. " + why, "recorder": cmd.Path}
}
fp := 0
for i := 0; i+1 < n; i += 2 {
v := int(int16(uint16(frame[i]) | uint16(frame[i+1])<<8))
sumSq += float64(v * v)
if v < 0 {
v = -v
}
if v > fp {
fp = v
}
}
if fp == 0 {
zeroFrames++
}
if fp > peak {
peak = fp
}
frames++
e.hub.send(event{"type": "level", "peak": float64(fp) / 32768, "test": true})
}
rms := 0.0
if frames > 0 {
rms = math.Sqrt(sumSq/float64(frames*1600)) / 32768
}
res := event{"ok": true, "peak": float64(peak) / 32768, "rms": rms, "frames": frames}
switch {
case zeroFrames == frames:
res["ok"] = false
res["verdict"] = "Pure silence — the mic is connected but sends nothing. It may be muted, or a Bluetooth headset's mic link may have dropped (try reconnecting the headset or pick another mic)."
case peak < 330: // below about -40 dBFS
res["verdict"] = "Very quiet — Rudder can hear the mic, but barely. Speak closer or raise its input volume."
default:
res["verdict"] = "Working — Rudder can hear you."
}
e.logs.add("mic test on %q: peak=%d rms=%.4f frames=%d verdict=%s", device, peak, rms, frames, res["verdict"])
return res
}
// ─── Optional Bluetooth headset reset ─────────────────────────────────────────────────────────────
//
// Some Bluetooth headsets drop the mic side of the HFP link (exact-zero audio while everything looks
// connected; the kernel logs `hci0: corrupted SCO packet`). Re-opening the recorder does not revive
// it; a full reset (disconnect → restart PipeWire/WirePlumber → reconnect) can. That is machine-specific,
// so Rudder only runs one if RUDDER_BT_RESET names an executable that does it (exit 0 = headset works
// again). Rate-limited: never more than once per 20 s.
func btResetPath() string {
p := os.Getenv("RUDDER_BT_RESET")
if p == "" {
return ""
}
if st, err := os.Stat(p); err == nil && !st.IsDir() && st.Mode()&0o111 != 0 {
return p
}
return ""
}
var (
resetMu sync.Mutex
lastReset time.Time
)
// runHeadsetReset runs the RUDDER_BT_RESET command and reports whether it said the headset works again.
func runHeadsetReset(logs *ring) (ok bool, ran bool) {
path := btResetPath()
if path == "" {
return false, false
}
resetMu.Lock()
if time.Since(lastReset) < 20*time.Second {
resetMu.Unlock()
logs.add("headset reset skipped: last one was %.0fs ago (rate limit 20s)", time.Since(lastReset).Seconds())
return false, false
}
lastReset = time.Now()
resetMu.Unlock()
logs.add("running %s (headset reset + CVSD)", path)
cmd := exec.Command(path)
cmd.Env = audioEnv()
done := make(chan struct{})
var out []byte
var err error
go func() { out, err = cmd.CombinedOutput(); close(done) }()
select {
case <-done:
case <-time.After(60 * time.Second):
_ = cmd.Process.Kill()
logs.add("headset reset timed out after 60s")
return false, true
}
for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if l != "" {
logs.add("[headset reset] %s", l)
}
}
if err != nil {
logs.add("headset reset failed: %v", err)
return false, true
}
return true, true
}
// ─── Bluetooth voice-profile helpers ────────────────────────────────────────────────────────────
// btCard maps a Bluetooth mic (bluez_input.<MAC>.<n>, or "" = the default input) to its card.
func btCard(device string) string {
if device == "" {
if b, err := runPactl("get-default-source"); err == nil {
device = strings.TrimSpace(string(b))
}
}
if !strings.HasPrefix(device, "bluez_input.") {
return ""
}
mac := strings.TrimPrefix(device, "bluez_input.")
if i := strings.LastIndex(mac, "."); i > 0 {
mac = mac[:i]
}
return "bluez_card." + mac
}
func currentProfile(card string) string {
if card == "" {
return "(not bluetooth)"
}
b, err := runPactl("-f", "json", "list", "cards")
if err != nil {
return "?"
}
var cards []struct {
Name string `json:"name"`
ActiveProfile string `json:"active_profile"`
}
if json.Unmarshal(b, &cards) != nil {
return "?"
}
for _, c := range cards {
if c.Name == card {
return c.ActiveProfile
}
}
return "?"
}
// switchProfile forces a fresh voice link: CVSD if the card is on anything else, otherwise a bounce
// through mSBC and back to CVSD (a same-profile set is a no-op and would not renegotiate the link).
func switchProfile(card string, logs *ring) error {
before := currentProfile(card)
target := "headset-head-unit-cvsd"
if before == target {
if _, err := runPactl("set-card-profile", card, "headset-head-unit-msbc"); err != nil {
logs.add("profile bounce via msbc failed: %v", err)
}
time.Sleep(700 * time.Millisecond)
}
if out, err := runPactl("set-card-profile", card, target); err != nil {
return fmt.Errorf("pactl set-card-profile %s %s: %v %s", card, target, err, strings.TrimSpace(string(out)))
}
time.Sleep(1500 * time.Millisecond) // let the new SCO link and the source come up
logs.add("voice profile %s → %s", before, currentProfile(card))
return nil
}
// watchKernelBluetooth copies kernel Bluetooth messages into the log while a session runs
// (`journalctl -k` is readable without root on this system).
func watchKernelBluetooth(ctx context.Context, logs *ring) {
cmd := exec.CommandContext(ctx, "journalctl", "-k", "-f", "-n", "0", "-o", "cat")
out, err := cmd.StdoutPipe()
if err != nil || cmd.Start() != nil {
return
}
sc := bufio.NewScanner(out)
for sc.Scan() {
if l := sc.Text(); strings.Contains(strings.ToLower(l), "bluetooth") {
logs.add("kernel: %s", l)
}
}
_ = cmd.Wait()
}

View file

@ -0,0 +1,5 @@
module dx-maps/backend/rudder-engine
go 1.22.2
require github.com/gorilla/websocket v1.5.3

View file

@ -0,0 +1,2 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

View file

@ -0,0 +1,797 @@
// rudder-engine — the local speech backend for the rudder-voice plugin.
//
// WHY THIS EXISTS: rudder-voice is a DESKTOP-target plugin. Speech-to-text runs on the user's own
// machine, never in the browser and never on a server. In dev, `bun run dev` starts this process and
// Vite forwards /api/rudder/* to it; in the desktop app, Tauri runs it as a sidecar. Same binary,
// same API, so "works in the Plugin Studio" means "works on desktop".
//
// browser / webview ──HTTP+SSE──► rudder-engine (127.0.0.1:RUDDER_PORT)
// ├─ recorder: parec → pw-record → arecord (mic open ONLY while recording)
// └─ WebSocket ──► nemo-speech serve (127.0.0.1:RUDDER_NEMO_PORT)
// NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF
//
// NO TELEMETRY, NO NETWORK: the runtime is NVIDIA NeMo-Speech.cpp (ggml). ONNX-based runtimes were
// REJECTED: measured 2026-09-26, the tested ONNX GenAI runtime looked up a vendor telemetry host even
// with telemetry "disabled". NeMo-Speech.cpp was traced with strace: zero sockets, zero DNS.
// Its only network path is an optional model download that shells out to `curl` when the model is not
// a local file — so this engine REFUSES to start unless RUDDER_MODEL is an existing local file.
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"os"
"os/exec"
"os/signal"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/gorilla/websocket"
)
// ─── Config (env, with local defaults) ──────────────────────────────────────────────────────────
type config struct {
Port string // this engine's HTTP port
NemoPort string // the warm NeMo-Speech server's port (loopback only)
NemoBin string // path to the nemo-speech executable
Model string // local GGUF path — MUST exist (see the no-network note above)
Language string // default language code
EndpointMs int // silence that ends an utterance
}
func envOr(k, d string) string {
if v := os.Getenv(k); v != "" {
return v
}
return d
}
func loadConfig() config {
home, _ := os.UserHomeDir()
base := filepath.Join(home, ".local", "share", "rudder")
return config{
Port: envOr("RUDDER_PORT", "5193"),
NemoPort: envOr("RUDDER_NEMO_PORT", "5194"),
NemoBin: envOr("RUDDER_NEMO_BIN", filepath.Join(base, "bin", "nemo-speech-0.1.0-linux-x86_64-cpu", "bin", "nemo-speech")),
Model: envOr("RUDDER_MODEL", filepath.Join(base, "models", "nemotron-3.5-asr-streaming-0.6b.q8_0.gguf")),
Language: envOr("RUDDER_LANG", "en-US"),
EndpointMs: 800,
}
}
// ─── Log ring: every line is also served at /api/rudder/log so the Debug panel can copy it ──────
type ring struct {
mu sync.Mutex
lines []string
}
// ★ PRIVACY: the log is what people paste to an agent (🐞 Debug → copy report), so it must never carry
// anything that identifies them. Every line is redacted as it is written: device MAC addresses (inside
// Bluetooth device names like bluez_input.AA_BB_…) and the home folder (which contains the username).
// Transcribed words are never logged at all.
var (
macRe = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:_]){5}[0-9a-f]{2}\b`)
homeDir = func() string { h, _ := os.UserHomeDir(); return h }()
)
func redact(s string) string {
s = macRe.ReplaceAllString(s, "XX:XX:XX:XX:XX:XX")
if homeDir != "" && homeDir != "/" {
s = strings.ReplaceAll(s, homeDir, "~")
}
return s
}
func (r *ring) add(format string, a ...any) {
line := redact(time.Now().Format("15:04:05.000") + " " + fmt.Sprintf(format, a...))
log.Println(line)
r.mu.Lock()
r.lines = append(r.lines, line)
if len(r.lines) > 3000 {
r.lines = r.lines[len(r.lines)-3000:]
}
r.mu.Unlock()
}
func (r *ring) text() string {
r.mu.Lock()
defer r.mu.Unlock()
return strings.Join(r.lines, "\n")
}
// ─── Event hub: Server-Sent Events to every open plugin ─────────────────────────────────────────
type event map[string]any
type hub struct {
mu sync.Mutex
clients map[chan []byte]struct{}
}
func (h *hub) subscribe() chan []byte {
c := make(chan []byte, 64)
h.mu.Lock()
h.clients[c] = struct{}{}
h.mu.Unlock()
return c
}
func (h *hub) unsubscribe(c chan []byte) {
h.mu.Lock()
delete(h.clients, c)
h.mu.Unlock()
}
func (h *hub) send(e event) {
b, _ := json.Marshal(e)
h.mu.Lock()
defer h.mu.Unlock()
for c := range h.clients {
select {
case c <- b:
default: // a slow client drops an event rather than stalling the audio path
}
}
}
// ─── Engine: supervises nemo-speech serve and one recording session at a time ───────────────────
type engine struct {
cfg config
logs *ring
hub *hub
mu sync.Mutex
state string // starting | ready | error
stateErr string
nemo *exec.Cmd
recording bool
recorder string
language string
stopSession func()
device string
devs *deviceWatcher
nemoKey string
}
func (e *engine) setState(s, errText string) {
e.mu.Lock()
e.state, e.stateErr = s, errText
e.mu.Unlock()
e.hub.send(event{"type": "status", "engine": s, "error": errText})
}
func (e *engine) status() event {
e.mu.Lock()
defer e.mu.Unlock()
return event{
"engine": e.state, "error": e.stateErr, "recording": e.recording, "recorder": e.recorder,
"language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device,
"runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model),
}
}
// startNemo launches the warm speech server. The model loads and warms up once (~20 s on CPU),
// so starting and stopping a recording is instant afterwards.
func (e *engine) startNemo() {
if st, err := os.Stat(e.cfg.Model); err != nil || st.IsDir() {
e.setState("error", "model file not found: "+e.cfg.Model+" (refusing to start: a non-local model would make the runtime download it)")
return
}
if _, err := os.Stat(e.cfg.NemoBin); err != nil {
e.setState("error", "nemo-speech runtime not found: "+e.cfg.NemoBin)
return
}
e.setState("starting", "")
// A fresh random key each start: the speech server answers only this engine, not any other local
// program or web page that finds its port.
cmd := exec.Command(e.cfg.NemoBin, "serve",
"--host", "127.0.0.1", "--port", e.cfg.NemoPort,
"--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey)
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} // dies with us
out, _ := cmd.StdoutPipe()
cmd.Stderr = cmd.Stdout
if err := cmd.Start(); err != nil {
e.setState("error", "could not start nemo-speech: "+err.Error())
return
}
e.mu.Lock()
e.nemo = cmd
e.mu.Unlock()
e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort)
go func() {
buf := make([]byte, 4096)
for {
n, err := out.Read(buf)
for _, l := range strings.Split(strings.TrimSpace(string(buf[:n])), "\n") {
if l != "" {
e.logs.add("[nemo] %s", l)
}
}
if err != nil {
return
}
}
}()
go func() {
err := cmd.Wait()
e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err))
}()
// Ready = the model list answers.
go func() {
url := "http://127.0.0.1:" + e.cfg.NemoPort + "/v1/models"
for i := 0; i < 600; i++ {
time.Sleep(500 * time.Millisecond)
req, _ := http.NewRequest(http.MethodGet, url, nil)
req.Header.Set("Authorization", "Bearer "+e.nemoKey)
resp, err := http.DefaultClient.Do(req)
if err == nil {
resp.Body.Close()
if resp.StatusCode == 200 {
e.logs.add("nemo-speech ready after %.1fs", float64(i+1)*0.5)
e.setState("ready", "")
return
}
}
e.mu.Lock()
s := e.state
e.mu.Unlock()
if s == "error" {
return
}
}
e.setState("error", "nemo-speech did not become ready within 5 minutes")
}()
}
// recorderCmds: raw s16le 16 kHz mono PCM on stdout. parec first — on PipeWire/Bluetooth desktops it
// talks to the sound server directly (the old Rudder's hardest-won lesson: PortAudio/ALSA-only
// capture stalls on Bluetooth headsets).
// RUDDER_RECORDER overrides the list with one space-separated command (e.g. a test that replays a
// PCM file), so the engine can be tested end to end without a microphone.
// device is a sound-server source name from /api/rudder/devices; empty = the system default input.
func recorderCmds(device string) [][]string {
if v := os.Getenv("RUDDER_RECORDER"); v != "" {
return [][]string{strings.Fields(v)}
}
parec := []string{"parec", "--format=s16le", "--rate=16000", "--channels=1", "--raw", "--latency-msec=100"}
pw := []string{"pw-record", "--format=s16", "--rate=16000", "--channels=1"}
if device != "" {
parec = append(parec, "--device="+device)
pw = append(pw, "--target="+device)
}
pw = append(pw, "-")
cmds := [][]string{parec, pw}
if device == "" { // arecord can't address a sound-server source by name
cmds = append(cmds, []string{"arecord", "-q", "-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"})
}
return cmds
}
func (e *engine) start(lang, device string) error {
e.mu.Lock()
if e.state != "ready" {
e.mu.Unlock()
return fmt.Errorf("engine is %s — wait for it to be ready", e.state)
}
if e.recording {
e.mu.Unlock()
return nil
}
if lang == "" {
lang = e.cfg.Language
}
e.mu.Unlock()
ws, _, err := websocket.DefaultDialer.Dial("ws://127.0.0.1:"+e.cfg.NemoPort+"/v1/audio/transcriptions/realtime",
http.Header{"Authorization": {"Bearer " + e.nemoKey}})
if err != nil {
return fmt.Errorf("connect to speech server: %w", err)
}
upd, _ := json.Marshal(event{"type": "session.update", "session": event{
"sample_rate": 16000, "language": lang, "automatic_punctuation": true, "endpointing_ms": e.cfg.EndpointMs,
}})
if err := ws.WriteMessage(websocket.TextMessage, upd); err != nil {
ws.Close()
return fmt.Errorf("session.update: %w", err)
}
// Bluetooth headsets start on CVSD, not mSBC. Measured 2026-09-27 on a Bluetooth headset: on mSBC the
// mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after
// switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an
// 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out.
if card := btCard(device); card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" {
if p := currentProfile(card); p == "headset-head-unit-msbc" {
e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)")
e.hub.send(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"})
if err := switchProfile(card, e.logs); err != nil {
e.logs.add("pre-switch to CVSD failed: %v", err)
} else {
e.hub.send(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."})
}
}
}
first, err := spawnRecorder(device, e.logs)
if err != nil {
ws.Close()
return err
}
recName := first.name
var recMu sync.Mutex
cur := first // the live recorder; replaced when a dead mic link is recovered
ctx, cancel := context.WithCancel(context.Background())
var writeMu sync.Mutex
done := make(chan struct{})
e.mu.Lock()
e.recording, e.recorder, e.language, e.device = true, recName, lang, device
e.stopSession = func() {
cancel()
recMu.Lock()
cur.kill()
recMu.Unlock()
writeMu.Lock()
_ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`))
writeMu.Unlock()
select { // let the final words arrive
case <-done:
case <-time.After(4 * time.Second):
}
ws.Close()
}
e.mu.Unlock()
e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device)))
go watchKernelBluetooth(ctx, e.logs)
e.hub.send(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device})
for _, d := range e.devs.current().Devices {
if (d.ID == device || (device == "" && d.Default)) && d.Muted {
e.hub.send(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."})
}
}
// Audio pump: 100 ms frames → speech server, plus a level meter and dead-mic RECOVERY.
go func() {
frame := make([]byte, 3200)
var silentSince time.Time
recoveries := 0
var recovering atomic.Bool
var resetting atomic.Bool
tracePeak, lastTrace := 0, time.Now()
var healthySince time.Time // a mic that stays healthy 30 s earns back its 3 reconnects
gaveUp := false
lastLevel := time.Time{}
for {
recMu.Lock()
r := cur
recMu.Unlock()
if _, err := io.ReadFull(r.out, frame); err != nil {
if ctx.Err() != nil {
return
}
recMu.Lock()
swapped := cur != r
recMu.Unlock()
if swapped { // we killed it ourselves during a recovery: read from the new one
continue
}
// A headset reset restarts PipeWire, which kills parec mid-read. That is expected:
// wait for the reset to hand over a fresh recorder instead of ending the session.
if resetting.Load() {
for resetting.Load() && ctx.Err() == nil {
time.Sleep(200 * time.Millisecond)
}
continue
}
// The recorder died on its own (bad device, sound server unreachable, device unplugged).
// End the session so the UI stops showing "Listening", and say WHY in plain words.
_ = r.cmd.Wait()
why := strings.TrimSpace(r.errBuf.String())
e.logs.add("recorder %s ended unexpectedly: %v | stderr: %s", r.name, err, why)
msg := "The microphone stopped delivering audio."
switch {
case strings.Contains(why, "Connection refused") || strings.Contains(why, "pa_context_connect"):
msg = "Can't reach the computer's sound system (PipeWire/PulseAudio). Is it running?"
case strings.Contains(why, "No such entity") || strings.Contains(why, "not found"):
msg = "That microphone isn't available any more — it may have been unplugged or disconnected. Pick another one."
}
if why != "" {
msg += " (" + r.name + ": " + why + ")"
}
e.hub.send(event{"type": "error", "message": msg})
go e.stop()
return
}
peak := 0
for i := 0; i+1 < len(frame); i += 2 {
v := int(int16(binary.LittleEndian.Uint16(frame[i:])))
if v < 0 {
v = -v
}
if v > peak {
peak = v
}
}
// ★ DEAD MIC LINK. Exact-zero samples are not quiet speech (a live mic always has a noise
// floor). Measured 2026-09-27 on a Bluetooth headset (HFP, mSBC): four words
// transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1,
// the headset losing the mic side of the link. Re-opening the source restarts the audio
// stream; the speech session and the transcript carry on untouched.
if peak == 0 {
if silentSince.IsZero() {
silentSince = time.Now()
} else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond {
recoveries++
switch {
case recoveries == 1:
// Step 1: reopen the recorder (cheap; fixes a stalled stream).
e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone")
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"})
if next, err := spawnRecorder(device, e.logs); err == nil {
recMu.Lock()
old := cur
cur = next
recMu.Unlock()
old.kill()
recovering.Store(true)
}
case recoveries == 2 && btCard(device) != "":
// Step 2: force a NEW Bluetooth voice link by switching the headset's voice profile to
// CVSD (the old Rudder's proven-stabler choice). Unlike a detect-first reset script this does not
// "detect first", so it can't decide a dead link is fine.
card := btCard(device)
e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card)
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"})
resetting.Store(true)
go func() {
defer resetting.Store(false)
if err := switchProfile(card, e.logs); err != nil {
e.hub.send(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()})
return
}
if next, err := spawnRecorder(device, e.logs); err == nil {
recMu.Lock()
old := cur
cur = next
recMu.Unlock()
old.kill()
recovering.Store(true)
}
}()
case recoveries == 3 && btResetPath() != "":
// Step 3: full headset reset (disconnect → restart PipeWire → reconnect).
e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET")
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"})
resetting.Store(true)
go func() {
defer resetting.Store(false)
ok, ran := runHeadsetReset(e.logs)
if !ran {
return
}
if !ok {
e.hub.send(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"})
}
if next, err := spawnRecorder(device, e.logs); err == nil {
recMu.Lock()
old := cur
cur = next
recMu.Unlock()
old.kill()
recovering.Store(true)
}
}()
default:
gaveUp = true
e.logs.add("mic still dead after reopen + headset resets — giving up for this session")
msg := "Your microphone keeps going silent. This is the Bluetooth headset dropping its mic link (the kernel reports corrupted voice packets), not Rudder. Try a wired or built-in mic, or turn off the headset's noise cancellation / adaptive audio in its app."
if btResetPath() == "" {
msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)"
}
e.hub.send(event{"type": "warning", "message": msg})
}
silentSince = time.Now()
}
} else {
if recovering.Load() {
recovering.Store(false)
healthySince = time.Now()
e.logs.add("mic recovered after %d reconnect(s)", recoveries)
e.hub.send(event{"type": "recovered", "message": "Microphone is back — keep talking."})
}
if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second {
recoveries, gaveUp, healthySince = 0, false, time.Time{}
}
silentSince = time.Time{}
}
// Flight recorder: the mic level twice a second, so a drop can be lined up against kernel and
// sound-server events in the same log (/api/rudder/log).
tracePeak = max(tracePeak, peak)
if time.Since(lastTrace) >= 500*time.Millisecond {
e.logs.add("trace level peak=%d", tracePeak)
tracePeak, lastTrace = 0, time.Now()
}
if time.Since(lastLevel) > 150*time.Millisecond {
lastLevel = time.Now()
e.hub.send(event{"type": "level", "peak": float64(peak) / 32768})
}
writeMu.Lock()
err := ws.WriteMessage(websocket.BinaryMessage, frame)
writeMu.Unlock()
if err != nil {
if ctx.Err() == nil {
e.hub.send(event{"type": "error", "message": "speech server connection lost: " + err.Error()})
}
return
}
}
}()
// Result reader: map the server's events onto the plugin's small vocabulary.
go func() {
defer close(done)
for {
_, msg, err := ws.ReadMessage()
if err != nil {
return
}
var m map[string]any
if json.Unmarshal(msg, &m) != nil {
continue
}
t, _ := m["type"].(string)
text := firstString(m, "transcript", "delta", "text")
switch {
case strings.HasSuffix(t, "input_audio_transcription.delta"):
// Deltas APPEND to the utterance in progress; the server sends many empty ones.
if text != "" {
e.hub.send(event{"type": "partial", "text": text})
}
case strings.HasSuffix(t, "input_audio_transcription.completed"):
e.hub.send(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)})
case t == "error":
e.logs.add("speech server error: %s", string(msg))
e.hub.send(event{"type": "error", "message": string(msg)})
case t == "input_audio_buffer.committed":
return
default:
e.logs.add("speech event: %s", t)
}
}
}()
return nil
}
// recorderProc is one running recorder process.
type recorderProc struct {
cmd *exec.Cmd
out io.ReadCloser
name string
errBuf *bytes.Buffer
}
func (r *recorderProc) kill() {
if r != nil && r.cmd != nil && r.cmd.Process != nil {
_ = r.cmd.Process.Kill()
go func() { _ = r.cmd.Wait() }()
}
}
// spawnRecorder starts the first recorder that works for this device.
func spawnRecorder(device string, logs *ring) (*recorderProc, error) {
for _, c := range recorderCmds(device) {
if _, err := exec.LookPath(c[0]); err != nil {
continue
}
cmd := exec.Command(c[0], c[1:]...)
cmd.Env = audioEnv()
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
errBuf := &bytes.Buffer{}
cmd.Stderr = errBuf
out, err := cmd.StdoutPipe()
if err != nil {
continue
}
if err := cmd.Start(); err != nil {
logs.add("recorder %s failed to start: %v", c[0], err)
continue
}
return &recorderProc{cmd: cmd, out: out, name: c[0], errBuf: errBuf}, nil
}
return nil, errors.New("no microphone recorder found (need parec, pw-record or arecord)")
}
func firstString(m map[string]any, keys ...string) string {
for _, k := range keys {
if s, ok := m[k].(string); ok {
return s
}
}
if item, ok := m["item"].(map[string]any); ok {
return firstString(item, keys...)
}
return ""
}
func (e *engine) stop() {
e.mu.Lock()
stop := e.stopSession
e.stopSession = nil
was := e.recording
e.recording = false
e.mu.Unlock()
if !was || stop == nil {
return
}
stop()
e.logs.add("recording stopped")
e.hub.send(event{"type": "recording", "recording": false})
}
func (e *engine) shutdown() {
e.stop()
e.mu.Lock()
n := e.nemo
e.mu.Unlock()
if n != nil && n.Process != nil {
_ = n.Process.Kill()
}
}
// ─── HTTP API ────────────────────────────────────────────────────────────────────────────────────
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(v)
}
// guard: two protections for a server that streams what you say.
// - Host check (DNS rebinding): a web page on any domain can make the browser resolve its own hostname
// to 127.0.0.1 and then read this API as "same-origin". Only requests addressed to 127.0.0.1 or
// localhost are served.
// - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows)
// may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the
// dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests).
func guard(next http.Handler) http.Handler {
extra := os.Getenv("RUDDER_ALLOW_ORIGIN")
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
host := r.Host
if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") {
host = host[:i]
}
if host != "127.0.0.1" && host != "localhost" && host != "[::1]" {
http.Error(w, "forbidden host", http.StatusForbidden)
return
}
o := r.Header.Get("Origin")
if o == "tauri://localhost" || o == "http://tauri.localhost" || (extra != "" && o == extra) {
w.Header().Set("Access-Control-Allow-Origin", o)
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
w.Header().Set("Vary", "Origin")
}
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
}
next.ServeHTTP(w, r)
})
}
func main() {
cfg := loadConfig()
logs := &ring{}
h := &hub{clients: map[chan []byte]struct{}{}}
key := make([]byte, 24)
if _, err := rand.Read(key); err != nil {
log.Fatal("no randomness for the speech-server key: ", err)
}
e := &engine{cfg: cfg, logs: logs, hub: h, state: "starting", devs: &deviceWatcher{hub: h, logs: logs}, nemoKey: hex.EncodeToString(key)}
e.devs.run()
mux := http.NewServeMux()
mux.HandleFunc("/api/rudder/status", func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, e.status())
})
mux.HandleFunc("/api/rudder/start", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSON(w, 405, event{"error": "POST only"})
return
}
var body struct {
Language string `json:"language"`
Device string `json:"device"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
if err := e.start(body.Language, body.Device); err != nil {
e.logs.add("start failed: %v", err)
writeJSON(w, 409, event{"error": err.Error()})
return
}
writeJSON(w, 200, e.status())
})
mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) {
e.stop()
writeJSON(w, 200, e.status())
})
mux.HandleFunc("/api/rudder/devices", func(w http.ResponseWriter, r *http.Request) {
e.devs.refresh("request")
writeJSON(w, 200, e.devs.current())
})
mux.HandleFunc("/api/rudder/test", func(w http.ResponseWriter, r *http.Request) {
var body struct {
Device string `json:"device"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
writeJSON(w, 200, e.micTest(body.Device))
})
mux.HandleFunc("/api/rudder/log", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = io.WriteString(w, logs.text())
})
mux.HandleFunc("/api/rudder/events", func(w http.ResponseWriter, r *http.Request) {
fl, ok := w.(http.Flusher)
if !ok {
http.Error(w, "streaming unsupported", 500)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("X-Accel-Buffering", "no")
c := e.hub.subscribe()
defer e.hub.unsubscribe(c)
first, _ := json.Marshal(event{"type": "status", "status": e.status()})
fmt.Fprintf(w, "data: %s\n\n", first)
devs, _ := json.Marshal(event{"type": "devices", "devices": e.devs.current()})
fmt.Fprintf(w, "data: %s\n\n", devs)
fl.Flush()
ping := time.NewTicker(15 * time.Second)
defer ping.Stop()
for {
select {
case <-r.Context().Done():
return
case b := <-c:
fmt.Fprintf(w, "data: %s\n\n", b)
fl.Flush()
case <-ping.C:
fmt.Fprint(w, ": ping\n\n")
fl.Flush()
}
}
})
srv := &http.Server{Addr: "127.0.0.1:" + cfg.Port, Handler: guard(mux)}
logs.add("rudder-engine listening on http://127.0.0.1:%s (runtime=%s model=%s)", cfg.Port, cfg.NemoBin, cfg.Model)
e.startNemo()
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sig
e.shutdown()
os.Exit(0)
}()
if err := srv.ListenAndServe(); err != nil {
e.shutdown()
log.Fatal(err)
}
}

View file

@ -0,0 +1,59 @@
#!/usr/bin/env bash
# setup-runtime.sh — fetch Rudder's speech runtime + model, verified by SHA-256, then build the engine.
#
# Rudder (plugin-designs/rudder-voice) transcribes on THIS computer. It needs two things that are too big
# for git: NVIDIA's NeMo-Speech.cpp runtime (~4 MB, CPU build) and the Nemotron 3.5 ASR Streaming GGUF
# (~742 MB). Both are pinned by version and checksum; a mismatch aborts instead of running an unverified
# file. After this, `bun run dev` starts the engine and Rudder appears in the Plugin Studio.
#
# bash backend/rudder-engine/setup-runtime.sh # installs into ~/.local/share/rudder
# RUDDER_HOME=/some/dir bash backend/rudder-engine/setup-runtime.sh
#
# Linux x86_64 only for now (the runtime ships other platforms; add them when a desktop build needs them).
set -euo pipefail
RUDDER_HOME="${RUDDER_HOME:-$HOME/.local/share/rudder}"
RT_VER="0.1.0"
RT_NAME="nemo-speech-${RT_VER}-linux-x86_64-cpu"
RT_URL="https://github.com/NVIDIA/NeMo-Speech.cpp/releases/download/v${RT_VER}/${RT_NAME}.tar.gz"
RT_SHA="0f74131d631ad2c694cf0ec53490866bb6461147959589a69fb6fc231944065b"
# NVIDIA re-converted the GGUF on 2026-09-10 (embedded tokenizer); this revision is the one Rudder is tested with.
MODEL_REV="ea30d66debe3"
MODEL_FILE="nemotron-3.5-asr-streaming-0.6b.q8_0.gguf"
MODEL_URL="https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${MODEL_REV}/${MODEL_FILE}"
MODEL_SHA="3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1"
say() { printf '\033[1;34m→\033[0m %s\n' "$*"; }
die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; }
[ "$(uname -s)-$(uname -m)" = "Linux-x86_64" ] || die "This script currently supports Linux x86_64 only."
command -v curl >/dev/null || die "curl is required."
command -v sha256sum >/dev/null || die "sha256sum is required."
fetch() { # url dest sha
local url="$1" dest="$2" sha="$3"
if [ -f "$dest" ] && [ "$(sha256sum "$dest" | cut -d' ' -f1)" = "$sha" ]; then
say "already have $(basename "$dest") (checksum OK)"; return
fi
say "downloading $(basename "$dest")"
curl -fL --retry 3 -C - -o "$dest.part" "$url"
[ "$(sha256sum "$dest.part" | cut -d' ' -f1)" = "$sha" ] || { rm -f "$dest.part"; die "checksum mismatch for $(basename "$dest") — refusing to use it"; }
mv "$dest.part" "$dest"
}
mkdir -p "$RUDDER_HOME/bin" "$RUDDER_HOME/models"
fetch "$RT_URL" "$RUDDER_HOME/bin/$RT_NAME.tar.gz" "$RT_SHA"
tar -xzf "$RUDDER_HOME/bin/$RT_NAME.tar.gz" -C "$RUDDER_HOME/bin"
fetch "$MODEL_URL" "$RUDDER_HOME/models/$MODEL_FILE" "$MODEL_SHA"
if command -v go >/dev/null; then
say "building backend/bin/rudder-engine"
(cd "$(dirname "$0")" && go build -o ../bin/rudder-engine .)
else
printf '! Go not found — install Go, then: cd backend/rudder-engine && go build -o ../bin/rudder-engine .\n'
fi
for tool in parec pactl; do command -v "$tool" >/dev/null || printf '! %s not found — install pipewire-pulse or pulseaudio-utils for microphone capture.\n' "$tool"; done
say "done. Runtime: $RUDDER_HOME/bin/$RT_NAME Model: $RUDDER_HOME/models/$MODEL_FILE"
say "run \`bun run dev\` and open Rudder (rudder-voice) in the Plugin Studio."

View file

@ -0,0 +1,61 @@
/**
* internal/backend-dev-plugin.mjs — `bun run dev` starts the local plugin backends.
*
* WHY: some plugins ship to DESKTOP with a local backend (rudder-voice → backend/bin/rudder-engine,
* run as a Tauri sidecar). In the Plugin Studio the same backend must be running, or the plugin can't
* be tested in the browser. This plugin starts each built backend with the dev server, forwards its
* /api/<name>/ routes, and stops it when the dev server stops. A backend that isn't built is skipped
* with a hint: plugins without a backend never notice this file.
*
* DEV ONLY (`apply: 'serve'`). A production build never starts or bundles a backend.
*/
import { spawn } from 'node:child_process'
import { existsSync } from 'node:fs'
import { resolve } from 'node:path'
// name → how to run it. `route` is proxied to `port` on 127.0.0.1.
export const DEV_BACKENDS = [
{
name: 'rudder-engine',
bin: 'backend/bin/rudder-engine',
build: 'cd backend/rudder-engine && go build -o ../bin/rudder-engine .',
route: '/api/rudder',
port: Number(process.env.RUDDER_PORT || 5193),
env: { RUDDER_PORT: String(process.env.RUDDER_PORT || 5193), RUDDER_NEMO_PORT: String(process.env.RUDDER_NEMO_PORT || 5194) },
},
]
// The proxy table for vite's `server.proxy`. SSE (`/events`) streams through unbuffered.
export function devBackendProxy() {
return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false }]))
}
export function backendDevPlugin(root) {
const children = []
return {
name: 'dx-backend-dev',
apply: 'serve',
configureServer(server) {
for (const b of DEV_BACKENDS) {
const bin = resolve(root, b.bin)
if (!existsSync(bin)) {
server.config.logger.warn(`[backend] ${b.name} not built — its plugin will show "engine offline". Build: ${b.build}`)
continue
}
const child = spawn(bin, [], { cwd: root, env: { ...process.env, ...b.env }, stdio: ['ignore', 'pipe', 'pipe'] })
const tag = `[${b.name}]`
const out = (d) => String(d).trim().split('\n').forEach((l) => l && server.config.logger.info(`${tag} ${l}`))
child.stdout.on('data', out)
child.stderr.on('data', out)
child.on('exit', (code) => server.config.logger.warn(`${tag} exited (${code})`))
children.push(child)
server.config.logger.info(`${tag} started → ${b.route} on 127.0.0.1:${b.port}`)
}
const stop = () => children.forEach((c) => { try { c.kill('SIGTERM') } catch {} })
server.httpServer?.on('close', stop)
process.once('exit', stop)
process.once('SIGINT', () => { stop(); process.exit(0) })
process.once('SIGTERM', () => { stop(); process.exit(0) })
},
}
}

View file

@ -0,0 +1,55 @@
# rudder-voice — Rudder, private voice-to-text (desktop target)
**Rudder is a plugin that isn't a map.** It shows that the same dx-maps pattern — one folder in
`plugin-designs/`, developed live in the Plugin Studio — can ship a **desktop app with its own local
backend**, not only embeddable maps. It is also a genuinely useful tool for the people who build here:
talk to your agents instead of typing, with nothing leaving your computer.
- **Streaming.** Words appear in the text box while you speak; talk for as long as you like.
- **Type and talk at once.** Type, paste links or code anywhere in the box; everything you say is
added to the end. While you're typing, Rudder holds its words and adds them when you pause.
- **CPU only.** No graphics card. About 1 GB of memory; keeps up with normal speech on a laptop CPU.
- **Offline, no telemetry.** NVIDIA Nemotron 3.5 ASR Streaming (0.6B) run by NVIDIA's open-source
NeMo-Speech.cpp (ggml). Traced with `strace` while transcribing: zero sockets, zero DNS lookups.
- **Microphones like a video-call app.** Pick the input, hot-plug updates, a 3-second mic test, and
plain-language errors (muted, disconnected, dead Bluetooth mic link — which it tries to recover).
## Run it in the Plugin Studio
```bash
bash backend/rudder-engine/setup-runtime.sh # once: runtime + model (verified by SHA-256) + engine build
bun run dev # starts the engine too; open "rudder-voice" in the studio
```
Linux with PipeWire or PulseAudio (`parec`, `pactl`). The engine listens on `127.0.0.1` only.
## How it fits together
```
<rudder-voice> (this folder) ──HTTP + SSE /api/rudder──► backend/rudder-engine (Go)
├─ parec / pw-record (mic, open only while recording)
└─ WebSocket ► nemo-speech serve (127.0.0.1, model stays loaded)
```
`bun run dev` starts `backend/bin/rudder-engine` through `internal/backend-dev-plugin.mjs` and forwards
`/api/rudder` to it, so the plugin calls the same API in the browser as it will inside the desktop app,
where Tauri runs the engine as a sidecar and passes its address in `profile-config` (`apiBase`).
`rudder-voice.meta.json` declares `"targets": ["desktop"]`: on a plain website there is no local engine,
so this plugin is not meant for web embeds.
## Settings (environment variables)
| Variable | Default | What it does |
|---|---|---|
| `RUDDER_PORT` | `5193` | engine HTTP port (loopback) |
| `RUDDER_NEMO_PORT` | `5194` | speech server port (loopback) |
| `RUDDER_MODEL` / `RUDDER_NEMO_BIN` | `~/.local/share/rudder/...` | model and runtime paths; the engine refuses a model that isn't a local file |
| `RUDDER_LANG` | `en-US` | default language (the model supports ~30) |
| `RUDDER_KEEP_MSBC` | unset | keep Bluetooth headsets on mSBC instead of switching them to the more stable CVSD |
| `RUDDER_BT_RESET` | unset | optional command that fully resets a Bluetooth headset, used when its mic link dies |
## Not yet
- Desktop build (`desktop:build`) with the engine as a Tauri sidecar.
- A saved word-correction table (PocketBase) for technical terms the model mishears.

View file

@ -0,0 +1,32 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>doxa-maps · rudder-voice staging</title>
<link rel="icon" href="data:,">
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
html, body { background: #0b0f14; color: #e5e7eb; min-height: 100%;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; }
.wrap { max-width: 960px; margin: 0 auto; padding: 28px 20px 48px; }
h1 { font-size: 22px; font-weight: 700; letter-spacing: -0.02em; }
.sub { color: #9ca3af; font-size: 13px; margin: 6px 0 24px; max-width: 820px; }
.sub code { color: #c4b5fd; }
.host { border: 1px solid #1f2937; border-radius: 12px; overflow: hidden; }
rudder-voice { display: block; width: 100%; }
</style>
</head>
<body>
<div class="wrap">
<h1>rudder-voice · staging</h1>
<p class="sub">Desktop-target plugin: private, offline, streaming voice-to-text. The speech model runs in the
local backend <code>backend/rudder-engine</code> (NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR), started by
<code>bun run dev</code> and reached at <code>/api/rudder</code>.</p>
<div class="host">
<rudder-voice></rudder-voice>
</div>
</div>
<script type="module" src="./index.js"></script>
</body>
</html>

View file

@ -0,0 +1,23 @@
/**
* rudder-voice / index.js — Bundle entry (Capability-1 single element, DESKTOP target).
*
* <rudder-voice> — private, offline, streaming voice-to-text. The UI is this bundle; the speech model
* runs in the local backend (backend/rudder-engine), started by `bun run dev` in the Plugin Studio and
* by Tauri as a sidecar in the desktop app. See rudder-voice.vue for the contract.
*
* Desktop-only: rudder-voice.meta.json declares "targets": ["desktop"]. On a plain website there is no
* local engine, so the element shows "Engine offline" instead of pretending to work.
*
* Usage (desktop shell / Plugin Studio):
* <rudder-voice profile-config='{"apiBase":"http://127.0.0.1:5193/api/rudder"}'></rudder-voice>
*/
import { defineCustomElement } from 'vue'
import RudderVoice from './rudder-voice.vue'
const RudderVoiceElement = defineCustomElement(RudderVoice)
if (typeof customElements !== 'undefined') {
if (!customElements.get('rudder-voice')) customElements.define('rudder-voice', RudderVoiceElement)
}
export default RudderVoiceElement

View file

@ -0,0 +1,7 @@
{
"_comment": "Rudder — private, offline, streaming voice-to-text. DESKTOP target: the speech model runs in the local backend (backend/rudder-engine: NVIDIA NeMo-Speech.cpp + Nemotron 3.5 ASR GGUF), started by `bun run dev` and run by Tauri as a sidecar. Not for public web embeds: a website has no local engine.",
"type": "single",
"label": "Rudder — voice to text (desktop)",
"targets": ["desktop"],
"backend": "backend/rudder-engine"
}

View file

@ -0,0 +1,650 @@
<script setup>
/**
* rudder-voice.vue — Rudder: private, offline, streaming voice-to-text.
*
* @element rudder-voice
* @description A DESKTOP-target plugin (rudder-voice.meta.json "targets": ["desktop"]). The speech
* model runs in the local backend (backend/rudder-engine → NVIDIA NeMo-Speech.cpp + Nemotron 3.5
* ASR), never in the browser and never on a server. In the Plugin Studio, `bun run dev` starts that
* backend and forwards /api/rudder; in the desktop app, Tauri runs it as a sidecar and passes its
* address in `profile-config` {"apiBase": "http://127.0.0.1:<port>/api/rudder"}.
*
* Everything is agent-interfaceable (the old Rudder's prime directive): the 🐞 panel copies the
* engine log + status with a header telling an agent what to do with it.
*/
import { computed, onBeforeUnmount, onMounted, ref, nextTick } from 'vue'
defineOptions({ inheritAttrs: false })
const props = defineProps({ profileConfig: { type: String, default: '' } })
const cfg = computed(() => {
try { return props.profileConfig ? JSON.parse(props.profileConfig) : {} } catch { return {} }
})
const apiBase = computed(() => (cfg.value.apiBase || '/api/rudder').replace(/\/$/, ''))
const LANGUAGES = [
['en-US', 'English (US)'], ['en-GB', 'English (UK)'], ['es-US', 'Spanish (US)'], ['es-ES', 'Spanish (Spain)'],
['fr-FR', 'French'], ['de-DE', 'German'], ['it-IT', 'Italian'], ['pt-BR', 'Portuguese (Brazil)'],
['nl-NL', 'Dutch'], ['ru-RU', 'Russian'], ['uk-UA', 'Ukrainian'], ['tr-TR', 'Turkish'], ['ar-AR', 'Arabic'],
['hi-IN', 'Hindi'], ['ja-JP', 'Japanese'], ['ko-KR', 'Korean'], ['vi-VN', 'Vietnamese'], ['zh-CN', 'Mandarin'],
]
const STORE_KEY = 'rudder-voice:v1'
function load() {
try { return JSON.parse(localStorage.getItem(STORE_KEY) || '{}') } catch { return {} }
}
function save() {
try { localStorage.setItem(STORE_KEY, JSON.stringify({ language: language.value, device: device.value, draft: draft.value })) } catch {}
}
const saved = load()
const engine = ref('connecting') // connecting | starting | ready | error | offline
const engineError = ref('')
const runtime = ref('')
const model = ref('')
const recording = ref(false)
const busy = ref(false)
const language = ref(saved.language || cfg.value.language || 'en-US')
// The draft: a text area you type and paste into; every finished utterance is APPENDED to its end.
// (v1 kept a list of lines — migrate it so nothing already dictated is lost.)
const draft = ref(typeof saved.draft === 'string' ? saved.draft
: Array.isArray(saved.lines) ? saved.lines.map((l) => l.text).join(' ') : '')
const partial = ref('') // the utterance being heard right now (streamed into the box's live tail)
const box = ref(null)
const level = ref(0)
const notice = ref('')
const noticeKind = ref('info')
const copied = ref('')
const showDebug = ref(false)
const showInfo = ref(false)
const clientLog = ref([])
// ─── Microphones (the Jitsi device-selection pattern, served natively by the engine) ───
const devices = ref([]) // [{id,label,bus,default,muted,volume,state}]
const offCards = ref([]) // sound cards with a mic that are switched off (e.g. the laptop's own)
const deviceError = ref('')
const device = ref(saved.device || '') // '' = follow the system default input
const testing = ref(false)
const testResult = ref(null) // {ok, verdict, peak}
const BUS_ICON = { bluetooth: '🎧', usb: '🔌', pci: '💻' }
function deviceLabel(d) {
const bits = [(BUS_ICON[d.bus] || '🎙') + ' ' + d.label]
if (d.muted) bits.push('(muted)')
else if (d.volume) bits.push('· ' + d.volume)
return bits.join(' ')
}
const defaultDevice = computed(() => devices.value.find((d) => d.default))
const chosen = computed(() => devices.value.find((d) => d.id === device.value) || (device.value ? null : defaultDevice.value))
function applyDevices(list) {
if (!list) return
devices.value = list.devices || []
offCards.value = list.offCards || []
deviceError.value = list.error || ''
// A saved choice that is gone (headset switched off) falls back to the default, and says so.
if (device.value && !devices.value.some((d) => d.id === device.value)) {
if (!recording.value) {
say('Your chosen microphone is no longer connected — using the system default instead.', 'warn')
device.value = ''
save()
}
}
}
async function refreshDevices() {
try { applyDevices(await (await fetch(apiBase.value + '/devices')).json()) } catch (err) { logc('devices fetch failed: ' + err) }
}
async function testMic() {
if (testing.value || recording.value) return
testing.value = true
testResult.value = null
try {
testResult.value = await post('/test', { device: device.value })
} catch (err) {
testResult.value = { ok: false, verdict: String(err.message || err) }
} finally {
testing.value = false
level.value = 0
}
}
let es = null
let retry = null
function logc(msg) {
clientLog.value.push(new Date().toISOString().slice(11, 23) + ' ' + msg)
if (clientLog.value.length > 200) clientLog.value.splice(0, clientLog.value.length - 200)
}
function say(msg, kind = 'info') { notice.value = msg; noticeKind.value = kind }
// ─── Writing into the box ────────────────────────────────────────────────────────────────────────
// Every word streams straight into the END of the box while you keep typing, pasting and editing
// anywhere. The sentence in progress lives in a "live tail" — the last `live.text.length` characters
// of the draft — and each new word rewrites only that tail.
//
// ★ TYPING WHILE TALKING (bug fixed 2026-09-27): once you type into or after the tail it is yours, and
// Rudder must continue with ONLY the words it hasn't written yet. v1 re-wrote the whole sentence-so-far
// after every keystroke, so each key press duplicated the utterance. Now `written` counts the words of
// this utterance already in the box, and the finished (punctuated) sentence is lined up word-by-word
// against them, so nothing is ever written twice.
let live = null // { start, text, shown } — the tail Rudder may still rewrite; shown = words in it
let written = 0 // words of the current utterance committed to the box before the current tail
// The tail is still Rudder's while the box ENDS with it — wherever it now starts. Typing or pasting
// anywhere above it shifts its position, and that must not count as touching it (bug 2026-09-27: a
// position-based check released the tail on every keystroke above it, cutting "speaking" to "spe").
function liveIntact() {
if (!live || !draft.value.endsWith(live.text)) return false
live.start = draft.value.length - live.text.length
return true
}
// Change only the end of the draft, keeping the person's caret/selection and scroll where they were.
function writeEnd(next) {
const el = box.value
const focused = el && el.getRootNode().activeElement === el
const selStart = el ? el.selectionStart : 0
const selEnd = el ? el.selectionEnd : 0
const atBottom = !el || el.scrollHeight - el.scrollTop - el.clientHeight < 32
draft.value = next
save()
nextTick(() => {
if (!el) return
if (focused) el.setSelectionRange(Math.min(selStart, next.length), Math.min(selEnd, next.length))
if (atBottom || !focused) el.scrollTop = el.scrollHeight
})
}
const sepFor = (text) => (!text || /\s$/.test(text) ? '' : ' ')
const wordsOf = (text) => (text || '').trim().split(/\s+/).filter(Boolean)
// The person typed into or after the tail, so it's theirs now. Count its words as written — except a
// HALF-FINISHED last word: the model streams words in pieces ("for" … "tnight"), and typing between the
// pieces used to freeze "for" and lose the rest (measured 2026-09-27: "for a for [typed]"). If the word
// has since grown, the fragment is removed (it sits just before the person's text; their caret is
// shifted so it stays put relative to what they typed) and the whole word is written after it.
const norm = (x) => (x || '').toLowerCase().replace(/[^\p{L}\p{N}']/gu, '')
function releaseTail(utteranceWords) {
const shownWords = wordsOf(live.text)
const lastShown = shownWords[shownWords.length - 1] || ''
const nowWord = utteranceWords[written + live.shown - 1]
const grew = live.shown > 0 && nowWord !== undefined && norm(nowWord) !== norm(lastShown) && norm(nowWord).startsWith(norm(lastShown))
// Find the tail by its text: typing above it may have moved it since it was written.
const found = draft.value.lastIndexOf(live.text)
if (found >= 0) live.start = found
const tailEnd = live.start + live.text.length
const prefixOk = found >= 0
if (grew && prefixOk && lastShown) {
let fragStart = tailEnd - lastShown.length
if (fragStart > live.start && draft.value[fragStart - 1] === ' ') fragStart--
const cut = tailEnd - fragStart
const el = box.value
const focused = el && el.getRootNode().activeElement === el
const shift = (i) => (i >= tailEnd ? i - cut : i > fragStart ? fragStart : i)
const a = el ? shift(el.selectionStart) : 0
const b = el ? shift(el.selectionEnd) : 0
draft.value = draft.value.slice(0, fragStart) + draft.value.slice(tailEnd)
if (focused) nextTick(() => el.setSelectionRange(a, b))
written += live.shown - 1
} else {
written += live.shown
}
live = null
}
// Write `words` as the live tail: rewrite it if it's still ours, otherwise open a new one at the end.
function writeTail(words, utterance) {
const t = words.join(' ')
if (liveIntact()) {
const lead = live.text.match(/^\s*/)[0]
live.text = t ? lead + t : ''
live.shown = words.length
writeEnd(draft.value.slice(0, live.start) + live.text)
return
}
if (live) releaseTail(utterance) // the person took over the old tail: keep it
if (!t) return
live = { start: draft.value.length, text: sepFor(draft.value) + t, shown: words.length }
writeEnd(draft.value + live.text)
}
// ─── Hold the stream while the person is typing ─────────────────────────────────────────────────
// Typing and streaming both write at the end of the box; a word landing between two keystrokes splices
// into what is being typed (measured 2026-09-27: "he'[typed]ll"). So while keys are being pressed the
// stream is buffered, and flushed — in order, nothing dropped — 700 ms after the last keystroke.
const TYPING_HOLD_MS = 700
let lastTypedAt = 0
let pendingPartial = null
const pendingFinals = []
let flushTimer = null
function markTyping() { lastTypedAt = Date.now() }
function onUserInput() {
markTyping()
save()
}
function typingNow() { return Date.now() - lastTypedAt < TYPING_HOLD_MS }
function scheduleFlush() {
clearTimeout(flushTimer)
flushTimer = setTimeout(() => {
if (typingNow()) return scheduleFlush()
const finals = pendingFinals.splice(0)
const p = pendingPartial
pendingPartial = null
for (const f of finals) finishUtterance(f)
if (p !== null) streamPartial(p)
}, TYPING_HOLD_MS)
}
function onPartial(full) {
if (typingNow() || pendingFinals.length) { pendingPartial = full; scheduleFlush(); return }
streamPartial(full)
}
function onFinal(text) {
if (typingNow() || pendingFinals.length) { pendingFinals.push(text); pendingPartial = null; scheduleFlush(); return }
finishUtterance(text)
}
// A word arrived: show the words of this utterance that aren't in the box yet.
function streamPartial(full) {
const w = wordsOf(full)
if (live && !liveIntact()) releaseTail(w)
writeTail(w.slice(written), w)
}
// The sentence is finished: the same span of words, now punctuated and capitalised.
function finishUtterance(text) {
const w = wordsOf(text)
if (live && !liveIntact()) releaseTail(w)
writeTail(w.slice(written), w)
live = null
written = 0
}
function applyStatus(s) {
if (!s) return
if (s.engine) engine.value = s.engine
engineError.value = s.error || ''
if (s.runtime) runtime.value = s.runtime
if (s.model) model.value = s.model
if (typeof s.recording === 'boolean') recording.value = s.recording
}
function connect() {
clearTimeout(retry)
try { es?.close() } catch {}
es = new window.EventSource(apiBase.value + '/events')
es.onopen = () => logc('events connected')
es.onmessage = (m) => {
let e
try { e = JSON.parse(m.data) } catch { return }
switch (e.type) {
case 'status':
applyStatus(e.status || e)
if (e.engine === 'error') say(e.error || 'Speech engine error', 'error')
if (e.engine === 'ready' && noticeKind.value !== 'error') say('')
break
case 'recording':
recording.value = !!e.recording
if (!e.recording) { level.value = 0; flushPartial() }
break
case 'partial':
partial.value += e.text
onPartial(partial.value)
break
case 'final':
partial.value = ''
onFinal(e.text)
break
case 'recovering':
say(e.message, 'warn')
break
case 'recovered':
say(e.message, 'ok')
setTimeout(() => { if (noticeKind.value === 'ok') say('') }, 4000)
break
case 'devices':
applyDevices(e.devices)
break
case 'level':
level.value = Math.min(1, (e.peak || 0) * 3)
break
case 'warning':
say(e.message, 'warn')
break
case 'error':
say(e.message, 'error')
logc('engine error: ' + e.message)
break
}
}
es.onerror = () => {
engine.value = 'offline'
recording.value = false
logc('events disconnected; retrying in 3s')
try { es.close() } catch {}
retry = setTimeout(connect, 3000)
}
}
// Recording stopped mid-sentence: the streamed words are already in the box, so just let go of them.
function flushPartial() {
clearTimeout(flushTimer)
const finals = pendingFinals.splice(0)
const p = pendingPartial
pendingPartial = null
for (const f of finals) finishUtterance(f)
if (p !== null) streamPartial(p)
partial.value = ''
live = null
written = 0
written = 0
}
async function post(path, body) {
const r = await fetch(apiBase.value + path, {
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body || {}),
})
const j = await r.json().catch(() => ({}))
if (!r.ok) throw new Error(j.error || `HTTP ${r.status}`)
return j
}
async function toggle() {
if (busy.value) return
busy.value = true
try {
if (recording.value) {
await post('/stop')
} else {
say('')
testResult.value = null
applyStatus(await post('/start', { language: language.value, device: device.value }))
}
} catch (err) {
say(String(err.message || err), 'error')
logc('toggle failed: ' + err)
} finally {
busy.value = false
}
}
const words = computed(() => (draft.value.trim() ? draft.value.trim().split(/\s+/).length : 0))
async function copy(text, what) {
try {
await navigator.clipboard.writeText(text)
} catch {
const ta = document.createElement('textarea')
ta.value = text; document.body.appendChild(ta); ta.select()
try { document.execCommand('copy') } catch {}
ta.remove()
}
copied.value = what
setTimeout(() => { if (copied.value === what) copied.value = '' }, 1500)
}
function clearAll() {
if (draft.value.trim() && !window.confirm('Clear the whole text box? This cannot be undone.')) return
draft.value = ''
partial.value = ''
live = null
save()
}
// ★ PRIVACY: the report is meant to be pasted to an agent, so it describes the setup without
// identifying the person: no browser fingerprint (user-agent), no device names ("Anna's AirPods"), no
// device IDs (they embed the Bluetooth MAC address), and never the text in the box — only its length.
// The engine log it includes is already redacted at the source (MACs, home folder).
const MAC_RE = /\b([0-9a-f]{2}[:_]){5}[0-9a-f]{2}\b/gi
function privateDevices() {
return devices.value.map((d, i) => ({ mic: 'mic ' + (i + 1), bus: d.bus, default: d.default, muted: d.muted, volume: d.volume, state: d.state }))
}
function browserFamily() {
const ua = navigator.userAgent
return /firefox/i.test(ua) ? 'Firefox' : /edg\//i.test(ua) ? 'Edge' : /chrom/i.test(ua) ? 'Chromium-based' : /safari/i.test(ua) ? 'Safari' : 'other'
}
async function copyDebug() {
let engineLog = '(unreachable)'
let status = '(unreachable)'
try { engineLog = await (await fetch(apiBase.value + '/log')).text() } catch {}
try { status = JSON.stringify(await (await fetch(apiBase.value + '/status')).json(), null, 2) } catch {}
const text = [
'// RUDDER DEBUG REPORT — for a coding agent.',
'// Read the engine status and log below, find why transcription failed or misbehaved, and propose a fix',
'// in backend/rudder-engine/main.go or plugin-designs/rudder-voice/rudder-voice.vue. Do not guess:',
'// quote the log lines that prove the cause.',
'',
'== status ==', status,
'== plugin ==', JSON.stringify({ apiBase: apiBase.value, draftChars: draft.value.length, engine: engine.value, recording: recording.value, language: language.value, device: device.value ? 'chosen mic' : '(system default)', browser: browserFamily() }, null, 2),
'== microphones ==', JSON.stringify({ devices: privateDevices(), offCards: offCards.value.length, error: deviceError.value, lastTest: testResult.value }, null, 2),
'== client log ==', clientLog.value.join('\n'),
'== engine log ==', engineLog,
].join('\n').replace(MAC_RE, 'XX:XX:XX:XX:XX:XX')
copy(text, 'debug')
}
function onKey(ev) {
// Ctrl+Shift+Space toggles recording without reaching for the mouse.
if (ev.ctrlKey && ev.shiftKey && ev.code === 'Space') { ev.preventDefault(); toggle() }
}
onMounted(() => {
connect()
refreshDevices()
window.addEventListener('keydown', onKey)
})
onBeforeUnmount(() => {
clearTimeout(retry)
try { es?.close() } catch {}
window.removeEventListener('keydown', onKey)
})
const statusText = computed(() => ({
connecting: 'Connecting…',
starting: 'Loading speech model… (about 20 s the first time)',
ready: recording.value ? 'Listening' : 'Ready',
error: 'Engine error',
offline: 'Engine offline',
}[engine.value] || engine.value))
</script>
<template>
<div class="rudder" :class="{ rec: recording }">
<header class="top">
<div class="brand">
<span class="logo" aria-hidden="true">⎈</span>
<div>
<h1>Rudder</h1>
<p class="sub">Voice-to-text for vibe coders · private · runs on this computer · no internet</p>
</div>
</div>
<button class="info-btn" :aria-expanded="showInfo" title="What is Rudder?" aria-label="What is Rudder?" @click="showInfo = !showInfo">ⓘ</button>
<div class="status" :data-state="engine" role="status">
<span class="dot" aria-hidden="true"></span>{{ statusText }}
</div>
</header>
<section v-if="showInfo" class="info">
<h2>What is Rudder?</h2>
<p>A voice-to-text tool for vibe coders: talk to your agents instead of typing to them. Paste links, code
and notes into the box while you talk; everything you say is added to the end, ready to copy into a chat.</p>
<ul>
<li><b>Streaming.</b> Words appear while you speak, and you can keep talking for as long as you like —
an hour is fine. There is no time limit and nothing to wait for at the end.</li>
<li><b>Runs on your CPU.</b> No graphics card needed. The model uses about 1 GB of memory and keeps up with
normal speech on an ordinary laptop processor.</li>
<li><b>Private and offline.</b> Your voice never leaves this computer. No account, no cloud, no telemetry —
checked with a network tracer: zero connections while transcribing.</li>
<li><b>The model.</b> NVIDIA Nemotron 3.5 ASR Streaming (0.6B), run by NVIDIA's open-source
NeMo-Speech.cpp. It adds punctuation and capital letters by itself.</li>
<li><b>Why it lives in dx-maps.</b> Rudder is a plugin like the maps: built in the Plugin Studio, tested in the
browser, and shipped as a desktop app with its own local backend — proof that one dx-maps codebase can
build desktop apps, not only maps.</li>
</ul>
<p class="fine">Bluetooth headsets: Rudder switches them to their stable voice mode (CVSD), because the
higher-quality mode (mSBC) kept dropping the mic after a few words.</p>
</section>
<section class="controls">
<button
class="mic"
:class="{ on: recording }"
:disabled="busy || engine !== 'ready'"
:aria-pressed="recording"
:aria-label="recording ? 'Stop recording' : 'Start recording'"
@click="toggle"
>
<span class="mic-icon" aria-hidden="true">{{ recording ? '■' : '●' }}</span>
{{ recording ? 'Stop' : 'Record' }}
</button>
<div class="meter" aria-hidden="true"><div class="fill" :style="{ width: (level * 100).toFixed(0) + '%' }"></div></div>
<label class="lang">
<span>Language</span>
<select v-model="language" :disabled="recording" @change="save">
<option v-for="[code, name] in LANGUAGES" :key="code" :value="code">{{ name }}</option>
</select>
</label>
</section>
<section class="mics">
<label class="mic-pick">
<span>Microphone</span>
<select v-model="device" :disabled="recording || testing" @change="save(); testResult = null">
<option value="">System default{{ defaultDevice ? ' — ' + defaultDevice.label : '' }}</option>
<option v-for="d in devices" :key="d.id" :value="d.id">{{ deviceLabel(d) }}</option>
</select>
</label>
<button class="small" :disabled="testing || recording || engine === 'offline'" @click="testMic">
{{ testing ? 'Testing… speak now' : 'Test mic' }}
</button>
<button class="small ghost" title="Look for microphones again" aria-label="Look for microphones again" :disabled="recording" @click="refreshDevices">↻</button>
</section>
<p v-if="testResult" class="notice" :data-kind="testResult.ok ? (testResult.peak < 0.01 ? 'warn' : 'ok') : 'error'">
{{ testResult.ok ? '✓ ' : '✗ ' }}{{ testResult.verdict }}
</p>
<p v-if="chosen && chosen.muted" class="notice" data-kind="warn">“{{ chosen.label }}” is muted in your system sound settings — Rudder won't hear anything until it's unmuted.</p>
<p v-if="deviceError" class="notice" data-kind="error">{{ deviceError }}</p>
<p v-else-if="!devices.length && engine !== 'offline'" class="notice" data-kind="warn">No microphone found. Connect a headset or turn on your computer's microphone in Sound settings.</p>
<p v-if="offCards.length" class="hint">Also available but switched off in Sound settings: {{ offCards.join(', ') }}. Turn it on there to use it here.</p>
<p v-if="notice" class="notice" :data-kind="noticeKind">{{ notice }}</p>
<p v-else-if="engine === 'offline'" class="notice" data-kind="warn">
The local speech engine isn't reachable at <code>{{ apiBase }}</code>. In the Plugin Studio it starts with
<code>bun run dev</code> once built: <code>cd backend/rudder-engine &amp;&amp; go build -o ../bin/rudder-engine .</code>
</p>
<section class="compose">
<textarea
ref="box"
v-model="draft"
class="draft"
spellcheck="true"
aria-label="Your text. Type or paste here; everything Rudder hears is added to the end."
placeholder="Type or paste here — and press Record (Ctrl+Shift+Space) to talk. Your words stream into the end of this box as you speak."
@input="onUserInput"
@keydown="markTyping"
></textarea>
<p class="hearing" aria-live="polite">
<template v-if="recording"><span class="tag">listening</span> words appear at the end of the box as you speak — keep typing or pasting anywhere</template>
</p>
</section>
<footer class="bottom">
<button :disabled="!draft.trim()" @click="copy(draft, 'all')">{{ copied === 'all' ? 'Copied ✓' : 'Copy all' }}</button>
<button :disabled="!draft.trim() || recording" @click="clearAll">Clear</button>
<span class="count">{{ words }} {{ words === 1 ? 'word' : 'words' }}</span>
<span class="spacer"></span>
<button class="ghost" :aria-expanded="showDebug" @click="showDebug = !showDebug">🐞 Debug</button>
</footer>
<section v-if="showDebug" class="debug">
<dl>
<dt>API</dt><dd><code>{{ apiBase }}</code></dd>
<dt>Runtime</dt><dd>{{ runtime || '—' }}</dd>
<dt>Model</dt><dd>{{ model || '—' }}</dd>
<dt>Engine</dt><dd>{{ engine }}<span v-if="engineError"> — {{ engineError }}</span></dd>
</dl>
<button @click="copyDebug">{{ copied === 'debug' ? 'Copied ✓' : 'Copy debug report for an agent' }}</button>
</section>
</div>
</template>
<style>
:host { display: block; }
.rudder {
--bg: #0b0f14; --panel: #111823; --line: #1f2937; --text: #e5e7eb; --muted: #9ca3af;
--accent: #38bdf8; --rec: #ef4444; --ok: #22c55e; --warn: #f59e0b;
box-sizing: border-box; display: flex; flex-direction: column; gap: 14px; min-height: 480px; height: 100%;
padding: 18px; background: var(--bg); color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; font-size: 15px;
}
.rudder *, .rudder *::before, .rudder *::after { box-sizing: border-box; }
.top { display: flex; align-items: center; justify-content: space-between; gap: 12px; flex-wrap: wrap; }
.brand { display: flex; align-items: center; gap: 12px; }
.logo { font-size: 30px; color: var(--accent); line-height: 1; }
h1 { margin: 0; font-size: 20px; letter-spacing: -0.01em; }
.sub { margin: 2px 0 0; color: var(--muted); font-size: 12.5px; }
.status { display: inline-flex; align-items: center; gap: 8px; padding: 6px 12px; border: 1px solid var(--line);
border-radius: 999px; font-size: 13px; color: var(--muted); background: var(--panel); }
.status .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--muted); }
.status[data-state='ready'] .dot { background: var(--ok); }
.status[data-state='starting'] .dot, .status[data-state='connecting'] .dot { background: var(--warn); animation: pulse 1.2s infinite; }
.status[data-state='error'] .dot, .status[data-state='offline'] .dot { background: var(--rec); }
.rec .status .dot { background: var(--rec); animation: pulse 1s infinite; }
@keyframes pulse { 50% { opacity: .35; } }
.controls { display: flex; align-items: center; gap: 14px; flex-wrap: wrap; }
.mic { display: inline-flex; align-items: center; gap: 10px; padding: 12px 22px; border-radius: 999px;
border: 1px solid var(--line); background: var(--accent); color: #04121c; font-weight: 700; font-size: 16px; cursor: pointer; }
.mic.on { background: var(--rec); color: #fff; }
.mic:disabled { opacity: .45; cursor: not-allowed; }
.mic:focus-visible, button:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
.mic-icon { font-size: 14px; }
.meter { flex: 1 1 140px; min-width: 100px; height: 8px; border-radius: 4px; background: var(--panel); border: 1px solid var(--line); overflow: hidden; }
.meter .fill { height: 100%; background: linear-gradient(90deg, var(--ok), var(--warn), var(--rec)); transition: width .12s linear; }
.lang { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 13px; }
select { background: var(--panel); color: var(--text); border: 1px solid var(--line); border-radius: 8px; padding: 7px 10px; font-size: 14px; }
.notice { margin: 0; padding: 10px 12px; border-radius: 8px; border: 1px solid var(--line); background: var(--panel); font-size: 13.5px; line-height: 1.45; }
.notice[data-kind='error'] { border-color: #7f1d1d; background: #1f0f12; }
.notice[data-kind='warn'] { border-color: #78350f; background: #1c140a; }
.notice[data-kind='ok'] { border-color: #14532d; background: #0b1a11; }
.mics { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
.mic-pick { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 13px; flex: 1 1 260px; }
.mic-pick select { flex: 1; min-width: 0; }
.small { background: var(--panel); color: var(--text); border: 1px solid var(--line); border-radius: 8px; padding: 7px 12px; cursor: pointer; font-size: 13.5px; }
.small:disabled { opacity: .45; cursor: not-allowed; }
.small.ghost { background: transparent; }
.hint { margin: 0; color: var(--muted); font-size: 12.5px; }
code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12.5px; color: #c4b5fd; overflow-wrap: anywhere; }
.compose { flex: 1 1 auto; display: flex; flex-direction: column; gap: 6px; min-height: 260px; }
.draft { flex: 1 1 auto; width: 100%; min-height: 240px; resize: vertical; padding: 14px; border: 1px solid var(--line);
border-radius: 12px; background: var(--panel); color: var(--text); font: inherit; font-size: 16px; line-height: 1.6; }
.draft:focus { outline: 2px solid var(--accent); outline-offset: 1px; }
.draft::placeholder { color: #6b7280; }
.hearing { margin: 0; min-height: 1.5em; color: var(--accent); font-size: 14px; overflow-wrap: anywhere; }
.hearing .tag { display: inline-block; margin-right: 6px; padding: 1px 7px; border-radius: 999px; background: #0c2a3a;
color: var(--accent); font-size: 11px; text-transform: uppercase; letter-spacing: .04em; }
.count { color: var(--muted); font-size: 12.5px; }
.bottom { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; }
.bottom button, .debug button { background: var(--panel); color: var(--text); border: 1px solid var(--line); border-radius: 8px; padding: 8px 14px; cursor: pointer; font-size: 14px; }
.bottom button:disabled { opacity: .45; cursor: not-allowed; }
.bottom .ghost { background: transparent; }
.spacer { flex: 1; }
.info-btn { margin-left: auto; background: transparent; border: 1px solid var(--line); color: var(--muted); border-radius: 999px;
width: 30px; height: 30px; font-size: 16px; cursor: pointer; }
.info-btn:hover, .info-btn[aria-expanded='true'] { color: var(--accent); border-color: var(--accent); }
.info { border: 1px solid var(--line); border-radius: 12px; background: var(--panel); padding: 14px 16px; font-size: 14px; line-height: 1.55; }
.info h2 { margin: 0 0 6px; font-size: 16px; }
.info p { margin: 0 0 8px; }
.info ul { margin: 0 0 8px; padding-left: 18px; }
.info li { margin-bottom: 6px; }
.info .fine { color: var(--muted); font-size: 12.5px; margin: 0; }
.debug { border: 1px dashed var(--line); border-radius: 10px; padding: 12px; font-size: 13px; }
.debug dl { display: grid; grid-template-columns: 90px 1fr; gap: 6px 10px; margin: 0 0 10px; }
.debug dt { color: var(--muted); }
.debug dd { margin: 0; overflow-wrap: anywhere; }
</style>

View file

@ -24,6 +24,7 @@ import vue from '@vitejs/plugin-vue'
import cssInjectedByJsPlugin from 'vite-plugin-css-injected-by-js'
import { autoSpaDevPlugin } from './internal/plugin-gallery/dev-plugin.mjs'
import { stripShadowCssCommentsPlugin } from './internal/vite-plugin-strip-shadow-css-comments.mjs'
import { backendDevPlugin, devBackendProxy } from './internal/backend-dev-plugin.mjs'
import { fileURLToPath } from 'node:url'
import { resolve, dirname } from 'node:path'
import { readdirSync, statSync, existsSync } from 'node:fs'
@ -118,7 +119,9 @@ export default defineConfig(({ command, mode }) => {
// The dev server listens on THIS MACHINE ONLY. A reachable one serves the source tree, node_modules
// and the token baked into /; a remote viewer reaches it through an ssh tunnel (ssh -L), never a public
// port. `--host` on the command line still overrides this for a deliberate LAN session.
server: { host: '127.0.0.1' },
// Local plugin backends (backend/, e.g. rudder-engine) are started by backendDevPlugin and reached
// through these same-origin routes, so a plugin calls /api/<name>/… in dev exactly as on desktop.
server: { host: '127.0.0.1', proxy: devBackendProxy() },
define: {
// ★ NO 'unsafe-eval' (CRITIC-4, 2026-09-26). vue-i18n's default message compiler builds each message
// with `new Function`, so a partner with a strict Content-Security-Policy saw every translated string
@ -159,6 +162,7 @@ export default defineConfig(({ command, mode }) => {
vue({ customElement: true }),
cssInjectedByJsPlugin(),
autoSpaDevPlugin(env.VITE_MAPBOX_TOKEN || process.env.VITE_MAPBOX_TOKEN || ''),
backendDevPlugin(__dirname),
],
resolve: {
alias: { '@map': resolve(__dirname, 'library') },