From a3a7c4b5b608e4ee6f4335fe3edf17e046f30fec Mon Sep 17 00:00:00 2001 From: Doxa Maps Date: Mon, 28 Sep 2026 13:53:45 +0000 Subject: [PATCH] =?UTF-8?q?Rudder:=20Robot=20voice=20=E2=80=94=20speak=20e?= =?UTF-8?q?ach=20sentence=20in=20a=20synthetic=20voice,=20offline,=20for?= =?UTF-8?q?=20videos=20without=20your=20real=20voice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Robot voice: a checkbox and a voice dropdown. When on, every finished sentence is spoken back (sentences, not words: TTS needs context for its intonation). speak(text) is the single entry point, so a future conversational mode can reuse the same voice. - Providers, both on this computer only: Piper (offline neural TTS run by the engine on the CPU; six LibriTTS-R speakers offered as "Offline voice A–F") and the browser's own voices, restricted to localService === true (some browsers synthesise "online" voices on a server — never offered). - backend/rudder-engine/tts.go: POST /api/rudder/tts (text → audio/wav, 1–2000 characters, one synthesis at a time), GET /api/rudder/tts/voices, POST /api/rudder/duck. The spoken text is never logged, only its length. Piper 2023.11.14-2 (MIT) bundles ONNX Runtime 1.14.1: no telemetry client; a synthesis traced with strace made 0 network calls. ~150 MB RAM; a 10.7 s sentence rendered in 2.2 s on CPU. - While the robot speaks through loudspeakers, the engine feeds the transcriber silence, so it never transcribes itself (and that silence is not mistaken for a dead mic). With a Bluetooth headset or headphones it keeps listening — pausing would only lose the person's words. - setup-runtime.sh: Piper and the en_US-libritts_r-medium voice (CC BY 4.0), pinned by SHA-256. - Verified in Firefox against a dev server: dropdown lists A–F, each finished sentence is synthesised (200 audio/wav) between duck on/off, zero requests to any other host. --- backend/rudder-engine/main.go | 10 +- backend/rudder-engine/setup-runtime.sh | 15 ++ backend/rudder-engine/tts.go | 253 +++++++++++++++++++ plugin-designs/rudder-voice/README.md | 6 + plugin-designs/rudder-voice/rudder-voice.vue | 108 +++++++- 5 files changed, 389 insertions(+), 3 deletions(-) create mode 100644 backend/rudder-engine/tts.go diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go index 1f792d8bc..4c5dd3c78 100644 --- a/backend/rudder-engine/main.go +++ b/backend/rudder-engine/main.go @@ -186,6 +186,7 @@ type engine struct { device string devs *deviceWatcher nemoKey string + ducked atomic.Bool // the robot voice is speaking: feed the transcriber silence } func (e *engine) setState(s, errText string) { @@ -502,7 +503,13 @@ func (e *engine) startWith(lang, device string, src *recorderProc, sink func(eve // transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1, // the headset losing the mic side of the link. Re-opening the source restarts the audio // stream; the speech session and the transcript carry on untouched. - if peak == 0 && !browser { + // ROBOT VOICE SPEAKING (tts.go, /api/rudder/duck): the transcriber hears silence, so the robot's own + // voice through a loudspeaker is never transcribed back into the box — and that deliberate silence + // must not look like a dead mic link to the check below. + if e.ducked.Load() { + clear(frame) + silentSince = time.Time{} + } else if peak == 0 && !browser { if silentSince.IsZero() { silentSince = time.Now() } else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond { @@ -899,6 +906,7 @@ func main() { _ = json.NewDecoder(r.Body).Decode(&body) writeJSON(w, 200, e.micTest(body.Device)) }) + registerTTS(mux, e, newTTS(filepath.Dir(filepath.Dir(cfg.Model)), logs)) mux.HandleFunc("/api/rudder/log", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") _, _ = io.WriteString(w, logs.text()) diff --git a/backend/rudder-engine/setup-runtime.sh b/backend/rudder-engine/setup-runtime.sh index 57c6291b3..fed978335 100755 --- a/backend/rudder-engine/setup-runtime.sh +++ b/backend/rudder-engine/setup-runtime.sh @@ -22,6 +22,15 @@ MODEL_REV="ea30d66debe3" MODEL_FILE="nemotron-3.5-asr-streaming-0.6b.q8_0.gguf" MODEL_URL="https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b/resolve/${MODEL_REV}/${MODEL_FILE}" MODEL_SHA="3fc991d3badad7277c11030a7519832cddaf2057aafed6d4b25147e953a070b1" +# Robot voice (optional text-to-speech): Piper 2023.11.14-2 (MIT; ONNX Runtime 1.14.1 inside — no telemetry +# client, 0 network calls in a traced synthesis) and one CC BY 4.0 voice (LibriTTS-R, 904 speakers). +PIPER_URL="https://github.com/rhasspy/piper/releases/download/2023.11.14-2/piper_linux_x86_64.tar.gz" +PIPER_SHA="a50cb45f355b7af1f6d758c1b360717877ba0a398cc8cbe6d2a7a3a26e225992" +VOICE_REV="c10ece1aade47bb51c153c893d14e5bf8e5b7117" +VOICE="en_US-libritts_r-medium" +VOICE_BASE="https://huggingface.co/rhasspy/piper-voices/resolve/${VOICE_REV}/en/en_US/libritts_r/medium" +VOICE_SHA="10bb85e071d616fcf4071f369f1799d0491492ab3c5d552ec19fb548fac13195" +VOICE_JSON_SHA="b471dc60d2d8335e819c393d196d6fbf792817f40051257b269878505bc9afb3" say() { printf '\033[1;34m→\033[0m %s\n' "$*"; } die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; } @@ -46,6 +55,12 @@ fetch "$RT_URL" "$RUDDER_HOME/bin/$RT_NAME.tar.gz" "$RT_SHA" tar -xzf "$RUDDER_HOME/bin/$RT_NAME.tar.gz" -C "$RUDDER_HOME/bin" fetch "$MODEL_URL" "$RUDDER_HOME/models/$MODEL_FILE" "$MODEL_SHA" +mkdir -p "$RUDDER_HOME/tts" +fetch "$PIPER_URL" "$RUDDER_HOME/tts/piper_linux_x86_64.tar.gz" "$PIPER_SHA" +tar -xzf "$RUDDER_HOME/tts/piper_linux_x86_64.tar.gz" -C "$RUDDER_HOME/tts" +fetch "$VOICE_BASE/$VOICE.onnx" "$RUDDER_HOME/tts/$VOICE.onnx" "$VOICE_SHA" +fetch "$VOICE_BASE/$VOICE.onnx.json" "$RUDDER_HOME/tts/$VOICE.onnx.json" "$VOICE_JSON_SHA" + if command -v go >/dev/null; then say "building backend/bin/rudder-engine" (cd "$(dirname "$0")" && go build -o ../bin/rudder-engine .) diff --git a/backend/rudder-engine/tts.go b/backend/rudder-engine/tts.go new file mode 100644 index 000000000..3b593512a --- /dev/null +++ b/backend/rudder-engine/tts.go @@ -0,0 +1,253 @@ +// tts.go — Robot voice: speak what Rudder transcribes in a synthetic voice, so someone can record a video +// that uses dictation without their real voice in it. +// +// PROVIDER: Piper (rhasspy/piper 2023.11.14-2, MIT, archived — a frozen, pinnable release), run here in the +// engine rather than in the page: one small native binary on the CPU, the same audio in the Plugin Studio, +// the desktop webview and the Studio corner, and no WebAssembly runtime shipped through the page. +// +// OFFLINE, NO TELEMETRY (measured 2026-09-28): the bundled ONNX Runtime is 1.14.1, which carries none of +// the telemetry client later versions ship (0 "OneCollector"/events strings); a synthesis traced with strace +// made 0 network calls. The voice files are local and pinned by SHA-256 (setup-runtime.sh). +// +// PRIVACY: the text to speak is never logged — only its length. +// +// POST /api/rudder/tts {"text", "voice", "speaker", "speed"} → audio/wav +// GET /api/rudder/tts/voices the voices installed under RUDDER_HOME/tts +// POST /api/rudder/duck {"on": bool} — while on, the transcriber hears silence (the robot's own +// voice through a loudspeaker must not be transcribed back into the box) +package main + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type ttsVoice struct { + ID string `json:"id"` + Label string `json:"label"` + Speakers int `json:"speakers"` + SampleRate int `json:"sampleRate"` + model string +} + +type ttsEngine struct { + dir string // RUDDER_TTS_DIR (default RUDDER_HOME/tts): piper/ + .onnx + .onnx.json + piper string + logs *ring + mu sync.Mutex // one synthesis at a time: CPU is shared with speech-to-text +} + +func newTTS(base string, logs *ring) *ttsEngine { + dir := envOr("RUDDER_TTS_DIR", filepath.Join(base, "tts")) + bin := "piper" + if runtime.GOOS == "windows" { + bin = "piper.exe" + } + return &ttsEngine{dir: dir, piper: envOr("RUDDER_PIPER_BIN", filepath.Join(dir, "piper", bin)), logs: logs} +} + +// voices lists every .onnx with its .onnx.json next to it. +func (t *ttsEngine) voices() ([]ttsVoice, error) { + if _, err := os.Stat(t.piper); err != nil { + return nil, errors.New("Piper is not installed (run backend/rudder-engine/setup-runtime.sh)") + } + matches, _ := filepath.Glob(filepath.Join(t.dir, "*.onnx")) + var out []ttsVoice + for _, m := range matches { + b, err := os.ReadFile(m + ".json") + if err != nil { + continue + } + var cfg struct { + Audio struct { + SampleRate int `json:"sample_rate"` + } `json:"audio"` + NumSpeakers int `json:"num_speakers"` + } + if json.Unmarshal(b, &cfg) != nil || cfg.Audio.SampleRate == 0 { + continue + } + id := strings.TrimSuffix(filepath.Base(m), ".onnx") + out = append(out, ttsVoice{ID: id, Label: voiceLabel(id), Speakers: max(cfg.NumSpeakers, 1), SampleRate: cfg.Audio.SampleRate, model: m}) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + if len(out) == 0 { + return nil, errors.New("no Piper voice installed (run backend/rudder-engine/setup-runtime.sh)") + } + return out, nil +} + +// "en_US-libritts_r-medium" → "English (US) · libritts_r" +func voiceLabel(id string) string { + parts := strings.Split(id, "-") + if len(parts) < 2 { + return id + } + lang := map[string]string{"en_US": "English (US)", "en_GB": "English (UK)"}[parts[0]] + if lang == "" { + lang = parts[0] + } + return lang + " · " + parts[1] +} + +var errTTSInput = errors.New("bad request") + +func (t *ttsEngine) synthesize(ctx context.Context, text, voiceID string, speaker int, speed float64) ([]byte, error) { + text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " ")) + if text == "" || len([]rune(text)) > 2000 { + return nil, fmt.Errorf("%w: text must be 1–2000 characters", errTTSInput) + } + vs, err := t.voices() + if err != nil { + return nil, err + } + v := vs[0] + for _, x := range vs { + if x.ID == voiceID { + v = x + } + } + if speaker < 0 || speaker >= v.Speakers { + speaker = 0 + } + // Piper's length_scale is the inverse of speed (1.0 = normal; larger = slower). + if speed < 0.5 || speed > 2 { + speed = 1 + } + t.mu.Lock() + defer t.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, t.piper, "--model", v.model, "--output-raw", + "--speaker", strconv.Itoa(speaker), "--length_scale", strconv.FormatFloat(1/speed, 'f', 3, 64)) + cmd.Dir = filepath.Dir(t.piper) // its libraries and espeak-ng-data sit next to it + cmd.Stdin = strings.NewReader(text + "\n") + var pcm, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &pcm, &stderr + start := time.Now() + if err := cmd.Run(); err != nil { + t.logs.add("robot voice failed after %s: %v", time.Since(start).Round(time.Millisecond), err) + return nil, errors.New("the voice engine failed — see the engine log") + } + t.logs.add("robot voice: %d characters → %.1f s of audio in %s (voice %s, speaker %d)", + len([]rune(text)), float64(pcm.Len())/2/float64(v.SampleRate), time.Since(start).Round(time.Millisecond), v.ID, speaker) + return wav(pcm.Bytes(), v.SampleRate), nil +} + +// wav wraps 16-bit mono little-endian PCM in a WAV header. +func wav(pcm []byte, rate int) []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + b.WriteString("RIFF") + w(uint32(36 + len(pcm))) + b.WriteString("WAVEfmt ") + w(uint32(16)) + w(uint16(1)) // PCM + w(uint16(1)) // mono + w(uint32(rate)) + w(uint32(rate * 2)) + w(uint16(2)) + w(uint16(16)) + b.WriteString("data") + w(uint32(len(pcm))) + b.Write(pcm) + return b.Bytes() +} + +func registerTTS(mux *http.ServeMux, e *engine, t *ttsEngine) { + mux.HandleFunc("/api/rudder/tts/voices", func(w http.ResponseWriter, r *http.Request) { + vs, err := t.voices() + if err != nil { + writeJSON(w, 200, event{"available": false, "error": err.Error(), "voices": []ttsVoice{}}) + return + } + writeJSON(w, 200, event{"available": true, "provider": "piper", "voices": vs}) + }) + mux.HandleFunc("/api/rudder/tts", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, 405, event{"error": "POST only"}) + return + } + var body struct { + Text string `json:"text"` + Voice string `json:"voice"` + Speaker int `json:"speaker"` + Speed float64 `json:"speed"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&body); err != nil { + writeJSON(w, 400, event{"error": "expected JSON {text, voice, speaker, speed}"}) + return + } + audio, err := t.synthesize(r.Context(), body.Text, body.Voice, body.Speaker, body.Speed) + if err != nil { + code := 500 + if errors.Is(err, errTTSInput) { + code = 400 + } + writeJSON(w, code, event{"error": err.Error()}) + return + } + w.Header().Set("Content-Type", "audio/wav") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(audio) + }) + mux.HandleFunc("/api/rudder/duck", func(w http.ResponseWriter, r *http.Request) { + var body struct { + On bool `json:"on"` + } + _ = json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<10)).Decode(&body) + // Headphones hear the robot; the mic doesn't — so there is no echo to prevent, and pausing would only + // throw away whatever the person says while the robot talks. + if body.On && headphonesInUse() { + body.On = false + } + if e.ducked.Swap(body.On) != body.On { + e.logs.add("%s", map[bool]string{true: "listening paused: the robot voice is speaking", false: "listening resumed"}[body.On]) + } + writeJSON(w, 200, event{"ducked": body.On}) + }) +} + +// headphonesInUse: the default output is a Bluetooth device or a wired headphone port, so the robot voice +// cannot leak back into the microphone. Unknown → false (pause, the safe choice for loudspeakers). +func headphonesInUse() bool { + b, err := runPactl("get-default-sink") + if err != nil { + return false + } + sink := strings.TrimSpace(string(b)) + if strings.HasPrefix(sink, "bluez_output.") { + return true + } + b, err = runPactl("-f", "json", "list", "sinks") + if err != nil { + return false + } + var sinks []struct { + Name string `json:"name"` + ActivePort string `json:"active_port"` + } + if json.Unmarshal(b, &sinks) != nil { + return false + } + for _, s := range sinks { + if s.Name == sink { + return strings.Contains(strings.ToLower(s.ActivePort), "headphone") + } + } + return false +} diff --git a/plugin-designs/rudder-voice/README.md b/plugin-designs/rudder-voice/README.md index 06c50fe9f..3cea40173 100644 --- a/plugin-designs/rudder-voice/README.md +++ b/plugin-designs/rudder-voice/README.md @@ -14,6 +14,12 @@ talk to your agents instead of typing, with nothing leaving your computer. - **Microphones like a video-call app.** Pick the input, hot-plug updates, a 3-second mic test, and plain-language errors (muted, disconnected, dead Bluetooth mic link — which it tries to recover). +- **Robot voice (optional).** Settings → *Robot voice*: every sentence you say is spoken back in a + synthetic voice, so you can record a video that uses dictation without your real voice in it. Offline + neural voices (Piper, run by the engine on the CPU) or the browser's own on-device voices; nothing leaves + the computer. While the robot talks through loudspeakers, the engine stops listening, so it never + transcribes itself; with headphones it keeps listening. + ## Run it in the Plugin Studio ```bash diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue index 8e6246fa7..791720cff 100644 --- a/plugin-designs/rudder-voice/rudder-voice.vue +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -51,7 +51,7 @@ function load() { try { return JSON.parse(localStorage.getItem(STORE_KEY) || '{}') } catch { return {} } } function save() { - try { localStorage.setItem(STORE_KEY, JSON.stringify({ language: language.value, device: device.value, draft: draft.value })) } catch {} + try { localStorage.setItem(STORE_KEY, JSON.stringify({ language: language.value, device: device.value, draft: draft.value, robotOn: robotOn.value, robotVoice: robotVoice.value })) } catch {} } const saved = load() @@ -75,6 +75,91 @@ const copied = ref('') const showDebug = ref(false) const showInfo = ref(false) const showSettings = ref(false) + +// ─── ROBOT VOICE ─────────────────────────────────────────────────────────────────────────────────── +// Speaks each finished sentence back in a synthetic voice, so a video can use dictation without the +// speaker's real voice in it. Providers (all on this computer, nothing leaves it): +// piper:: the engine's Piper voice (tts.go) — offline neural TTS on the CPU +// system: the browser's own voices, ONLY those with localService === true (some +// browsers synthesise "online" voices on a server — never offered here) +// Sentences, not words: TTS needs a few words of context for its intonation; single words sound choppy. +// speak(text) is the one entry point, so a future conversational mode can reuse the same voice. +const robotOn = ref(saved.robotOn === true) +const robotVoice = ref(typeof saved.robotVoice === 'string' ? saved.robotVoice : '') +const robotOptions = ref([]) // [{ value, label }] +const robotError = ref('') +// A handful of LibriTTS-R speakers as distinct presets — 904 is too many to list. +const PIPER_SPEAKERS = [0, 12, 40, 118, 262, 500] + +async function loadRobotVoices() { + const opts = [] + robotError.value = '' + try { + const r = await (await fetch(apiBase.value + '/tts/voices')).json() + if (r.available) { + for (const v of r.voices) { + const speakers = v.speakers > 1 ? PIPER_SPEAKERS.filter((n) => n < v.speakers) : [0] + speakers.forEach((n, i) => opts.push({ value: `piper:${v.id}:${n}`, label: `Offline voice ${String.fromCharCode(65 + i)} · ${v.label}` })) + } + } else if (r.error) robotError.value = r.error + } catch { /* engine offline: system voices may still work */ } + const synth = typeof window !== 'undefined' && window.speechSynthesis + if (synth) { + for (const v of synth.getVoices().filter((x) => x.localService)) opts.push({ value: 'system:' + v.voiceURI, label: 'System voice · ' + v.name }) + } + robotOptions.value = opts + if (!opts.some((o) => o.value === robotVoice.value)) robotVoice.value = opts[0] ? opts[0].value : '' +} + +const sayQueue = [] +let saying = false +function speak(text) { + const t = (text || '').trim() + if (!t || !robotVoice.value) return + sayQueue.push(t) + if (!saying) drainSpeech() +} +// While the robot talks, the engine feeds the transcriber silence (unless the sound goes to headphones), +// so a loudspeaker never transcribes the robot back into the box. +async function drainSpeech() { + saying = true + post('/duck', { on: true }).catch(() => {}) + while (sayQueue.length) { + try { await speakOne(sayQueue.shift()) } catch (err) { robotError.value = String(err.message || err) } + } + await new Promise((r) => setTimeout(r, 250)) // let the room's echo die down + post('/duck', { on: false }).catch(() => {}) + saying = false +} +async function speakOne(text) { + const v = robotVoice.value + if (v.startsWith('piper:')) { + const [, voice, speaker] = v.split(':') + const r = await fetch(apiBase.value + '/tts', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, voice, speaker: Number(speaker) || 0 }), + }) + if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed') + const url = URL.createObjectURL(await r.blob()) + try { + await new Promise((done) => { const a = new Audio(url); a.onended = a.onerror = () => done(); a.play().catch(() => done()) }) + } finally { URL.revokeObjectURL(url) } + } else if (v.startsWith('system:') && window.speechSynthesis) { + const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService) + if (!voice) throw new Error('That system voice is no longer available') + await new Promise((done) => { + const u = new window.SpeechSynthesisUtterance(text) + u.voice = voice + u.onend = u.onerror = () => done() + window.speechSynthesis.speak(u) + }) + } +} +function toggleRobot() { + save() + if (robotOn.value) { loadRobotVoices(); speak('Robot voice on.') } + else { sayQueue.length = 0; window.speechSynthesis?.cancel?.() } +} const clientLog = ref([]) // ─── Microphones (the Jitsi device-selection pattern, served natively by the engine) ─── @@ -335,6 +420,7 @@ function onEvent(data) { case 'final': partial.value = '' onFinal(e.text) + if (robotOn.value) speak(e.text) break case 'recovering': say(e.message, 'warn') @@ -567,6 +653,9 @@ onMounted(() => { fetchStatus() connect() refreshDevices() + loadRobotVoices() + // System voices load asynchronously in most browsers. + if (window.speechSynthesis) window.speechSynthesis.addEventListener?.('voiceschanged', loadRobotVoices) window.addEventListener('keydown', onKey) }) onBeforeUnmount(() => { @@ -614,7 +703,7 @@ const statusText = computed(() => ({ {{ statusText }} - +
+ + +

{{ robotError }}

+

{{ testResult.ok ? '✓ ' : '✗ ' }}{{ testResult.verdict }}

@@ -762,6 +862,10 @@ const statusText = computed(() => ({