From 416cc06eac99fa8641d79f6b39b72197db17fcb3 Mon Sep 17 00:00:00 2001 From: Doxa Maps Date: Mon, 28 Sep 2026 14:13:19 +0000 Subject: [PATCH] Rudder speaks and fills the box sentence by sentence; Robot voice in a Settings menu, male by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Engine: start the speech server with --asr.endpointing.enable=true. End-of-sentence detection is OFF in NeMo-Speech.cpp by default, and session.update's endpointing_ms only sets the pause length, so every recording came back as ONE final when Stop was pressed (measured: a minute of speech → one 437-character final; the robot voice then spoke a minute late). Now each sentence is final ~1 s after it ends: in a Firefox run the robot spoke the first sentence 13 s before Stop. - Settings menu in the full layout too (⚙, closed by default, works at phone width): Language, Microphone and Robot voice live in it; Record and the level meter stay visible. - Robot voice: exactly ten presets, five male then five female, chosen by measured pitch (men ~92–108 Hz, women ~206–237 Hz); Male voice 1 is the default. The browser's own voices appear only as a fallback when the offline voices are missing, at most ten. A blocked or failed playback is shown instead of silence. --- backend/rudder-engine/main.go | 7 ++- plugin-designs/rudder-voice/rudder-voice.vue | 50 +++++++++++++++----- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go index 4c5dd3c78..c0d319823 100644 --- a/backend/rudder-engine/main.go +++ b/backend/rudder-engine/main.go @@ -224,7 +224,12 @@ func (e *engine) startNemo() { // program or web page that finds its port. cmd := exec.Command(e.cfg.NemoBin, "serve", "--host", "127.0.0.1", "--port", e.cfg.NemoPort, - "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey) + "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", + // End-of-sentence detection is OFF in the server by default (asr.endpointing.enable: false), and + // session.update's endpointing_ms only sets the pause length. Off, every recording came back as ONE + // final when Stop was pressed (measured 2026-09-28: a minute of speech → one 437-character final), so + // nothing reached the box or the robot voice sentence by sentence. + "--asr.endpointing.enable=true", "--api-key", e.nemoKey) cmd.SysProcAttr = childAttr() // dies with us out, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue index 791720cff..721c4c750 100644 --- a/plugin-designs/rudder-voice/rudder-voice.vue +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -88,8 +88,22 @@ const robotOn = ref(saved.robotOn === true) const robotVoice = ref(typeof saved.robotVoice === 'string' ? saved.robotVoice : '') const robotOptions = ref([]) // [{ value, label }] const robotError = ref('') -// A handful of LibriTTS-R speakers as distinct presets — 904 is too many to list. -const PIPER_SPEAKERS = [0, 12, 40, 118, 262, 500] +// A handful of LibriTTS-R speakers as presets — 904 is too many to list, and the voice file does not say +// which are men. MEASURED 2026-09-28: the same sentence synthesised for 61 speakers, median pitch per +// speaker (men ~85–150 Hz, women ~170–260 Hz); the default was speaker 0, a woman; a man was asked for. +// Exactly five of each, no more ("5 male, 5 female, don't overdo it"); the first is the default. +const PIPER_SPEAKERS = [ + { n: 90, label: 'Male voice 1' }, // 92 Hz + { n: 435, label: 'Male voice 2' }, // 96 Hz + { n: 870, label: 'Male voice 3' }, // 97 Hz + { n: 735, label: 'Male voice 4' }, // 99 Hz + { n: 375, label: 'Male voice 5' }, // 108 Hz + { n: 525, label: 'Female voice 1' }, // 206 Hz + { n: 450, label: 'Female voice 2' }, // 208 Hz + { n: 15, label: 'Female voice 3' }, // 216 Hz + { n: 285, label: 'Female voice 4' }, // 223 Hz + { n: 315, label: 'Female voice 5' }, // 237 Hz +] async function loadRobotVoices() { const opts = [] @@ -98,14 +112,16 @@ async function loadRobotVoices() { const r = await (await fetch(apiBase.value + '/tts/voices')).json() if (r.available) { for (const v of r.voices) { - const speakers = v.speakers > 1 ? PIPER_SPEAKERS.filter((n) => n < v.speakers) : [0] - speakers.forEach((n, i) => opts.push({ value: `piper:${v.id}:${n}`, label: `Offline voice ${String.fromCharCode(65 + i)} · ${v.label}` })) + const speakers = v.speakers > 1 ? PIPER_SPEAKERS.filter((x) => x.n < v.speakers) : [{ n: 0, label: 'Voice' }] + for (const x of speakers) opts.push({ value: `piper:${v.id}:${x.n}`, label: `${x.label} · offline` }) } } else if (r.error) robotError.value = r.error } catch { /* engine offline: system voices may still work */ } + // System voices are a FALLBACK only (the offline voices not installed), at most ten: Firefox on Linux alone + // lists dozens of on-device eSpeak voices, which would bury the ten real choices. const synth = typeof window !== 'undefined' && window.speechSynthesis - if (synth) { - for (const v of synth.getVoices().filter((x) => x.localService)) opts.push({ value: 'system:' + v.voiceURI, label: 'System voice · ' + v.name }) + if (synth && !opts.length) { + for (const v of synth.getVoices().filter((x) => x.localService).slice(0, 10)) opts.push({ value: 'system:' + v.voiceURI, label: 'System voice · ' + v.name }) } robotOptions.value = opts if (!opts.some((o) => o.value === robotVoice.value)) robotVoice.value = opts[0] ? opts[0].value : '' @@ -125,7 +141,7 @@ async function drainSpeech() { saying = true post('/duck', { on: true }).catch(() => {}) while (sayQueue.length) { - try { await speakOne(sayQueue.shift()) } catch (err) { robotError.value = String(err.message || err) } + try { await speakOne(sayQueue.shift()); robotError.value = '' } catch (err) { robotError.value = String(err.message || err) } } await new Promise((r) => setTimeout(r, 250)) // let the room's echo die down post('/duck', { on: false }).catch(() => {}) @@ -142,7 +158,15 @@ async function speakOne(text) { if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed') const url = URL.createObjectURL(await r.blob()) try { - await new Promise((done) => { const a = new Audio(url); a.onended = a.onerror = () => done(); a.play().catch(() => done()) }) + await new Promise((done, fail) => { + const a = new Audio(url) + a.onended = () => done() + a.onerror = () => fail(new Error('The robot voice could not be played')) + // A browser can refuse to play sound until you have clicked in the page; say so instead of staying silent. + a.play().catch((err) => fail(new Error(err && err.name === 'NotAllowedError' + ? 'The browser blocked the robot voice — click anywhere in Rudder once, then it can speak' + : 'The robot voice could not be played: ' + ((err && err.message) || err)))) + }) } finally { URL.revokeObjectURL(url) } } else if (v.startsWith('system:') && window.speechSynthesis) { const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService) @@ -718,7 +742,8 @@ const statusText = computed(() => ({

Voice-to-text for vibe coders · private · runs on this computer · no internet

- + +
{{ statusText }}
@@ -758,7 +783,7 @@ const statusText = computed(() => ({ {{ recording ? 'Stop' : 'Record' }} -