diff --git a/backend/rudder-engine/main.go b/backend/rudder-engine/main.go index b4da4b9d5..8ce37c51f 100644 --- a/backend/rudder-engine/main.go +++ b/backend/rudder-engine/main.go @@ -28,6 +28,7 @@ import ( "fmt" "io" "log" + "net" "net/http" "net/url" "os" @@ -186,6 +187,8 @@ type engine struct { device string devs *deviceWatcher nemoKey string + eou atomic.Bool // wanted: sentence detection (robot voice on) + nemoEOU bool // what the running speech server was started with ducked atomic.Bool // the robot voice is speaking: feed the transcriber silence } @@ -224,12 +227,16 @@ func (e *engine) startNemo() { // program or web page that finds its port. cmd := exec.Command(e.cfg.NemoBin, "serve", "--host", "127.0.0.1", "--port", e.cfg.NemoPort, - "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", - // End-of-sentence detection is OFF in the server by default (asr.endpointing.enable: false), and - // session.update's endpointing_ms only sets the pause length. Off, every recording came back as ONE - // final when Stop was pressed (measured 2026-09-28: a minute of speech → one 437-character final), so - // nothing reached the box or the robot voice sentence by sentence. - "--asr.endpointing.enable=true", "--api-key", e.nemoKey) + "--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey) + // ★ SENTENCE DETECTION ONLY FOR THE ROBOT VOICE (2026-09-28). With asr.endpointing.enable the server ends an + // utterance at every ~0.8 s pause and the next one starts fresh; it was switched on for the robot voice, and + // plain dictation was found to be "doing better before". So the DEFAULT is plain streaming — the server as + // it shipped, no pause cutting, full context — and the flag is added only while the robot voice is on + // (POST /api/rudder/mode), where speaking sentence by sentence needs it. Changing it restarts this server. + eou := e.eou.Load() + if eou { + cmd.Args = append(cmd.Args, "--asr.endpointing.enable=true") + } cmd.SysProcAttr = childAttr() // dies with us out, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout @@ -239,8 +246,9 @@ func (e *engine) startNemo() { } e.mu.Lock() e.nemo = cmd + e.nemoEOU = eou e.mu.Unlock() - e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort) + e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s, sentence detection %s", cmd.Process.Pid, e.cfg.NemoPort, map[bool]string{true: "ON (robot voice)", false: "off (plain streaming)"}[eou]) go func() { buf := make([]byte, 4096) for { @@ -257,7 +265,12 @@ func (e *engine) startNemo() { }() go func() { err := cmd.Wait() - e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err)) + e.mu.Lock() + replaced := e.nemo != cmd // a planned restart (applyEOU) already moved on — not a crash + e.mu.Unlock() + if !replaced { + e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err)) + } }() // Ready = the model list answers. go func() { @@ -810,6 +823,32 @@ func firstString(m map[string]any, keys ...string) string { return "" } +// applyEOU restarts the speech server when the wanted sentence-detection setting differs from the running one. +// Never mid-recording: then it waits, and stop() applies it. +func (e *engine) applyEOU() { + e.mu.Lock() + if e.recording || e.nemoEOU == e.eou.Load() || e.nemo == nil { + e.mu.Unlock() + return + } + old := e.nemo + e.nemo = nil + e.mu.Unlock() + e.logs.add("restarting the speech server: sentence detection %s", map[bool]string{true: "on", false: "off"}[e.eou.Load()]) + if old.Process != nil { + _ = old.Process.Kill() + } + for i := 0; i < 30; i++ { // let the port free up before the new server binds it + c, err := net.DialTimeout("tcp", "127.0.0.1:"+e.cfg.NemoPort, 200*time.Millisecond) + if err != nil { + break + } + c.Close() + time.Sleep(100 * time.Millisecond) + } + e.startNemo() +} + func (e *engine) stop() { e.mu.Lock() stop := e.stopSession @@ -821,7 +860,8 @@ func (e *engine) stop() { return } e.logs.add("recording stopping") - stop() // sends "recording false" to the session's own reader once its last words are in + stop() // sends "recording false" to the session's own reader once its last words are in + go e.applyEOU() // a robot-voice switch made while recording is applied now } func (e *engine) shutdown() { diff --git a/backend/rudder-engine/tts.go b/backend/rudder-engine/tts.go index 5c5e01148..7c1674517 100644 --- a/backend/rudder-engine/tts.go +++ b/backend/rudder-engine/tts.go @@ -239,6 +239,9 @@ func registerTTS(mux *http.ServeMux, e *engine, t *ttsEngine) { if t.highPerf.Swap(body.Robot) != body.Robot { e.logs.add("mode: %s", map[bool]string{true: "high performance (robot voice on, experimental)", false: "thin-client (default)"}[body.Robot]) } + // Sentence detection follows the robot voice: plain streaming by default (main.go startNemo). + e.eou.Store(body.Robot) + go e.applyEOU() } writeJSON(w, 200, event{"mode": map[bool]string{true: "high-performance", false: "thin-client"}[t.highPerf.Load()]}) }) diff --git a/plugin-designs/rudder-voice/rudder-voice.vue b/plugin-designs/rudder-voice/rudder-voice.vue index f8823048b..485a2eff6 100644 --- a/plugin-designs/rudder-voice/rudder-voice.vue +++ b/plugin-designs/rudder-voice/rudder-voice.vue @@ -214,47 +214,65 @@ function speak(text) { } // While the robot talks, the engine feeds the transcriber silence (unless the sound goes to headphones), // so a loudspeaker never transcribes the robot back into the box. +// ★ PIPELINED (robot mode only): the next sentence's audio is synthesised WHILE the current one plays, so the +// robot no longer pauses ~1 s between sentences to render the next — the same work, overlapped. async function drainSpeech() { saying = true post('/duck', { on: true }).catch(() => {}) - while (sayQueue.length) { - try { await speakOne(sayQueue.shift()); robotError.value = '' } catch (err) { robotError.value = String(err.message || err) } + const nextJob = () => { + const text = sayQueue.shift() + if (!text) return null + const audio = robotVoice.value.startsWith('piper:') ? renderPiper(text) : null + audio?.catch(() => {}) // awaited later; don't report it as unhandled meanwhile + return { text, audio } + } + let job = nextJob() + while (job) { + const ahead = nextJob() // start rendering the next sentence now + try { + if (job.audio) await playAudio(await job.audio) + else await speakSystem(job.text) + robotError.value = '' + } catch (err) { robotError.value = String(err.message || err) } + job = ahead || nextJob() } await new Promise((r) => setTimeout(r, 250)) // let the room's echo die down post('/duck', { on: false }).catch(() => {}) saying = false } -async function speakOne(text) { +async function renderPiper(text) { + const [, voice, speaker] = robotVoice.value.split(':') + const r = await fetch(apiBase.value + '/tts', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, voice, speaker: Number(speaker) || 0 }), + }) + if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed') + return URL.createObjectURL(await r.blob()) +} +async function playAudio(url) { + try { + await new Promise((done, fail) => { + const a = new Audio(url) + a.onended = () => done() + a.onerror = () => fail(new Error('The robot voice could not be played')) + // A browser can refuse to play sound until you have clicked in the page; say so instead of staying silent. + a.play().catch((err) => fail(new Error(err && err.name === 'NotAllowedError' + ? 'The browser blocked the robot voice — click anywhere in Rudder once, then it can speak' + : 'The robot voice could not be played: ' + ((err && err.message) || err)))) + }) + } finally { URL.revokeObjectURL(url) } +} +async function speakSystem(text) { const v = robotVoice.value - if (v.startsWith('piper:')) { - const [, voice, speaker] = v.split(':') - const r = await fetch(apiBase.value + '/tts', { - method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text, voice, speaker: Number(speaker) || 0 }), - }) - if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed') - const url = URL.createObjectURL(await r.blob()) - try { - await new Promise((done, fail) => { - const a = new Audio(url) - a.onended = () => done() - a.onerror = () => fail(new Error('The robot voice could not be played')) - // A browser can refuse to play sound until you have clicked in the page; say so instead of staying silent. - a.play().catch((err) => fail(new Error(err && err.name === 'NotAllowedError' - ? 'The browser blocked the robot voice — click anywhere in Rudder once, then it can speak' - : 'The robot voice could not be played: ' + ((err && err.message) || err)))) - }) - } finally { URL.revokeObjectURL(url) } - } else if (v.startsWith('system:') && window.speechSynthesis) { - const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService) - if (!voice) throw new Error('That system voice is no longer available') - await new Promise((done) => { - const u = new window.SpeechSynthesisUtterance(text) - u.voice = voice - u.onend = u.onerror = () => done() - window.speechSynthesis.speak(u) - }) - } + if (!v.startsWith('system:') || !window.speechSynthesis) return + const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService) + if (!voice) throw new Error('That system voice is no longer available') + await new Promise((done) => { + const u = new window.SpeechSynthesisUtterance(text) + u.voice = voice + u.onend = u.onerror = () => done() + window.speechSynthesis.speak(u) + }) } // ─── Thin-client by default; the robot voice is an EXPERIMENTAL opt-in ───────────────────────────── // Default ("thin-client mode"): no robot process runs and no voice list is even loaded — the whole budget