Rudder: plain streaming by default; sentence detection only while the robot voice is on
Dictation was "doing better before" sentence detection was switched on for the robot voice: with asr.endpointing.enable the server ends an utterance at every ~0.8 s pause and the next one starts fresh. - Default (robot voice off): the speech server runs exactly as it shipped — plain streaming, no pause cutting. - Robot voice on (High performance mode): the server is restarted with sentence detection, which speaking sentence by sentence needs; turning the robot off restarts it without. A switch made mid-recording waits until Stop, so a recording is never cut. A planned restart is not reported as a crash. - Robot voice pipelined: the next sentence is synthesised while the current one plays, so the ~1 s gap between spoken sentences is gone (same work, overlapped). - Verified against a stand-in speech server (go -race): startup without the flag; robot on → restart with it; robot off while recording → deferred; after Stop → restart without; 0 data races.
This commit is contained in:
Doxa Maps
parent
21ebb4f264
commit
f9b4fed2cf
3 changed files with 102 additions and 41 deletions
|
|
@ -28,6 +28,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
|
|
@ -186,6 +187,8 @@ type engine struct {
|
|||
device string
|
||||
devs *deviceWatcher
|
||||
nemoKey string
|
||||
eou atomic.Bool // wanted: sentence detection (robot voice on)
|
||||
nemoEOU bool // what the running speech server was started with
|
||||
ducked atomic.Bool // the robot voice is speaking: feed the transcriber silence
|
||||
}
|
||||
|
||||
|
|
@ -224,12 +227,16 @@ func (e *engine) startNemo() {
|
|||
// program or web page that finds its port.
|
||||
cmd := exec.Command(e.cfg.NemoBin, "serve",
|
||||
"--host", "127.0.0.1", "--port", e.cfg.NemoPort,
|
||||
"--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui",
|
||||
// End-of-sentence detection is OFF in the server by default (asr.endpointing.enable: false), and
|
||||
// session.update's endpointing_ms only sets the pause length. Off, every recording came back as ONE
|
||||
// final when Stop was pressed (measured 2026-09-28: a minute of speech → one 437-character final), so
|
||||
// nothing reached the box or the robot voice sentence by sentence.
|
||||
"--asr.endpointing.enable=true", "--api-key", e.nemoKey)
|
||||
"--asr-model", e.cfg.Model, "--device", "cpu", "--no-ui", "--api-key", e.nemoKey)
|
||||
// ★ SENTENCE DETECTION ONLY FOR THE ROBOT VOICE (2026-09-28). With asr.endpointing.enable the server ends an
|
||||
// utterance at every ~0.8 s pause and the next one starts fresh; it was switched on for the robot voice, and
|
||||
// plain dictation was found to be "doing better before". So the DEFAULT is plain streaming — the server as
|
||||
// it shipped, no pause cutting, full context — and the flag is added only while the robot voice is on
|
||||
// (POST /api/rudder/mode), where speaking sentence by sentence needs it. Changing it restarts this server.
|
||||
eou := e.eou.Load()
|
||||
if eou {
|
||||
cmd.Args = append(cmd.Args, "--asr.endpointing.enable=true")
|
||||
}
|
||||
cmd.SysProcAttr = childAttr() // dies with us
|
||||
out, _ := cmd.StdoutPipe()
|
||||
cmd.Stderr = cmd.Stdout
|
||||
|
|
@ -239,8 +246,9 @@ func (e *engine) startNemo() {
|
|||
}
|
||||
e.mu.Lock()
|
||||
e.nemo = cmd
|
||||
e.nemoEOU = eou
|
||||
e.mu.Unlock()
|
||||
e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s", cmd.Process.Pid, e.cfg.NemoPort)
|
||||
e.logs.add("nemo-speech serve started (pid %d) on 127.0.0.1:%s, sentence detection %s", cmd.Process.Pid, e.cfg.NemoPort, map[bool]string{true: "ON (robot voice)", false: "off (plain streaming)"}[eou])
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
|
|
@ -257,7 +265,12 @@ func (e *engine) startNemo() {
|
|||
}()
|
||||
go func() {
|
||||
err := cmd.Wait()
|
||||
e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err))
|
||||
e.mu.Lock()
|
||||
replaced := e.nemo != cmd // a planned restart (applyEOU) already moved on — not a crash
|
||||
e.mu.Unlock()
|
||||
if !replaced {
|
||||
e.setState("error", fmt.Sprintf("nemo-speech exited: %v", err))
|
||||
}
|
||||
}()
|
||||
// Ready = the model list answers.
|
||||
go func() {
|
||||
|
|
@ -810,6 +823,32 @@ func firstString(m map[string]any, keys ...string) string {
|
|||
return ""
|
||||
}
|
||||
|
||||
// applyEOU restarts the speech server when the wanted sentence-detection setting differs from the running one.
|
||||
// Never mid-recording: then it waits, and stop() applies it.
|
||||
func (e *engine) applyEOU() {
|
||||
e.mu.Lock()
|
||||
if e.recording || e.nemoEOU == e.eou.Load() || e.nemo == nil {
|
||||
e.mu.Unlock()
|
||||
return
|
||||
}
|
||||
old := e.nemo
|
||||
e.nemo = nil
|
||||
e.mu.Unlock()
|
||||
e.logs.add("restarting the speech server: sentence detection %s", map[bool]string{true: "on", false: "off"}[e.eou.Load()])
|
||||
if old.Process != nil {
|
||||
_ = old.Process.Kill()
|
||||
}
|
||||
for i := 0; i < 30; i++ { // let the port free up before the new server binds it
|
||||
c, err := net.DialTimeout("tcp", "127.0.0.1:"+e.cfg.NemoPort, 200*time.Millisecond)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
c.Close()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
e.startNemo()
|
||||
}
|
||||
|
||||
func (e *engine) stop() {
|
||||
e.mu.Lock()
|
||||
stop := e.stopSession
|
||||
|
|
@ -821,7 +860,8 @@ func (e *engine) stop() {
|
|||
return
|
||||
}
|
||||
e.logs.add("recording stopping")
|
||||
stop() // sends "recording false" to the session's own reader once its last words are in
|
||||
stop() // sends "recording false" to the session's own reader once its last words are in
|
||||
go e.applyEOU() // a robot-voice switch made while recording is applied now
|
||||
}
|
||||
|
||||
func (e *engine) shutdown() {
|
||||
|
|
|
|||
|
|
@ -239,6 +239,9 @@ func registerTTS(mux *http.ServeMux, e *engine, t *ttsEngine) {
|
|||
if t.highPerf.Swap(body.Robot) != body.Robot {
|
||||
e.logs.add("mode: %s", map[bool]string{true: "high performance (robot voice on, experimental)", false: "thin-client (default)"}[body.Robot])
|
||||
}
|
||||
// Sentence detection follows the robot voice: plain streaming by default (main.go startNemo).
|
||||
e.eou.Store(body.Robot)
|
||||
go e.applyEOU()
|
||||
}
|
||||
writeJSON(w, 200, event{"mode": map[bool]string{true: "high-performance", false: "thin-client"}[t.highPerf.Load()]})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -214,47 +214,65 @@ function speak(text) {
|
|||
}
|
||||
// While the robot talks, the engine feeds the transcriber silence (unless the sound goes to headphones),
|
||||
// so a loudspeaker never transcribes the robot back into the box.
|
||||
// ★ PIPELINED (robot mode only): the next sentence's audio is synthesised WHILE the current one plays, so the
|
||||
// robot no longer pauses ~1 s between sentences to render the next — the same work, overlapped.
|
||||
async function drainSpeech() {
|
||||
saying = true
|
||||
post('/duck', { on: true }).catch(() => {})
|
||||
while (sayQueue.length) {
|
||||
try { await speakOne(sayQueue.shift()); robotError.value = '' } catch (err) { robotError.value = String(err.message || err) }
|
||||
const nextJob = () => {
|
||||
const text = sayQueue.shift()
|
||||
if (!text) return null
|
||||
const audio = robotVoice.value.startsWith('piper:') ? renderPiper(text) : null
|
||||
audio?.catch(() => {}) // awaited later; don't report it as unhandled meanwhile
|
||||
return { text, audio }
|
||||
}
|
||||
let job = nextJob()
|
||||
while (job) {
|
||||
const ahead = nextJob() // start rendering the next sentence now
|
||||
try {
|
||||
if (job.audio) await playAudio(await job.audio)
|
||||
else await speakSystem(job.text)
|
||||
robotError.value = ''
|
||||
} catch (err) { robotError.value = String(err.message || err) }
|
||||
job = ahead || nextJob()
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 250)) // let the room's echo die down
|
||||
post('/duck', { on: false }).catch(() => {})
|
||||
saying = false
|
||||
}
|
||||
async function speakOne(text) {
|
||||
async function renderPiper(text) {
|
||||
const [, voice, speaker] = robotVoice.value.split(':')
|
||||
const r = await fetch(apiBase.value + '/tts', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, voice, speaker: Number(speaker) || 0 }),
|
||||
})
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed')
|
||||
return URL.createObjectURL(await r.blob())
|
||||
}
|
||||
async function playAudio(url) {
|
||||
try {
|
||||
await new Promise((done, fail) => {
|
||||
const a = new Audio(url)
|
||||
a.onended = () => done()
|
||||
a.onerror = () => fail(new Error('The robot voice could not be played'))
|
||||
// A browser can refuse to play sound until you have clicked in the page; say so instead of staying silent.
|
||||
a.play().catch((err) => fail(new Error(err && err.name === 'NotAllowedError'
|
||||
? 'The browser blocked the robot voice — click anywhere in Rudder once, then it can speak'
|
||||
: 'The robot voice could not be played: ' + ((err && err.message) || err))))
|
||||
})
|
||||
} finally { URL.revokeObjectURL(url) }
|
||||
}
|
||||
async function speakSystem(text) {
|
||||
const v = robotVoice.value
|
||||
if (v.startsWith('piper:')) {
|
||||
const [, voice, speaker] = v.split(':')
|
||||
const r = await fetch(apiBase.value + '/tts', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, voice, speaker: Number(speaker) || 0 }),
|
||||
})
|
||||
if (!r.ok) throw new Error((await r.json().catch(() => ({}))).error || 'The robot voice failed')
|
||||
const url = URL.createObjectURL(await r.blob())
|
||||
try {
|
||||
await new Promise((done, fail) => {
|
||||
const a = new Audio(url)
|
||||
a.onended = () => done()
|
||||
a.onerror = () => fail(new Error('The robot voice could not be played'))
|
||||
// A browser can refuse to play sound until you have clicked in the page; say so instead of staying silent.
|
||||
a.play().catch((err) => fail(new Error(err && err.name === 'NotAllowedError'
|
||||
? 'The browser blocked the robot voice — click anywhere in Rudder once, then it can speak'
|
||||
: 'The robot voice could not be played: ' + ((err && err.message) || err))))
|
||||
})
|
||||
} finally { URL.revokeObjectURL(url) }
|
||||
} else if (v.startsWith('system:') && window.speechSynthesis) {
|
||||
const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService)
|
||||
if (!voice) throw new Error('That system voice is no longer available')
|
||||
await new Promise((done) => {
|
||||
const u = new window.SpeechSynthesisUtterance(text)
|
||||
u.voice = voice
|
||||
u.onend = u.onerror = () => done()
|
||||
window.speechSynthesis.speak(u)
|
||||
})
|
||||
}
|
||||
if (!v.startsWith('system:') || !window.speechSynthesis) return
|
||||
const voice = window.speechSynthesis.getVoices().find((x) => x.voiceURI === v.slice(7) && x.localService)
|
||||
if (!voice) throw new Error('That system voice is no longer available')
|
||||
await new Promise((done) => {
|
||||
const u = new window.SpeechSynthesisUtterance(text)
|
||||
u.voice = voice
|
||||
u.onend = u.onerror = () => done()
|
||||
window.speechSynthesis.speak(u)
|
||||
})
|
||||
}
|
||||
// ─── Thin-client by default; the robot voice is an EXPERIMENTAL opt-in ─────────────────────────────
|
||||
// Default ("thin-client mode"): no robot process runs and no voice list is even loaded — the whole budget
|
||||
|
|
|
|||
Loading…
Reference in a new issue