Rudder transcribes on a server: the page streams its own microphone to the engine
With RUDDER_AUDIO=browser the engine records no local microphone. The page captures the reader's mic, converts it to 16 kHz mono PCM in an AudioWorklet, and streams it over a WebSocket at /api/rudder/stream into the same speech session a local recorder feeds. The words come back on that socket to that reader only, never on the shared event stream. One session at a time; a second reader is told the engine is busy. The engine honours RUDDER_HOME, accepts the public host named in RUDDER_ALLOW_HOST, and refuses a socket from another origin. The dev proxy now carries WebSockets to backends.
This commit is contained in:
Doxa Maps
parent
744c3be317
commit
533ef04a9f
3 changed files with 245 additions and 47 deletions
|
|
@ -63,7 +63,8 @@ func envOr(k, d string) string {
|
|||
|
||||
func loadConfig() config {
|
||||
home, _ := os.UserHomeDir()
|
||||
base := filepath.Join(home, ".local", "share", "rudder")
|
||||
// RUDDER_HOME is where setup-runtime.sh installed the runtime and model (a server keeps it on a volume).
|
||||
base := envOr("RUDDER_HOME", filepath.Join(home, ".local", "share", "rudder"))
|
||||
return config{
|
||||
Port: envOr("RUDDER_PORT", "5193"),
|
||||
NemoPort: envOr("RUDDER_NEMO_PORT", "5194"),
|
||||
|
|
@ -153,6 +154,7 @@ func (h *hub) send(e event) {
|
|||
// ─── Engine: supervises nemo-speech serve and one recording session at a time ───────────────────
|
||||
|
||||
type engine struct {
|
||||
sink func(event) // the live browser session's own stream (RUDDER_AUDIO=browser); nil = the hub
|
||||
cfg config
|
||||
logs *ring
|
||||
hub *hub
|
||||
|
|
@ -181,9 +183,11 @@ func (e *engine) status() event {
|
|||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return event{
|
||||
"engine": e.state, "error": e.stateErr, "recording": e.recording, "recorder": e.recorder,
|
||||
// A browser session belongs to one reader; everyone else sees only that the engine is in use.
|
||||
"engine": e.state, "error": e.stateErr, "recording": e.recording && !browserAudio(), "inUse": e.recording, "recorder": e.recorder,
|
||||
"language": e.language, "defaultLanguage": e.cfg.Language, "device": e.device,
|
||||
"runtime": "NVIDIA NeMo-Speech.cpp (ggml, CPU)", "model": filepath.Base(e.cfg.Model),
|
||||
"audio": map[bool]string{true: "browser", false: "local"}[browserAudio()],
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -284,7 +288,16 @@ func recorderCmds(device string) [][]string {
|
|||
return cmds
|
||||
}
|
||||
|
||||
func (e *engine) start(lang, device string) error {
|
||||
// browserAudio: RUDDER_AUDIO=browser — the engine runs on a server (the cloud dev server), where there is
|
||||
// no microphone of the reader's to record. The page captures its own mic and streams 16 kHz s16le PCM to
|
||||
// /api/rudder/stream; that stream feeds the same pump and speech session as a local recorder.
|
||||
func browserAudio() bool { return os.Getenv("RUDDER_AUDIO") == "browser" }
|
||||
|
||||
func (e *engine) start(lang, device string) error { return e.startWith(lang, device, nil, nil) }
|
||||
|
||||
// startWith starts a session. src, when set, is the audio source (a browser stream); nil spawns a local recorder.
|
||||
func (e *engine) startWith(lang, device string, src *recorderProc, sink func(event)) error {
|
||||
browser := src != nil
|
||||
e.mu.Lock()
|
||||
if e.state != "ready" {
|
||||
e.mu.Unlock()
|
||||
|
|
@ -292,6 +305,9 @@ func (e *engine) start(lang, device string) error {
|
|||
}
|
||||
if e.recording {
|
||||
e.mu.Unlock()
|
||||
if browser {
|
||||
return errors.New("busy — someone else is recording on this server; try again when they stop")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if lang == "" {
|
||||
|
|
@ -316,22 +332,25 @@ func (e *engine) start(lang, device string) error {
|
|||
// mic link died 3-5 s into EVERY session that had speech (kernel: corrupted SCO packets); after
|
||||
// switching to CVSD it ran 9.5 minutes without a drop. CVSD is 8 kHz, and Nemotron's transcript of an
|
||||
// 8 kHz copy of the test clip differed by one word. RUDDER_KEEP_MSBC=1 opts out.
|
||||
if card := btCard(device); card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" {
|
||||
if card := btCard(device); !browser && card != "" && os.Getenv("RUDDER_KEEP_MSBC") == "" {
|
||||
if p := currentProfile(card); p == "headset-head-unit-msbc" {
|
||||
e.logs.add("bluetooth headset on mSBC — switching to CVSD before recording (stable voice link)")
|
||||
e.hub.send(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"})
|
||||
e.emit(event{"type": "recovering", "message": "Switching your Bluetooth headset to its stable voice mode…"})
|
||||
if err := switchProfile(card, e.logs); err != nil {
|
||||
e.logs.add("pre-switch to CVSD failed: %v", err)
|
||||
} else {
|
||||
e.hub.send(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."})
|
||||
e.emit(event{"type": "recovered", "message": "Headset is in its stable voice mode — go ahead and talk."})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
first, err := spawnRecorder(device, e.logs)
|
||||
if err != nil {
|
||||
ws.Close()
|
||||
return err
|
||||
first := src
|
||||
if first == nil {
|
||||
first, err = spawnRecorder(device, e.logs)
|
||||
if err != nil {
|
||||
ws.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
recName := first.name
|
||||
var recMu sync.Mutex
|
||||
|
|
@ -343,6 +362,7 @@ func (e *engine) start(lang, device string) error {
|
|||
|
||||
e.mu.Lock()
|
||||
e.recording, e.recorder, e.language, e.device = true, recName, lang, device
|
||||
e.sink = sink
|
||||
e.stopSession = func() {
|
||||
cancel()
|
||||
recMu.Lock()
|
||||
|
|
@ -351,19 +371,29 @@ func (e *engine) start(lang, device string) error {
|
|||
writeMu.Lock()
|
||||
_ = ws.WriteMessage(websocket.TextMessage, []byte(`{"type":"input_audio_buffer.commit"}`))
|
||||
writeMu.Unlock()
|
||||
select { // let the final words arrive
|
||||
// Let the final words arrive. A shared server's CPU can run behind real time (measured: a loaded
|
||||
// 8-core box finished an 11 s clip ~15 s after it ended), so a browser session waits longer.
|
||||
wait := 4 * time.Second
|
||||
if browser {
|
||||
wait = 30 * time.Second
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(4 * time.Second):
|
||||
case <-time.After(wait):
|
||||
}
|
||||
ws.Close()
|
||||
}
|
||||
e.mu.Unlock()
|
||||
e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device)))
|
||||
go watchKernelBluetooth(ctx, e.logs)
|
||||
e.hub.send(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device})
|
||||
if browser {
|
||||
e.logs.add("recording started: recorder=browser language=%s", lang)
|
||||
} else {
|
||||
e.logs.add("recording started: recorder=%s device=%q language=%s profile=%s", recName, device, lang, currentProfile(btCard(device)))
|
||||
go watchKernelBluetooth(ctx, e.logs)
|
||||
}
|
||||
e.emit(event{"type": "recording", "recording": true, "recorder": recName, "language": lang, "device": device})
|
||||
for _, d := range e.devs.current().Devices {
|
||||
if (d.ID == device || (device == "" && d.Default)) && d.Muted {
|
||||
e.hub.send(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."})
|
||||
e.emit(event{"type": "warning", "message": "Your microphone \"" + d.Label + "\" is muted in the system sound settings — unmute it or nothing will be heard."})
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -400,6 +430,12 @@ func (e *engine) start(lang, device string) error {
|
|||
}
|
||||
continue
|
||||
}
|
||||
if browser {
|
||||
e.logs.add("browser audio stream ended: %v", err)
|
||||
e.emit(event{"type": "error", "message": "The browser stopped sending microphone audio (the page was closed or lost its connection)."})
|
||||
go e.stop()
|
||||
return
|
||||
}
|
||||
// The recorder died on its own (bad device, sound server unreachable, device unplugged).
|
||||
// End the session so the UI stops showing "Listening", and say WHY in plain words.
|
||||
_ = r.cmd.Wait()
|
||||
|
|
@ -415,7 +451,7 @@ func (e *engine) start(lang, device string) error {
|
|||
if why != "" {
|
||||
msg += " (" + r.name + ": " + why + ")"
|
||||
}
|
||||
e.hub.send(event{"type": "error", "message": msg})
|
||||
e.emit(event{"type": "error", "message": msg})
|
||||
go e.stop()
|
||||
return
|
||||
}
|
||||
|
|
@ -434,7 +470,7 @@ func (e *engine) start(lang, device string) error {
|
|||
// transcribed, then literal zeros while parec stayed "healthy" — the old Rudder's BUG 1,
|
||||
// the headset losing the mic side of the link. Re-opening the source restarts the audio
|
||||
// stream; the speech session and the transcript carry on untouched.
|
||||
if peak == 0 {
|
||||
if peak == 0 && !browser {
|
||||
if silentSince.IsZero() {
|
||||
silentSince = time.Now()
|
||||
} else if !gaveUp && !resetting.Load() && time.Since(silentSince) > 1500*time.Millisecond {
|
||||
|
|
@ -443,7 +479,7 @@ func (e *engine) start(lang, device string) error {
|
|||
case recoveries == 1:
|
||||
// Step 1: reopen the recorder (cheap; fixes a stalled stream).
|
||||
e.logs.add("mic link dead (exact zeros 1.5s) — step 1: reopening the microphone")
|
||||
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"})
|
||||
e.emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone went silent — reconnecting it…"})
|
||||
if next, err := spawnRecorder(device, e.logs); err == nil {
|
||||
recMu.Lock()
|
||||
old := cur
|
||||
|
|
@ -458,12 +494,12 @@ func (e *engine) start(lang, device string) error {
|
|||
// "detect first", so it can't decide a dead link is fine.
|
||||
card := btCard(device)
|
||||
e.logs.add("mic still dead — step 2: forcing a new voice link: %s → headset-head-unit-cvsd", card)
|
||||
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"})
|
||||
e.emit(event{"type": "recovering", "attempt": recoveries, "message": "Microphone still silent — switching your Bluetooth headset to its more stable voice mode…"})
|
||||
resetting.Store(true)
|
||||
go func() {
|
||||
defer resetting.Store(false)
|
||||
if err := switchProfile(card, e.logs); err != nil {
|
||||
e.hub.send(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()})
|
||||
e.emit(event{"type": "warning", "message": "Couldn't switch the headset's voice mode: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if next, err := spawnRecorder(device, e.logs); err == nil {
|
||||
|
|
@ -478,7 +514,7 @@ func (e *engine) start(lang, device string) error {
|
|||
case recoveries == 3 && btResetPath() != "":
|
||||
// Step 3: full headset reset (disconnect → restart PipeWire → reconnect).
|
||||
e.logs.add("mic still dead — step 3: full headset reset via RUDDER_BT_RESET")
|
||||
e.hub.send(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"})
|
||||
e.emit(event{"type": "recovering", "attempt": recoveries, "message": "Still silent — fully resetting your Bluetooth headset (about 15 seconds)…"})
|
||||
resetting.Store(true)
|
||||
go func() {
|
||||
defer resetting.Store(false)
|
||||
|
|
@ -487,7 +523,7 @@ func (e *engine) start(lang, device string) error {
|
|||
return
|
||||
}
|
||||
if !ok {
|
||||
e.hub.send(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"})
|
||||
e.emit(event{"type": "warning", "message": "The headset reset didn't bring the mic back — is the headset on and in range?"})
|
||||
}
|
||||
if next, err := spawnRecorder(device, e.logs); err == nil {
|
||||
recMu.Lock()
|
||||
|
|
@ -505,7 +541,7 @@ func (e *engine) start(lang, device string) error {
|
|||
if btResetPath() == "" {
|
||||
msg += " (No headset-reset command is configured — see RUDDER_BT_RESET.)"
|
||||
}
|
||||
e.hub.send(event{"type": "warning", "message": msg})
|
||||
e.emit(event{"type": "warning", "message": msg})
|
||||
}
|
||||
silentSince = time.Now()
|
||||
}
|
||||
|
|
@ -514,7 +550,7 @@ func (e *engine) start(lang, device string) error {
|
|||
recovering.Store(false)
|
||||
healthySince = time.Now()
|
||||
e.logs.add("mic recovered after %d reconnect(s)", recoveries)
|
||||
e.hub.send(event{"type": "recovered", "message": "Microphone is back — keep talking."})
|
||||
e.emit(event{"type": "recovered", "message": "Microphone is back — keep talking."})
|
||||
}
|
||||
if recoveries > 0 && !healthySince.IsZero() && time.Since(healthySince) > 30*time.Second {
|
||||
recoveries, gaveUp, healthySince = 0, false, time.Time{}
|
||||
|
|
@ -530,14 +566,14 @@ func (e *engine) start(lang, device string) error {
|
|||
}
|
||||
if time.Since(lastLevel) > 150*time.Millisecond {
|
||||
lastLevel = time.Now()
|
||||
e.hub.send(event{"type": "level", "peak": float64(peak) / 32768})
|
||||
e.emit(event{"type": "level", "peak": float64(peak) / 32768})
|
||||
}
|
||||
writeMu.Lock()
|
||||
err := ws.WriteMessage(websocket.BinaryMessage, frame)
|
||||
writeMu.Unlock()
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
e.hub.send(event{"type": "error", "message": "speech server connection lost: " + err.Error()})
|
||||
e.emit(event{"type": "error", "message": "speech server connection lost: " + err.Error()})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -562,13 +598,13 @@ func (e *engine) start(lang, device string) error {
|
|||
case strings.HasSuffix(t, "input_audio_transcription.delta"):
|
||||
// Deltas APPEND to the utterance in progress; the server sends many empty ones.
|
||||
if text != "" {
|
||||
e.hub.send(event{"type": "partial", "text": text})
|
||||
e.emit(event{"type": "partial", "text": text})
|
||||
}
|
||||
case strings.HasSuffix(t, "input_audio_transcription.completed"):
|
||||
e.hub.send(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)})
|
||||
e.emit(event{"type": "final", "text": text, "at": time.Now().Format(time.RFC3339)})
|
||||
case t == "error":
|
||||
e.logs.add("speech server error: %s", string(msg))
|
||||
e.hub.send(event{"type": "error", "message": string(msg)})
|
||||
e.emit(event{"type": "error", "message": string(msg)})
|
||||
case t == "input_audio_buffer.committed":
|
||||
return
|
||||
default:
|
||||
|
|
@ -585,9 +621,13 @@ type recorderProc struct {
|
|||
out io.ReadCloser
|
||||
name string
|
||||
errBuf *bytes.Buffer
|
||||
closer io.Closer // a browser source: closing it ends the stream
|
||||
}
|
||||
|
||||
func (r *recorderProc) kill() {
|
||||
if r != nil && r.closer != nil {
|
||||
_ = r.closer.Close()
|
||||
}
|
||||
if r != nil && r.cmd != nil && r.cmd.Process != nil {
|
||||
_ = r.cmd.Process.Kill()
|
||||
go func() { _ = r.cmd.Wait() }()
|
||||
|
|
@ -642,7 +682,24 @@ func (e *engine) stop() {
|
|||
}
|
||||
stop()
|
||||
e.logs.add("recording stopped")
|
||||
e.hub.send(event{"type": "recording", "recording": false})
|
||||
e.emit(event{"type": "recording", "recording": false})
|
||||
e.mu.Lock()
|
||||
e.sink = nil
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// emit sends a session event. A browser session's events go back on that reader's own stream ONLY: on a
|
||||
// shared server the hub reaches every open Rudder page, and one person's words must never appear in
|
||||
// another's box.
|
||||
func (e *engine) emit(ev event) {
|
||||
e.mu.Lock()
|
||||
s := e.sink
|
||||
e.mu.Unlock()
|
||||
if s != nil {
|
||||
s(ev)
|
||||
return
|
||||
}
|
||||
e.hub.send(ev)
|
||||
}
|
||||
|
||||
func (e *engine) shutdown() {
|
||||
|
|
@ -670,6 +727,18 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
|
|||
// - CORS: only the desktop webview's origin (tauri://localhost, or http://tauri.localhost on Windows)
|
||||
// may read responses cross-origin. The Plugin Studio needs none: it reaches /api/rudder through the
|
||||
// dev server's same-origin proxy. RUDDER_ALLOW_ORIGIN adds one more exact origin (tests).
|
||||
//
|
||||
// allowedHost: RUDDER_ALLOW_HOST (comma-separated) names the public host of a server deployment, whose own
|
||||
// sign-in (oauth2-proxy) sits in front of the dev server that proxies /api/rudder here. Unset = loopback only.
|
||||
func allowedHost(host string) bool {
|
||||
for _, h := range strings.Split(os.Getenv("RUDDER_ALLOW_HOST"), ",") {
|
||||
if h = strings.TrimSpace(h); h != "" && h == host {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func guard(next http.Handler) http.Handler {
|
||||
extra := os.Getenv("RUDDER_ALLOW_ORIGIN")
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -677,7 +746,7 @@ func guard(next http.Handler) http.Handler {
|
|||
if i := strings.LastIndex(host, ":"); i >= 0 && !strings.HasSuffix(host, "]") {
|
||||
host = host[:i]
|
||||
}
|
||||
if host != "127.0.0.1" && host != "localhost" && host != "[::1]" {
|
||||
if host != "127.0.0.1" && host != "localhost" && host != "[::1]" && !allowedHost(host) {
|
||||
http.Error(w, "forbidden host", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
|
@ -721,6 +790,10 @@ func main() {
|
|||
Device string `json:"device"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if browserAudio() {
|
||||
writeJSON(w, 409, event{"error": "this engine takes the browser's microphone: stream to /api/rudder/stream"})
|
||||
return
|
||||
}
|
||||
if err := e.start(body.Language, body.Device); err != nil {
|
||||
e.logs.add("start failed: %v", err)
|
||||
writeJSON(w, 409, event{"error": err.Error()})
|
||||
|
|
@ -728,6 +801,53 @@ func main() {
|
|||
}
|
||||
writeJSON(w, 200, e.status())
|
||||
})
|
||||
// /api/rudder/stream (RUDDER_AUDIO=browser): a WebSocket of binary 16 kHz mono s16le PCM frames from the
|
||||
// page's microphone. Opening it starts a session, closing it stops one; the text comes back on /events.
|
||||
// The upgrader's default origin check (Origin must match Host) keeps other sites out.
|
||||
upgrader := websocket.Upgrader{ReadBufferSize: 8192}
|
||||
mux.HandleFunc("/api/rudder/stream", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !browserAudio() {
|
||||
writeJSON(w, 409, event{"error": "this engine records a local microphone (RUDDER_AUDIO is not browser)"})
|
||||
return
|
||||
}
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
pr, pw := io.Pipe()
|
||||
src := &recorderProc{out: pr, name: "browser", errBuf: &bytes.Buffer{}, closer: pr}
|
||||
var wmu sync.Mutex
|
||||
sink := func(ev event) {
|
||||
b, _ := json.Marshal(ev)
|
||||
wmu.Lock()
|
||||
_ = conn.WriteMessage(websocket.TextMessage, b)
|
||||
wmu.Unlock()
|
||||
}
|
||||
if err := e.startWith(r.URL.Query().Get("language"), "", src, sink); err != nil {
|
||||
e.logs.add("stream start failed: %v", err)
|
||||
_ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(4409, err.Error()))
|
||||
return
|
||||
}
|
||||
for {
|
||||
t, b, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if t != websocket.BinaryMessage {
|
||||
// {"type":"stop"}: end the session but keep the socket open, so the last words (sent after the
|
||||
// speech server's commit) still reach this reader; the page closes the socket after "recording false".
|
||||
if bytes.Contains(b, []byte(`"stop"`)) {
|
||||
go e.stop()
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := pw.Write(b); err != nil {
|
||||
break // the session stopped (its pipe was closed)
|
||||
}
|
||||
}
|
||||
e.stop()
|
||||
})
|
||||
mux.HandleFunc("/api/rudder/stop", func(w http.ResponseWriter, r *http.Request) {
|
||||
e.stop()
|
||||
writeJSON(w, 200, e.status())
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ export const DEV_BACKENDS = [
|
|||
|
||||
// The proxy table for vite's `server.proxy`. SSE (`/events`) streams through unbuffered.
|
||||
export function devBackendProxy() {
|
||||
return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false }]))
|
||||
// ws: a backend may take a WebSocket (rudder's /stream carries the browser's microphone on a server deployment).
|
||||
return Object.fromEntries(DEV_BACKENDS.map((b) => [b.route, { target: `http://127.0.0.1:${b.port}`, changeOrigin: false, ws: true }]))
|
||||
}
|
||||
|
||||
const REG = (globalThis.__dxDevBackends ||= { procs: new Map(), exitHooked: false, stopping: false })
|
||||
|
|
|
|||
|
|
@ -286,7 +286,9 @@ function applyStatus(s) {
|
|||
engineError.value = s.error || ''
|
||||
if (s.runtime) runtime.value = s.runtime
|
||||
if (s.model) model.value = s.model
|
||||
if (typeof s.recording === 'boolean') recording.value = s.recording
|
||||
if (s.audio) audioMode.value = s.audio
|
||||
// A browser session's state is this page's own; the server's shared status never reports it.
|
||||
if (typeof s.recording === 'boolean' && audioMode.value !== 'browser') recording.value = s.recording
|
||||
}
|
||||
|
||||
function connect() {
|
||||
|
|
@ -300,9 +302,20 @@ function connect() {
|
|||
}
|
||||
es = new window.EventSource(apiBase.value + '/events')
|
||||
es.onopen = () => logc('events connected')
|
||||
es.onmessage = (m) => {
|
||||
es.onmessage = (m) => onEvent(m.data)
|
||||
es.onerror = () => {
|
||||
engine.value = 'offline'
|
||||
if (audioMode.value !== 'browser') recording.value = false
|
||||
logc('events disconnected; retrying in 3s')
|
||||
try { es.close() } catch {}
|
||||
retry = setTimeout(connect, 3000)
|
||||
}
|
||||
}
|
||||
|
||||
// One vocabulary for both channels: the shared /events stream, and a browser session's own socket.
|
||||
function onEvent(data) {
|
||||
let e
|
||||
try { e = JSON.parse(m.data) } catch { return }
|
||||
try { e = JSON.parse(data) } catch { return }
|
||||
switch (e.type) {
|
||||
case 'status':
|
||||
applyStatus(e.status || e)
|
||||
|
|
@ -311,7 +324,7 @@ function connect() {
|
|||
break
|
||||
case 'recording':
|
||||
recording.value = !!e.recording
|
||||
if (!e.recording) { level.value = 0; flushPartial() }
|
||||
if (!e.recording) { level.value = 0; flushPartial(); closeStream() }
|
||||
break
|
||||
case 'partial':
|
||||
partial.value += e.text
|
||||
|
|
@ -342,14 +355,72 @@ function connect() {
|
|||
logc('engine error: ' + e.message)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// ─── BROWSER MICROPHONE (engine on a server, status.audio === 'browser') ─────────────────────────────
|
||||
// The engine cannot hear the reader's microphone from a server, so this page captures it, turns it into
|
||||
// 16 kHz mono 16-bit PCM in an AudioWorklet, and streams 100 ms frames to /api/rudder/stream. The words
|
||||
// come back on that same socket, to this reader only.
|
||||
const WORKLET = `class RudderPcm16k extends AudioWorkletProcessor {
|
||||
constructor () { super(); this.step = sampleRate / 16000; this.pos = 0; this.out = new Int16Array(1600); this.n = 0 }
|
||||
process (inputs) {
|
||||
const ch = inputs[0] && inputs[0][0]
|
||||
if (!ch) return true
|
||||
for (; this.pos < ch.length; this.pos += this.step) {
|
||||
const v = Math.max(-1, Math.min(1, ch[Math.floor(this.pos)]))
|
||||
this.out[this.n++] = v < 0 ? v * 32768 : v * 32767
|
||||
if (this.n === this.out.length) { this.port.postMessage(this.out.buffer, [this.out.buffer]); this.out = new Int16Array(1600); this.n = 0 }
|
||||
}
|
||||
this.pos -= ch.length
|
||||
return true
|
||||
}
|
||||
es.onerror = () => {
|
||||
engine.value = 'offline'
|
||||
recording.value = false
|
||||
logc('events disconnected; retrying in 3s')
|
||||
try { es.close() } catch {}
|
||||
retry = setTimeout(connect, 3000)
|
||||
}
|
||||
registerProcessor('rudder-pcm16k', RudderPcm16k)`
|
||||
const audioMode = ref('local')
|
||||
let sock = null, media = null, actx = null
|
||||
|
||||
async function startBrowser() {
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
throw new Error('This browser cannot share its microphone here (it needs a secure https:// page).')
|
||||
}
|
||||
media = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } })
|
||||
actx = new window.AudioContext()
|
||||
const url = URL.createObjectURL(new Blob([WORKLET], { type: 'text/javascript' }))
|
||||
await actx.audioWorklet.addModule(url)
|
||||
URL.revokeObjectURL(url)
|
||||
const node = new window.AudioWorkletNode(actx, 'rudder-pcm16k')
|
||||
const mute = actx.createGain()
|
||||
mute.gain.value = 0 // keeps the graph pulled without playing the mic back
|
||||
actx.createMediaStreamSource(media).connect(node)
|
||||
node.connect(mute).connect(actx.destination)
|
||||
const u = new URL(apiBase.value + '/stream', location.href)
|
||||
u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||
u.searchParams.set('language', language.value)
|
||||
sock = new window.WebSocket(u)
|
||||
sock.binaryType = 'arraybuffer'
|
||||
sock.onmessage = (m) => onEvent(m.data)
|
||||
sock.onclose = (c) => {
|
||||
if (c.code === 4409) say(c.reason || 'Someone else is recording on this server.', 'error')
|
||||
sock = null
|
||||
releaseMic()
|
||||
if (recording.value) { recording.value = false; level.value = 0; flushPartial() }
|
||||
}
|
||||
node.port.onmessage = (e) => { if (sock && sock.readyState === 1) sock.send(e.data) }
|
||||
}
|
||||
function releaseMic() {
|
||||
try { media && media.getTracks().forEach((t) => t.stop()) } catch {}
|
||||
try { actx && actx.close() } catch {}
|
||||
media = null
|
||||
actx = null
|
||||
}
|
||||
// Stop: the mic goes off at once; the socket stays open until the engine sends the last words.
|
||||
function stopBrowser() {
|
||||
releaseMic()
|
||||
if (sock && sock.readyState === 1) sock.send(JSON.stringify({ type: 'stop' }))
|
||||
else closeStream()
|
||||
}
|
||||
function closeStream() {
|
||||
if (sock) { try { sock.close() } catch {} }
|
||||
}
|
||||
|
||||
// Recording stopped mid-sentence: the streamed words are already in the box, so just let go of them.
|
||||
|
|
@ -379,7 +450,10 @@ async function toggle() {
|
|||
if (busy.value) return
|
||||
busy.value = true
|
||||
try {
|
||||
if (recording.value) {
|
||||
if (audioMode.value === 'browser') {
|
||||
if (recording.value) stopBrowser()
|
||||
else { say(''); await startBrowser() }
|
||||
} else if (recording.value) {
|
||||
await post('/stop')
|
||||
} else {
|
||||
say('')
|
||||
|
|
@ -387,7 +461,8 @@ async function toggle() {
|
|||
applyStatus(await post('/start', { language: language.value, device: device.value }))
|
||||
}
|
||||
} catch (err) {
|
||||
say(String(err.message || err), 'error')
|
||||
if (audioMode.value === 'browser') { releaseMic(); closeStream() }
|
||||
say(err && err.name === 'NotAllowedError' ? 'Microphone permission was refused — allow it for this site to record.' : String(err.message || err), 'error')
|
||||
logc('toggle failed: ' + err)
|
||||
} finally {
|
||||
busy.value = false
|
||||
|
|
@ -463,6 +538,8 @@ onMounted(() => {
|
|||
onBeforeUnmount(() => {
|
||||
clearTimeout(retry)
|
||||
try { es?.close() } catch {}
|
||||
releaseMic()
|
||||
closeStream()
|
||||
window.removeEventListener('keydown', onKey)
|
||||
})
|
||||
|
||||
|
|
@ -549,7 +626,7 @@ const statusText = computed(() => ({
|
|||
</label>
|
||||
</section>
|
||||
|
||||
<section v-if="!compact || showSettings" class="mics">
|
||||
<section v-if="(!compact || showSettings) && audioMode !== 'browser'" class="mics">
|
||||
<label class="mic-pick">
|
||||
<span>Microphone</span>
|
||||
<select v-model="device" :disabled="recording || testing" @change="save(); testResult = null">
|
||||
|
|
@ -567,7 +644,7 @@ const statusText = computed(() => ({
|
|||
</p>
|
||||
<p v-if="chosen && chosen.muted" class="notice" data-kind="warn">“{{ chosen.label }}” is muted in your system sound settings — Rudder won't hear anything until it's unmuted.</p>
|
||||
<p v-if="deviceError" class="notice" data-kind="error">{{ deviceError }}</p>
|
||||
<p v-else-if="!devices.length && engine !== 'offline'" class="notice" data-kind="warn">No microphone found. Connect a headset or turn on your computer's microphone in Sound settings.</p>
|
||||
<p v-else-if="!devices.length && engine !== 'offline' && audioMode !== 'browser'" class="notice" data-kind="warn">No microphone found. Connect a headset or turn on your computer's microphone in Sound settings.</p>
|
||||
<p v-if="offCards.length" class="hint">Also available but switched off in Sound settings: {{ offCards.join(', ') }}. Turn it on there to use it here.</p>
|
||||
|
||||
<p v-if="notice" class="notice" :data-kind="noticeKind">{{ notice }}</p>
|
||||
|
|
|
|||
Loading…
Reference in a new issue