From 0b44d7e1bf8d80c6907bf9f30ee578940ed26564 Mon Sep 17 00:00:00 2001 From: Iulian Date: Wed, 9 Sep 2026 19:06:06 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20live=20macOS=20dictation=20=E2=80=94=20?= =?UTF-8?q?push-to-talk=20mic=20into=20the=20realtime=20ASR=20session?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /dictate + ctrl+shift+d toggle one shared session: press to start, press again to stop, flush and paste into the prompt (a terminal gives no key-release event for hold-to-talk without the Kitty keyboard protocol) - client.recordMic(): spawns ffmpeg AVFoundation capture and appends s16le frames as they arrive (no pacer — the mic is already 1x paced); stop() = commit + the same quiescence drain transcribe() uses, since is_last_result never flips true - enumerates mics by slicing the "AVFoundation audio devices:" block off stderr (ffmpeg prints the table there, then fails opening the dummy input) - --device takes an index or name, --lang / ASR_DICTATE_LANGUAGE, --submit sends instead of pasting; asr_health now reports the mic inputs it enumerated - capture death surfaces via the emptyGraceMs path with the TCC hint (System Settings → Privacy & Security → Microphone) when ffmpeg exits non-zero at 0 bytes - env.sh sources the gitignored .env so a shell gets ASR_NIM_URL / ASR_API_KEY --- .gitignore | 1 + env.sh | 8 ++ extensions/asr.ts | 110 +++++++++++++++++++++++++- src/client.ts | 192 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 307 insertions(+), 4 deletions(-) create mode 100755 env.sh diff --git a/.gitignore b/.gitignore index 30cce9c..b1730d5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ yarn-error.log *.mp3 *.mp4 asr.json +.env .DS_Store *.log diff --git a/env.sh b/env.sh new file mode 100755 index 0000000..949e08e --- /dev/null +++ b/env.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Load pi-asr local env (secrets live in the gitignored .env). +# Run from the repo root: source ./env.sh +[ -f ./.env ] && { set -a; . ./.env; set +a; } || { echo "[pi-asr] no ./.env found (run this from the repo root)"; return 1 2>/dev/null || exit 1; } +_k="${ASR_API_KEY:-}" +[ -n "$_k" ] && _m="set (${#_k} chars, ${_k:0:5}...)" || _m="UNSET" +echo "[pi-asr] ASR_NIM_URL=${ASR_NIM_URL:-} key=${_m}" +unset _k _m diff --git a/extensions/asr.ts b/extensions/asr.ts index 83540ad..d69aa67 100644 --- a/extensions/asr.ts +++ b/extensions/asr.ts @@ -19,8 +19,8 @@ import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@mariozech import { Text } from "@mariozechner/pi-tui"; import { Type } from "@sinclair/typebox"; import { - DEFAULT_BASE_URL, RealtimeAsrClient, hasFfmpeg, loadPcm16, - type TranscriptResult, type WordInfo, + DEFAULT_BASE_URL, RealtimeAsrClient, hasFfmpeg, loadPcm16, listAudioInputs, + type MicRecording, type TranscriptResult, type WordInfo, } from "../src/client.js"; interface TranscribeParams { @@ -205,11 +205,12 @@ export default function asrExtension(pi: ExtensionAPI) { async execute(_id, _params, _signal, _onUpdate, ctx): Promise> { const client = new RealtimeAsrClient(); - const [live, ready, minted, ffmpeg] = await Promise.all([ + const [live, ready, minted, ffmpeg, inputs] = await Promise.all([ client.health("live", 10000).catch((e: any) => ({ error: e?.message ?? String(e) })), client.health("ready", 10000).catch((e: any) => ({ error: e?.message ?? String(e) })), client.mint(10000).catch((e: any) => ({ error: e?.message ?? String(e) })), hasFfmpeg(), + listAudioInputs().catch(() => [] as { index: number; name: string }[]), ]); const ok = live?.status === "live" && ready?.status === "ready" && !minted?.error; const key = client.cfg.apiKey; @@ -229,10 +230,11 @@ export default function asrExtension(pi: ExtensionAPI) { ` ws token single-use, expires in ${secs}s (mint immediately before each connect)`, ].join("\n"), `ffmpeg ${ffmpeg ? "available (any container/codec accepted)" : "MISSING — only 16kHz mono wav files will work"}`, + `mic inputs ${inputs.length ? inputs.map(d => `[${d.index}] ${d.name}`).join(", ") : ffmpeg ? "none enumerated (check Mic permission / AVFoundation build)" : "n/a (ffmpeg missing)"}`, `note the realtime decoder runs at ~1x realtime; a 10min file takes ~10min (speed=2 halves it)`, ].join("\n"); if (!ok) throw new Error(`ASR server is not usable:\n${text}`); - return { content: [{ type: "text", text }], details: { ok, live, ready, minted, ffmpeg } }; + return { content: [{ type: "text", text }], details: { ok, live, ready, minted, ffmpeg, inputs } }; }, renderResult(result: any, _options: any, theme: any) { @@ -300,4 +302,104 @@ export default function asrExtension(pi: ExtensionAPI) { } }, }); + + /* ------------------------------------------------------------- /dictate -- * + * macOS live dictation: ffmpeg AVFoundation mic capture -> the same realtime WS client. + * Toggle semantics (press to start, press again to stop + flush) because a plain terminal + * gives no key-release event for hold-to-talk without the Kitty keyboard protocol. + * Shared module state so the /dictate command and the ctrl+shift+d shortcut drive one session. + */ + let active: { rec: MicRecording; submit: boolean } | null = null; + let starting = false; + + async function resolveDevice(d?: string | null): Promise { + if (!d) return null; + if (/^\d+$/.test(d)) { + const hit = (await listAudioInputs()).find(x => x.index === +d); + if (!hit) throw new Error(`no audio input at index ${d} — list them with: /dictate devices`); + return hit.name; + } + return d; + } + + async function toggleDictation(ctx: ExtensionContext, opts: { device?: string | null; language?: string; submit?: boolean } = {}): Promise { + if (!ctx.hasUI) { ctx.ui.notify("dictation needs the interactive TUI", "error"); return; } + + // second press while live => stop, flush, paste the final transcript. + if (active) { + const { rec, submit } = active; active = null; + ctx.ui.setStatus("asr", undefined); + try { + const res = await rec.stop(); + const text = res.text.trim(); + if (rec.error) { ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; } + if (!text) { ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; } + if (submit) pi.sendUserMessage(text); + else { + const cur = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, ""); + ctx.ui.setEditorText(cur ? `${cur} ${text}` : text); + } + ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "prompt"}`, "info"); + } catch (e: any) { + rec.cancel(); + ctx.ui.notify(`dictation failed: ${e?.message ?? e}`, "error"); + } + return; + } + + if (starting) return; // ignore a double-press during mint+handshake + starting = true; + const language = opts.language ?? process.env.ASR_DICTATE_LANGUAGE ?? "en-US"; + let device: string | null; + try { device = await resolveDevice(opts.device); } + catch (e: any) { starting = false; ctx.ui.notify(`dictation: ${e?.message ?? e}`, "error"); return; } + + const client = new RealtimeAsrClient({ language, punctuation: true }); + ctx.ui.setStatus("asr", "● listening …"); + let rec: MicRecording; + try { + rec = await client.recordMic({ + device, + onDelta: (t) => { if (active) ctx.ui.setStatus("asr", `● ${t || "…listening"}`); }, + }); + } catch (e: any) { + starting = false; ctx.ui.setStatus("asr", undefined); + ctx.ui.notify(`could not start mic capture: ${e?.message ?? e}\n(ffmpeg on PATH w/ AVFoundation; grant the terminal Microphone access in System Settings → Privacy & Security)`, "error"); + return; + } + starting = false; + active = { rec, submit: !!opts.submit }; + ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — press ctrl+shift+d or /dictate again to stop`, "info"); + } + + pi.registerShortcut("ctrl+shift+d", { + description: "Toggle macOS live dictation (press to start, press again to stop and paste into the prompt)", + handler: async (ctx) => { await toggleDictation(ctx); }, + }); + + pi.registerCommand("dictate", { + description: "Live macOS dictation (mic → realtime ASR): /dictate [stop|devices] [--device ] [--lang xx-YY] [--submit]", + handler: async (args: string, ctx: ExtensionContext) => { + const toks = args.trim().split(/\s+/).filter(Boolean); + const head = (toks[0] ?? "").toLowerCase(); + if (head === "devices") { + const ds = await listAudioInputs(); + ctx.ui.notify(ds.length ? ds.map(d => `[${d.index}] ${d.name}`).join("\n") : "no audio inputs found (ffmpeg w/ AVFoundation? Mic permission?)", "info"); + return; + } + if (head === "stop" || head === "cancel") { + if (!active) { ctx.ui.notify("not currently recording", "info"); return; } + await toggleDictation(ctx); + return; + } + let device: string | undefined, language: string | undefined, submit = false; + for (let i = 0; i < toks.length; i++) { + const t = toks[i]; + if (t === "--device") device = toks[++i]; + else if (t === "--lang") language = toks[++i]; + else if (t === "--submit") submit = true; + } + await toggleDictation(ctx, { device, language, submit }); + }, + }); } diff --git a/src/client.ts b/src/client.ts index 528cfe1..74063db 100644 --- a/src/client.ts +++ b/src/client.ts @@ -277,6 +277,130 @@ export class RealtimeAsrClient { return { handle, minted }; } + /** + * Push-to-talk live dictation on macOS: spawn ffmpeg AVFoundation mic capture and stream + * s16le frames into a realtime ASR session as they arrive (no artificial pacer — the mic is + * already real-time paced). Returns a handle; call `.stop()` to commit + flush + get the final + * transcript, or `.cancel()` to drop it. `onDelta` drives a live status line. + * + * The completion drain reuses the same quiescence logic as transcribe(): the server emits + * cumulative `.delta`s and a words-bearing then words-less `.completed` per segment, and + * `is_last_result` never flips true, so after commit we wait until no new segment for quietMs. + */ + async recordMic(opts: { + device?: string | null; frameMs?: number; + quietMs?: number; emptyGraceMs?: number; finalTimeout?: number; tailMs?: number; + sampleRate?: number; channels?: number; session?: Record; + onDelta?: (t: string) => void; onLevel?: (peak: number) => void; onStart?: (d: { device: string | null }) => void; + signal?: AbortSignal; + } = {}): Promise { + this.requireKey(); + const sampleRate = opts.sampleRate ?? 16000, channels = opts.channels ?? 1; + const frameBytes = Math.max(320, Math.round((sampleRate * (opts.frameMs ?? 100)) / 1000) * 2 * channels); + const minted = await this.mint(); + if (opts.signal?.aborted) throw new Error("cancelled"); + const handle = await this.open({ token: minted.client_secret?.value ?? null }); + + handle.send({ type: "transcription_session.update", session: this.sessionConfig(minted, opts.session ?? {}) }); + await handle.expect(EV.sessionUpdated, 20000); + + const deltas: string[] = []; + const finals: Map = new Map(); + const failures: unknown[] = []; + const offs = [ + this.on(EV.delta, e => { const t = e.delta ?? ""; deltas.push(t); rec.partial = t; opts.onDelta?.(t); }), + this.on(EV.completed, (e: any) => { + const key = e.item_id ?? "_"; + const prev = finals.get(key); + const nw = e.words_info?.words?.length ?? 0, pw = prev?.words_info?.words?.length ?? 0; + finals.set(key, prev && pw > nw && prev.transcript ? { ...e, words_info: prev.words_info } : e); + }), + this.on(EV.failed, (e: any) => failures.push(e)), + ]; + + let pending: Buffer[] = [], carry: Buffer = Buffer.alloc(0), total = 0, peak = 0, cancelled = false; + let drainP: Promise | null = null; + const rec: MicRecording = { + partial: "", bytes: 0, seconds: 0, level: 0, error: null, + stop: () => (drainP ??= drain()), + cancel: () => { cancelled = true; try { handle.send?.({ type: "input_audio_buffer.clear" }); } catch {} for (const f of offs) f(); handle.close(); }, + }; + + function flush(force = false): void { + let buf = carry.length ? Buffer.concat([carry, ...pending]) : Buffer.concat(pending); + pending = []; + if (!force && buf.length < frameBytes) { carry = buf; return; } + carry = Buffer.alloc(0); + for (let i = 0; i < buf.length; i += frameBytes) { + handle.send({ type: "input_audio_buffer.append", audio: buf.subarray(i, Math.min(i + frameBytes, buf.length)).toString("base64") }); + } + } + + const child = spawn("ffmpeg", micArgs(opts.device, { sampleRate, channels }), { stdio: ["ignore", "pipe", "pipe"] }); + let capErr = ""; + child.stdout.on("data", (b: Buffer) => { + total += b.length; rec.bytes = total; rec.seconds = +(total / (sampleRate * 2 * channels)).toFixed(2); + const lv = rms16(b); if (lv > peak) { peak = lv; rec.level = lv; } opts.onLevel?.(lv); + pending.push(b); flush(); + }); + child.stderr.on("data", b => { capErr += b.toString(); }); + let capDied: (e: Error) => void = () => {}; + const capEnded = new Promise((_, rej) => { capDied = rej; }); + child.on("error", e => { const m = (e as any).code === "ENOENT" ? "ffmpeg is not installed / not on PATH" : e.message; rec.error = m; capDied(new Error(m)); }); + child.on("close", code => { if (code !== 0 && total === 0) { const m = `mic capture failed (ffmpeg exit ${code}). ${capErr.slice(-300)}\nIf macOS blocked the microphone: System Settings → Privacy & Security → Microphone → allow your terminal app.`; rec.error = m; capDied(new Error(m)); } }); + + opts.onStart?.({ device: opts.device ?? null }); + + function stopCapture(): void { try { child.kill("SIGINT"); } catch {} } + + async function drain(): Promise { + stopCapture(); + // let stdout finish flushing the tail, then force-send the last partial frame + await Promise.race([new Promise(r => child.once("close", r)), sleep(400), capEnded.catch(() => 0)]); + flush(true); + + handle.send({ type: "input_audio_buffer.commit" }); + const t0 = Date.now(), deadline = t0 + (opts.finalTimeout ?? 60000); + const quietMs = opts.quietMs ?? 1200, emptyGraceMs = opts.emptyGraceMs ?? 6000; + let lastCount = 0, lastChange = t0; + while (Date.now() < deadline) { + if (opts.signal?.aborted) break; + const n = finals.size; + if (n !== lastCount) { lastCount = n; lastChange = Date.now(); } + if (failures.length) break; + if (n > 0 && Date.now() - lastChange >= quietMs) break; + if (n === 0 && Date.now() - t0 >= emptyGraceMs) break; + await sleep(120); + } + handle.send({ type: "input_audio_buffer.done" }); + await sleep(opts.tailMs ?? 500); + + const list = Array.from(finals.values()); + const text = list.length + ? list.map(f => f.transcript ?? "").filter(Boolean).join(" ").replace(/\s+/g, " ").trim() + : (deltas.at(-1) ?? "").replace(/\s+/g, " ").trim(); + for (const f of offs) f(); + try { handle.send?.({ type: "input_audio_buffer.clear" }); } catch {} + handle.close(); + return build(list, text); + } + + function build(list: any[], text: string): TranscriptResult { + return { + text, + segments: list.map(f => ({ transcript: f.transcript ?? "", isLast: f.is_last_result ?? false })), + words: list.flatMap(f => f.words_info?.words ?? []), + failed: failures, + seconds: +(total / (sampleRate * 2 * channels)).toFixed(3), + model: minted.input_audio_transcription?.model ?? null, + sessionId: minted.id ?? "", + protocol: handle.protocol, + }; + } + void capEnded.catch(() => {}); // capture-death is surfaced through drain's emptyGraceMs path when .stop() runs + return rec; + } + /** * One-shot transcribe of raw s16le pcm16 audio. * `onDelta` receives cumulative partials (great for a live UI), return value has final text. @@ -444,6 +568,74 @@ export async function hasFfmpeg(): Promise { try { await run("ffmpeg", ["-version"], () => {}); return true; } catch { return false; } } +/* ------------------------------------------------------- live mic capture -- */ + +export interface AudioDevice { index: number; name: string } + +/** True when this ffmpeg build carries the AVFoundation input (macOS mic capture). */ +export async function supportsAvfoundation(): Promise { + try { + let out = ""; + await run("ffmpeg", ["-hide_banner", "-demuxers"], b => { out += b.toString(); }); + return /\bavfoundation\b/i.test(out); + } catch { return false; } +} + +/** + * Enumerate AVFoundation audio input devices via `ffmpeg -f avfoundation -list_devices true`. + * The device table is printed to **stderr** and ffmpeg then errors opening the dummy input, + * so we read stderr and slice the "AVFoundation audio devices:" block. Empty array when ffmpeg + * or the AVFoundation demuxer is missing (caller should surface the TCC / install hint). + */ +export async function listAudioInputs(): Promise { + let stderr = ""; + try { + const r = await run("ffmpeg", ["-hide_banner", "-f", "avfoundation", "-list_devices", "true", "-i", "dummy"], () => {}); + stderr = r.stderr; + } catch (e: any) { + // expected: ffmpeg exits non-zero after "Error opening input file dummy" + if (!/AVFoundation/i.test(stderr ?? e?.message ?? "")) throw new Error(e?.message ?? String(e)); + } + const block = stderr.split(/AVFoundation audio devices:/i)[1] ?? ""; + const devs: AudioDevice[] = []; + for (const line of block.split("\n")) { + if (/AVFoundation video devices:/i.test(line)) break; + const m = line.match(/\[(\d+)\]\s+(.*\S)\s*$/); + if (m) devs.push({ index: +m[1], name: m[2] }); + } + return devs; +} + +/** `-i none:` selects the audio half of the [video]:[audio] AVFoundation spec. */ +function micArgs(device: string | null | undefined, { sampleRate = 16000, channels = 1 }: { sampleRate?: number; channels?: number } = {}): string[] { + const spec = device && device.trim() && device.trim().toLowerCase() !== "default" ? `none:${device.trim()}` : "none:default"; + return ["-hide_banner", "-v", "error", "-f", "avfoundation", "-i", spec, "-ac", String(channels), "-ar", String(sampleRate), "-f", "s16le", "-"]; +} + +/** Peak-normalised RMS of an s16le buffer, 0..1 (drives the "●" level meter in the UI). */ +function rms16(buf: Buffer): number { + let sum = 0, n = 0; + for (let i = 0; i + 1 < buf.length; i += 2) { const v = buf.readInt16LE(i) / 32768; sum += v * v; n++; } + return n ? Math.sqrt(sum / n) : 0; +} + +export interface MicRecording { + /** Last cumulative partial the server sent (for a live status line). */ + partial: string; + /** Total s16le bytes captured so far. */ + bytes: number; + /** Seconds captured so far. */ + seconds: number; + /** Peak level 0..1 since recording started (for a VU hint). */ + level: number; + /** Capture error surfaced by ffmpeg (missing binary / blocked microphone), if any. */ + error: string | null; + /** Stop capture, commit, flush, and resolve with the final transcript. Idempotent. */ + stop(): Promise; + /** Abort without waiting for the final segment (drops the session). */ + cancel(): void; +} + function run(cmd: string, args: string[], onData: (b: Buffer) => void): Promise<{ code: number; stderr: string }> { return new Promise((res, rej) => { const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });