feat: live macOS dictation — push-to-talk mic into the realtime ASR session
- /dictate + ctrl+shift+d toggle one shared session: press to start, press again to stop, flush and paste into the prompt (a terminal gives no key-release event for hold-to-talk without the Kitty keyboard protocol) - client.recordMic(): spawns ffmpeg AVFoundation capture and appends s16le frames as they arrive (no pacer — the mic is already 1x paced); stop() = commit + the same quiescence drain transcribe() uses, since is_last_result never flips true - enumerates mics by slicing the "AVFoundation audio devices:" block off stderr (ffmpeg prints the table there, then fails opening the dummy input) - --device takes an index or name, --lang / ASR_DICTATE_LANGUAGE, --submit sends instead of pasting; asr_health now reports the mic inputs it enumerated - capture death surfaces via the emptyGraceMs path with the TCC hint (System Settings → Privacy & Security → Microphone) when ffmpeg exits non-zero at 0 bytes - env.sh sources the gitignored .env so a shell gets ASR_NIM_URL / ASR_API_KEY
This commit is contained in:
Iulian committed
1 parent
19faa8b088
commit
0b44d7e1bf
4 files changed
+307
-4
No files matched your search
+192
@@ -277,6 +277,130 @@ export class RealtimeAsrClient {
|
||||
return { handle, minted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Push-to-talk live dictation on macOS: spawn ffmpeg AVFoundation mic capture and stream
|
||||
* s16le frames into a realtime ASR session as they arrive (no artificial pacer — the mic is
|
||||
* already real-time paced). Returns a handle; call `.stop()` to commit + flush + get the final
|
||||
* transcript, or `.cancel()` to drop it. `onDelta` drives a live status line.
|
||||
*
|
||||
* The completion drain reuses the same quiescence logic as transcribe(): the server emits
|
||||
* cumulative `.delta`s and a words-bearing then words-less `.completed` per segment, and
|
||||
* `is_last_result` never flips true, so after commit we wait until no new segment for quietMs.
|
||||
*/
|
||||
async recordMic(opts: {
|
||||
device?: string | null; frameMs?: number;
|
||||
quietMs?: number; emptyGraceMs?: number; finalTimeout?: number; tailMs?: number;
|
||||
sampleRate?: number; channels?: number; session?: Record<string, any>;
|
||||
onDelta?: (t: string) => void; onLevel?: (peak: number) => void; onStart?: (d: { device: string | null }) => void;
|
||||
signal?: AbortSignal;
|
||||
} = {}): Promise<MicRecording> {
|
||||
this.requireKey();
|
||||
const sampleRate = opts.sampleRate ?? 16000, channels = opts.channels ?? 1;
|
||||
const frameBytes = Math.max(320, Math.round((sampleRate * (opts.frameMs ?? 100)) / 1000) * 2 * channels);
|
||||
const minted = await this.mint();
|
||||
if (opts.signal?.aborted) throw new Error("cancelled");
|
||||
const handle = await this.open({ token: minted.client_secret?.value ?? null });
|
||||
|
||||
handle.send({ type: "transcription_session.update", session: this.sessionConfig(minted, opts.session ?? {}) });
|
||||
await handle.expect(EV.sessionUpdated, 20000);
|
||||
|
||||
const deltas: string[] = [];
|
||||
const finals: Map<string, any> = new Map();
|
||||
const failures: unknown[] = [];
|
||||
const offs = [
|
||||
this.on(EV.delta, e => { const t = e.delta ?? ""; deltas.push(t); rec.partial = t; opts.onDelta?.(t); }),
|
||||
this.on(EV.completed, (e: any) => {
|
||||
const key = e.item_id ?? "_";
|
||||
const prev = finals.get(key);
|
||||
const nw = e.words_info?.words?.length ?? 0, pw = prev?.words_info?.words?.length ?? 0;
|
||||
finals.set(key, prev && pw > nw && prev.transcript ? { ...e, words_info: prev.words_info } : e);
|
||||
}),
|
||||
this.on(EV.failed, (e: any) => failures.push(e)),
|
||||
];
|
||||
|
||||
let pending: Buffer[] = [], carry: Buffer = Buffer.alloc(0), total = 0, peak = 0, cancelled = false;
|
||||
let drainP: Promise<TranscriptResult> | null = null;
|
||||
const rec: MicRecording = {
|
||||
partial: "", bytes: 0, seconds: 0, level: 0, error: null,
|
||||
stop: () => (drainP ??= drain()),
|
||||
cancel: () => { cancelled = true; try { handle.send?.({ type: "input_audio_buffer.clear" }); } catch {} for (const f of offs) f(); handle.close(); },
|
||||
};
|
||||
|
||||
function flush(force = false): void {
|
||||
let buf = carry.length ? Buffer.concat([carry, ...pending]) : Buffer.concat(pending);
|
||||
pending = [];
|
||||
if (!force && buf.length < frameBytes) { carry = buf; return; }
|
||||
carry = Buffer.alloc(0);
|
||||
for (let i = 0; i < buf.length; i += frameBytes) {
|
||||
handle.send({ type: "input_audio_buffer.append", audio: buf.subarray(i, Math.min(i + frameBytes, buf.length)).toString("base64") });
|
||||
}
|
||||
}
|
||||
|
||||
const child = spawn("ffmpeg", micArgs(opts.device, { sampleRate, channels }), { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let capErr = "";
|
||||
child.stdout.on("data", (b: Buffer) => {
|
||||
total += b.length; rec.bytes = total; rec.seconds = +(total / (sampleRate * 2 * channels)).toFixed(2);
|
||||
const lv = rms16(b); if (lv > peak) { peak = lv; rec.level = lv; } opts.onLevel?.(lv);
|
||||
pending.push(b); flush();
|
||||
});
|
||||
child.stderr.on("data", b => { capErr += b.toString(); });
|
||||
let capDied: (e: Error) => void = () => {};
|
||||
const capEnded = new Promise<never>((_, rej) => { capDied = rej; });
|
||||
child.on("error", e => { const m = (e as any).code === "ENOENT" ? "ffmpeg is not installed / not on PATH" : e.message; rec.error = m; capDied(new Error(m)); });
|
||||
child.on("close", code => { if (code !== 0 && total === 0) { const m = `mic capture failed (ffmpeg exit ${code}). ${capErr.slice(-300)}\nIf macOS blocked the microphone: System Settings → Privacy & Security → Microphone → allow your terminal app.`; rec.error = m; capDied(new Error(m)); } });
|
||||
|
||||
opts.onStart?.({ device: opts.device ?? null });
|
||||
|
||||
function stopCapture(): void { try { child.kill("SIGINT"); } catch {} }
|
||||
|
||||
async function drain(): Promise<TranscriptResult> {
|
||||
stopCapture();
|
||||
// let stdout finish flushing the tail, then force-send the last partial frame
|
||||
await Promise.race([new Promise(r => child.once("close", r)), sleep(400), capEnded.catch(() => 0)]);
|
||||
flush(true);
|
||||
|
||||
handle.send({ type: "input_audio_buffer.commit" });
|
||||
const t0 = Date.now(), deadline = t0 + (opts.finalTimeout ?? 60000);
|
||||
const quietMs = opts.quietMs ?? 1200, emptyGraceMs = opts.emptyGraceMs ?? 6000;
|
||||
let lastCount = 0, lastChange = t0;
|
||||
while (Date.now() < deadline) {
|
||||
if (opts.signal?.aborted) break;
|
||||
const n = finals.size;
|
||||
if (n !== lastCount) { lastCount = n; lastChange = Date.now(); }
|
||||
if (failures.length) break;
|
||||
if (n > 0 && Date.now() - lastChange >= quietMs) break;
|
||||
if (n === 0 && Date.now() - t0 >= emptyGraceMs) break;
|
||||
await sleep(120);
|
||||
}
|
||||
handle.send({ type: "input_audio_buffer.done" });
|
||||
await sleep(opts.tailMs ?? 500);
|
||||
|
||||
const list = Array.from(finals.values());
|
||||
const text = list.length
|
||||
? list.map(f => f.transcript ?? "").filter(Boolean).join(" ").replace(/\s+/g, " ").trim()
|
||||
: (deltas.at(-1) ?? "").replace(/\s+/g, " ").trim();
|
||||
for (const f of offs) f();
|
||||
try { handle.send?.({ type: "input_audio_buffer.clear" }); } catch {}
|
||||
handle.close();
|
||||
return build(list, text);
|
||||
}
|
||||
|
||||
function build(list: any[], text: string): TranscriptResult {
|
||||
return {
|
||||
text,
|
||||
segments: list.map(f => ({ transcript: f.transcript ?? "", isLast: f.is_last_result ?? false })),
|
||||
words: list.flatMap(f => f.words_info?.words ?? []),
|
||||
failed: failures,
|
||||
seconds: +(total / (sampleRate * 2 * channels)).toFixed(3),
|
||||
model: minted.input_audio_transcription?.model ?? null,
|
||||
sessionId: minted.id ?? "",
|
||||
protocol: handle.protocol,
|
||||
};
|
||||
}
|
||||
void capEnded.catch(() => {}); // capture-death is surfaced through drain's emptyGraceMs path when .stop() runs
|
||||
return rec;
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot transcribe of raw s16le pcm16 audio.
|
||||
* `onDelta` receives cumulative partials (great for a live UI), return value has final text.
|
||||
@@ -444,6 +568,74 @@ export async function hasFfmpeg(): Promise<boolean> {
|
||||
try { await run("ffmpeg", ["-version"], () => {}); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------- live mic capture -- */
|
||||
|
||||
export interface AudioDevice { index: number; name: string }
|
||||
|
||||
/** True when this ffmpeg build carries the AVFoundation input (macOS mic capture). */
|
||||
export async function supportsAvfoundation(): Promise<boolean> {
|
||||
try {
|
||||
let out = "";
|
||||
await run("ffmpeg", ["-hide_banner", "-demuxers"], b => { out += b.toString(); });
|
||||
return /\bavfoundation\b/i.test(out);
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate AVFoundation audio input devices via `ffmpeg -f avfoundation -list_devices true`.
|
||||
* The device table is printed to **stderr** and ffmpeg then errors opening the dummy input,
|
||||
* so we read stderr and slice the "AVFoundation audio devices:" block. Empty array when ffmpeg
|
||||
* or the AVFoundation demuxer is missing (caller should surface the TCC / install hint).
|
||||
*/
|
||||
export async function listAudioInputs(): Promise<AudioDevice[]> {
|
||||
let stderr = "";
|
||||
try {
|
||||
const r = await run("ffmpeg", ["-hide_banner", "-f", "avfoundation", "-list_devices", "true", "-i", "dummy"], () => {});
|
||||
stderr = r.stderr;
|
||||
} catch (e: any) {
|
||||
// expected: ffmpeg exits non-zero after "Error opening input file dummy"
|
||||
if (!/AVFoundation/i.test(stderr ?? e?.message ?? "")) throw new Error(e?.message ?? String(e));
|
||||
}
|
||||
const block = stderr.split(/AVFoundation audio devices:/i)[1] ?? "";
|
||||
const devs: AudioDevice[] = [];
|
||||
for (const line of block.split("\n")) {
|
||||
if (/AVFoundation video devices:/i.test(line)) break;
|
||||
const m = line.match(/\[(\d+)\]\s+(.*\S)\s*$/);
|
||||
if (m) devs.push({ index: +m[1], name: m[2] });
|
||||
}
|
||||
return devs;
|
||||
}
|
||||
|
||||
/** `-i none:<device|default>` selects the audio half of the [video]:[audio] AVFoundation spec. */
|
||||
function micArgs(device: string | null | undefined, { sampleRate = 16000, channels = 1 }: { sampleRate?: number; channels?: number } = {}): string[] {
|
||||
const spec = device && device.trim() && device.trim().toLowerCase() !== "default" ? `none:${device.trim()}` : "none:default";
|
||||
return ["-hide_banner", "-v", "error", "-f", "avfoundation", "-i", spec, "-ac", String(channels), "-ar", String(sampleRate), "-f", "s16le", "-"];
|
||||
}
|
||||
|
||||
/** Peak-normalised RMS of an s16le buffer, 0..1 (drives the "●" level meter in the UI). */
|
||||
function rms16(buf: Buffer): number {
|
||||
let sum = 0, n = 0;
|
||||
for (let i = 0; i + 1 < buf.length; i += 2) { const v = buf.readInt16LE(i) / 32768; sum += v * v; n++; }
|
||||
return n ? Math.sqrt(sum / n) : 0;
|
||||
}
|
||||
|
||||
export interface MicRecording {
|
||||
/** Last cumulative partial the server sent (for a live status line). */
|
||||
partial: string;
|
||||
/** Total s16le bytes captured so far. */
|
||||
bytes: number;
|
||||
/** Seconds captured so far. */
|
||||
seconds: number;
|
||||
/** Peak level 0..1 since recording started (for a VU hint). */
|
||||
level: number;
|
||||
/** Capture error surfaced by ffmpeg (missing binary / blocked microphone), if any. */
|
||||
error: string | null;
|
||||
/** Stop capture, commit, flush, and resolve with the final transcript. Idempotent. */
|
||||
stop(): Promise<TranscriptResult>;
|
||||
/** Abort without waiting for the final segment (drops the session). */
|
||||
cancel(): void;
|
||||
}
|
||||
|
||||
function run(cmd: string, args: string[], onData: (b: Buffer) => void): Promise<{ code: number; stderr: string }> {
|
||||
return new Promise((res, rej) => {
|
||||
const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
|
||||
Reference in new issue
Block a user