feat: live macOS dictation — push-to-talk mic into the realtime ASR session
- /dictate + ctrl+shift+d toggle one shared session: press to start, press again to stop, flush and paste into the prompt (a terminal gives no key-release event for hold-to-talk without the Kitty keyboard protocol) - client.recordMic(): spawns ffmpeg AVFoundation capture and appends s16le frames as they arrive (no pacer — the mic is already 1x paced); stop() = commit + the same quiescence drain transcribe() uses, since is_last_result never flips true - enumerates mics by slicing the "AVFoundation audio devices:" block off stderr (ffmpeg prints the table there, then fails opening the dummy input) - --device takes an index or name, --lang / ASR_DICTATE_LANGUAGE, --submit sends instead of pasting; asr_health now reports the mic inputs it enumerated - capture death surfaces via the emptyGraceMs path with the TCC hint (System Settings → Privacy & Security → Microphone) when ffmpeg exits non-zero at 0 bytes - env.sh sources the gitignored .env so a shell gets ASR_NIM_URL / ASR_API_KEY
This commit is contained in:
Iulian committed
1 parent
19faa8b088
commit
0b44d7e1bf
4 files changed
+307
-4
No files matched your search
+106
-4
@@ -19,8 +19,8 @@ import type { AgentToolResult, ExtensionAPI, ExtensionContext } from "@mariozech
|
||||
import { Text } from "@mariozechner/pi-tui";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import {
|
||||
DEFAULT_BASE_URL, RealtimeAsrClient, hasFfmpeg, loadPcm16,
|
||||
type TranscriptResult, type WordInfo,
|
||||
DEFAULT_BASE_URL, RealtimeAsrClient, hasFfmpeg, loadPcm16, listAudioInputs,
|
||||
type MicRecording, type TranscriptResult, type WordInfo,
|
||||
} from "../src/client.js";
|
||||
|
||||
interface TranscribeParams {
|
||||
@@ -205,11 +205,12 @@ export default function asrExtension(pi: ExtensionAPI) {
|
||||
|
||||
async execute(_id, _params, _signal, _onUpdate, ctx): Promise<AgentToolResult<any>> {
|
||||
const client = new RealtimeAsrClient();
|
||||
const [live, ready, minted, ffmpeg] = await Promise.all([
|
||||
const [live, ready, minted, ffmpeg, inputs] = await Promise.all([
|
||||
client.health("live", 10000).catch((e: any) => ({ error: e?.message ?? String(e) })),
|
||||
client.health("ready", 10000).catch((e: any) => ({ error: e?.message ?? String(e) })),
|
||||
client.mint(10000).catch((e: any) => ({ error: e?.message ?? String(e) })),
|
||||
hasFfmpeg(),
|
||||
listAudioInputs().catch(() => [] as { index: number; name: string }[]),
|
||||
]);
|
||||
const ok = live?.status === "live" && ready?.status === "ready" && !minted?.error;
|
||||
const key = client.cfg.apiKey;
|
||||
@@ -229,10 +230,11 @@ export default function asrExtension(pi: ExtensionAPI) {
|
||||
` ws token single-use, expires in ${secs}s (mint immediately before each connect)`,
|
||||
].join("\n"),
|
||||
`ffmpeg ${ffmpeg ? "available (any container/codec accepted)" : "MISSING — only 16kHz mono wav files will work"}`,
|
||||
`mic inputs ${inputs.length ? inputs.map(d => `[${d.index}] ${d.name}`).join(", ") : ffmpeg ? "none enumerated (check Mic permission / AVFoundation build)" : "n/a (ffmpeg missing)"}`,
|
||||
`note the realtime decoder runs at ~1x realtime; a 10min file takes ~10min (speed=2 halves it)`,
|
||||
].join("\n");
|
||||
if (!ok) throw new Error(`ASR server is not usable:\n${text}`);
|
||||
return { content: [{ type: "text", text }], details: { ok, live, ready, minted, ffmpeg } };
|
||||
return { content: [{ type: "text", text }], details: { ok, live, ready, minted, ffmpeg, inputs } };
|
||||
},
|
||||
|
||||
renderResult(result: any, _options: any, theme: any) {
|
||||
@@ -300,4 +302,104 @@ export default function asrExtension(pi: ExtensionAPI) {
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------- /dictate -- *
|
||||
* macOS live dictation: ffmpeg AVFoundation mic capture -> the same realtime WS client.
|
||||
* Toggle semantics (press to start, press again to stop + flush) because a plain terminal
|
||||
* gives no key-release event for hold-to-talk without the Kitty keyboard protocol.
|
||||
* Shared module state so the /dictate command and the ctrl+shift+d shortcut drive one session.
|
||||
*/
|
||||
let active: { rec: MicRecording; submit: boolean } | null = null;
|
||||
let starting = false;
|
||||
|
||||
async function resolveDevice(d?: string | null): Promise<string | null> {
|
||||
if (!d) return null;
|
||||
if (/^\d+$/.test(d)) {
|
||||
const hit = (await listAudioInputs()).find(x => x.index === +d);
|
||||
if (!hit) throw new Error(`no audio input at index ${d} — list them with: /dictate devices`);
|
||||
return hit.name;
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
async function toggleDictation(ctx: ExtensionContext, opts: { device?: string | null; language?: string; submit?: boolean } = {}): Promise<void> {
|
||||
if (!ctx.hasUI) { ctx.ui.notify("dictation needs the interactive TUI", "error"); return; }
|
||||
|
||||
// second press while live => stop, flush, paste the final transcript.
|
||||
if (active) {
|
||||
const { rec, submit } = active; active = null;
|
||||
ctx.ui.setStatus("asr", undefined);
|
||||
try {
|
||||
const res = await rec.stop();
|
||||
const text = res.text.trim();
|
||||
if (rec.error) { ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; }
|
||||
if (!text) { ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; }
|
||||
if (submit) pi.sendUserMessage(text);
|
||||
else {
|
||||
const cur = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, "");
|
||||
ctx.ui.setEditorText(cur ? `${cur} ${text}` : text);
|
||||
}
|
||||
ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "prompt"}`, "info");
|
||||
} catch (e: any) {
|
||||
rec.cancel();
|
||||
ctx.ui.notify(`dictation failed: ${e?.message ?? e}`, "error");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (starting) return; // ignore a double-press during mint+handshake
|
||||
starting = true;
|
||||
const language = opts.language ?? process.env.ASR_DICTATE_LANGUAGE ?? "en-US";
|
||||
let device: string | null;
|
||||
try { device = await resolveDevice(opts.device); }
|
||||
catch (e: any) { starting = false; ctx.ui.notify(`dictation: ${e?.message ?? e}`, "error"); return; }
|
||||
|
||||
const client = new RealtimeAsrClient({ language, punctuation: true });
|
||||
ctx.ui.setStatus("asr", "● listening …");
|
||||
let rec: MicRecording;
|
||||
try {
|
||||
rec = await client.recordMic({
|
||||
device,
|
||||
onDelta: (t) => { if (active) ctx.ui.setStatus("asr", `● ${t || "…listening"}`); },
|
||||
});
|
||||
} catch (e: any) {
|
||||
starting = false; ctx.ui.setStatus("asr", undefined);
|
||||
ctx.ui.notify(`could not start mic capture: ${e?.message ?? e}\n(ffmpeg on PATH w/ AVFoundation; grant the terminal Microphone access in System Settings → Privacy & Security)`, "error");
|
||||
return;
|
||||
}
|
||||
starting = false;
|
||||
active = { rec, submit: !!opts.submit };
|
||||
ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — press ctrl+shift+d or /dictate again to stop`, "info");
|
||||
}
|
||||
|
||||
pi.registerShortcut("ctrl+shift+d", {
|
||||
description: "Toggle macOS live dictation (press to start, press again to stop and paste into the prompt)",
|
||||
handler: async (ctx) => { await toggleDictation(ctx); },
|
||||
});
|
||||
|
||||
pi.registerCommand("dictate", {
|
||||
description: "Live macOS dictation (mic → realtime ASR): /dictate [stop|devices] [--device <name|index>] [--lang xx-YY] [--submit]",
|
||||
handler: async (args: string, ctx: ExtensionContext) => {
|
||||
const toks = args.trim().split(/\s+/).filter(Boolean);
|
||||
const head = (toks[0] ?? "").toLowerCase();
|
||||
if (head === "devices") {
|
||||
const ds = await listAudioInputs();
|
||||
ctx.ui.notify(ds.length ? ds.map(d => `[${d.index}] ${d.name}`).join("\n") : "no audio inputs found (ffmpeg w/ AVFoundation? Mic permission?)", "info");
|
||||
return;
|
||||
}
|
||||
if (head === "stop" || head === "cancel") {
|
||||
if (!active) { ctx.ui.notify("not currently recording", "info"); return; }
|
||||
await toggleDictation(ctx);
|
||||
return;
|
||||
}
|
||||
let device: string | undefined, language: string | undefined, submit = false;
|
||||
for (let i = 0; i < toks.length; i++) {
|
||||
const t = toks[i];
|
||||
if (t === "--device") device = toks[++i];
|
||||
else if (t === "--lang") language = toks[++i];
|
||||
else if (t === "--submit") submit = true;
|
||||
}
|
||||
await toggleDictation(ctx, { device, language, submit });
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user