iulianp/pi-asr
1
0 Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: live macOS dictation — push-to-talk mic into the realtime ASR session #1

Open
iulianp wants to merge 3 commits from feat/macos-live-dictation into main
pull from: feat/macos-live-dictation
merge into: iulianp:main
Conversation 0 Commits 3 Files Changed 4
+41 -17
1 changed files with 41 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ecd686073e - Show all commits
extensions/asr.ts
+41 -17
View File
@@ -309,10 +309,33 @@ export default function asrExtension(pi: ExtensionAPI) {
* gives no key-release event for hold-to-talk without the Kitty keyboard protocol.
* Shared module state so the /dictate command and the ctrl+shift+d shortcut drive one session.
*/
let active: { rec: MicRecording; submit: boolean; base: string } | null = null;
let active: { rec: MicRecording; submit: boolean; ourTail: string } | null = null;
let starting = false;
/** Compose the composer text from the user's pre-existing draft + the dictated tail (idempotent: never appended). */
const joinText = (base: string, t: string): string => { const tail = t.trim(); return base ? (tail ? `${base} ${tail}` : base) : tail; };
/**
* Append-as-you-go composer. Dictation owns ONLY a suffix at the very end of the editor (the words
* heard so far). Each refresh strips exactly the suffix we appended last time from the *current*
* text and re-appends the grown tail — so we never overwrite the user's draft and never revive text
* they deleted: we derive from the live editor, not a start-of-session snapshot.
*/
function paintTail(ctx: ExtensionContext, prevOurs: string, tail: string): string {
const cur = ctx.ui.getEditorText() ?? "";
let prefix = cur;
if (prevOurs) {
if (cur.endsWith(prevOurs)) prefix = cur.slice(0, cur.length - prevOurs.length);
else { const i = cur.lastIndexOf(prevOurs); if (i >= 0) prefix = cur.slice(0, i) + cur.slice(i + prevOurs.length); }
}
const t = tail.trim();
const ours = t ? (prefix === "" || /\s$/.test(prefix) ? "" : " ") + t : "";
ctx.ui.setEditorText(prefix + ours);
return ours;
}
/** Drop the suffix we own, leaving the user's own text untouched (cancel / no-speech / submit). */
function stripTail(ctx: ExtensionContext, ours: string): void {
if (!ours) return;
const cur = ctx.ui.getEditorText() ?? "";
if (cur.endsWith(ours)) ctx.ui.setEditorText(cur.slice(0, cur.length - ours.length));
else { const i = cur.lastIndexOf(ours); if (i >= 0) ctx.ui.setEditorText(cur.slice(0, i) + cur.slice(i + ours.length)); }
}
async function resolveDevice(d?: string | null): Promise<string | null> {
if (!d) return null;
@@ -329,18 +352,18 @@ export default function asrExtension(pi: ExtensionAPI) {
// second press while live => stop, flush, finalise the composer with the deduped transcript.
if (active) {
const { rec, submit, base } = active; active = null;
const { rec, submit, ourTail } = active; active = null;
ctx.ui.setStatus("asr", undefined);
try {
const res = await rec.stop();
const text = res.text.trim();
if (rec.error) { ctx.ui.setEditorText(base); ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; }
if (!text) { ctx.ui.setEditorText(base); ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; }
if (submit) { ctx.ui.setEditorText(base); pi.sendUserMessage(text); }
else ctx.ui.setEditorText(joinText(base, text)); // identical to the last live paint — replace, not append
ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "prompt"}`, "info");
if (rec.error) { stripTail(ctx, ourTail); ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; }
if (!text) { stripTail(ctx, ourTail); ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; }
if (submit) { stripTail(ctx, ourTail); pi.sendUserMessage(text); }
else paintTail(ctx, ourTail, text); // swap our live suffix for the finalised text; user's own text is untouched
ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "added"}`, "info");
} catch (e: any) {
rec.cancel(); ctx.ui.setEditorText(base);
rec.cancel(); stripTail(ctx, ourTail);
ctx.ui.notify(`dictation failed: ${e?.message ?? e}`, "error");
}
return;
@@ -353,16 +376,17 @@ export default function asrExtension(pi: ExtensionAPI) {
try { device = await resolveDevice(opts.device); }
catch (e: any) { starting = false; ctx.ui.notify(`dictation: ${e?.message ?? e}`, "error"); return; }
const base = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, "");
// No start-of-session snapshot: dictation appends to the *live* editor text (paintTail/stripTail),
// so any edits or deletions the user makes while we record are preserved and never revived.
const client = new RealtimeAsrClient({ language, punctuation: true });
ctx.ui.setStatus("asr", "● listening …");
let rec: MicRecording;
try {
rec = await client.recordMic({
device,
// stream word-by-word into the composer: setEditorText REPLACES (never appends), and the client's
// liveText is a per-segment cumulative partial, so each paint just overwrites the growing tail — no dupes.
onLive: (live) => { if (active) { const tail = live.trim(); ctx.ui.setEditorText(joinText(base, tail)); ctx.ui.setStatus("asr", `● ${tail.slice(-70) || "…listening"}`); } },
// stream word-by-word: paintTail owns only our appended suffix, so the user's draft is preserved
// and anything they delete stays deleted (no snapshot is ever written back). liveText is cumulative.
onLive: (live) => { if (active) { active.ourTail = paintTail(ctx, active.ourTail, live); ctx.ui.setStatus("asr", `● ${live.trim().slice(-70) || "…listening"}`); } },
});
} catch (e: any) {
starting = false; ctx.ui.setStatus("asr", undefined);
@@ -370,12 +394,12 @@ export default function asrExtension(pi: ExtensionAPI) {
return;
}
starting = false;
active = { rec, submit: !!opts.submit, base };
ctx.ui.setEditorText(joinText(base, rec.liveText())); // show current state immediately (deltas are cumulative, so nothing is lost)
active = { rec, submit: !!opts.submit, ourTail: "" };
active.ourTail = paintTail(ctx, "", rec.liveText()); // reflect anything already recognised without overwriting the user's draft
ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — speech streams into the prompt; ctrl+shift+d or /dictate to stop`, "info");
}
pi.registerShortcut("ctrl+shift+d", {
pi.registerShortcut("ctrl+shift+m", {
description: "Toggle macOS live dictation (press to start, press again to stop and paste into the prompt)",
handler: async (ctx) => { await toggleDictation(ctx); },
});