From ecd686073ec20de42fb57e1e96819e7ad8fb6277 Mon Sep 17 00:00:00 2001 From: Iulian Date: Thu, 10 Sep 2026 22:00:20 +0300 Subject: [PATCH] wip: incremental ASR outputs in realtime, few bugs --- extensions/asr.ts | 58 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/extensions/asr.ts b/extensions/asr.ts index 819d046..05b2743 100644 --- a/extensions/asr.ts +++ b/extensions/asr.ts @@ -309,10 +309,33 @@ export default function asrExtension(pi: ExtensionAPI) { * gives no key-release event for hold-to-talk without the Kitty keyboard protocol. * Shared module state so the /dictate command and the ctrl+shift+d shortcut drive one session. */ - let active: { rec: MicRecording; submit: boolean; base: string } | null = null; + let active: { rec: MicRecording; submit: boolean; ourTail: string } | null = null; let starting = false; - /** Compose the composer text from the user's pre-existing draft + the dictated tail (idempotent: never appended). */ - const joinText = (base: string, t: string): string => { const tail = t.trim(); return base ? (tail ? `${base} ${tail}` : base) : tail; }; + /** + * Append-as-you-go composer. Dictation owns ONLY a suffix at the very end of the editor (the words + * heard so far). Each refresh strips exactly the suffix we appended last time from the *current* + * text and re-appends the grown tail — so we never overwrite the user's draft and never revive text + * they deleted: we derive from the live editor, not a start-of-session snapshot. + */ + function paintTail(ctx: ExtensionContext, prevOurs: string, tail: string): string { + const cur = ctx.ui.getEditorText() ?? ""; + let prefix = cur; + if (prevOurs) { + if (cur.endsWith(prevOurs)) prefix = cur.slice(0, cur.length - prevOurs.length); + else { const i = cur.lastIndexOf(prevOurs); if (i >= 0) prefix = cur.slice(0, i) + cur.slice(i + prevOurs.length); } + } + const t = tail.trim(); + const ours = t ? (prefix === "" || /\s$/.test(prefix) ? "" : " ") + t : ""; + ctx.ui.setEditorText(prefix + ours); + return ours; + } + /** Drop the suffix we own, leaving the user's own text untouched (cancel / no-speech / submit). */ + function stripTail(ctx: ExtensionContext, ours: string): void { + if (!ours) return; + const cur = ctx.ui.getEditorText() ?? ""; + if (cur.endsWith(ours)) ctx.ui.setEditorText(cur.slice(0, cur.length - ours.length)); + else { const i = cur.lastIndexOf(ours); if (i >= 0) ctx.ui.setEditorText(cur.slice(0, i) + cur.slice(i + ours.length)); } + } async function resolveDevice(d?: string | null): Promise { if (!d) return null; @@ -329,18 +352,18 @@ export default function asrExtension(pi: ExtensionAPI) { // second press while live => stop, flush, finalise the composer with the deduped transcript. if (active) { - const { rec, submit, base } = active; active = null; + const { rec, submit, ourTail } = active; active = null; ctx.ui.setStatus("asr", undefined); try { const res = await rec.stop(); const text = res.text.trim(); - if (rec.error) { ctx.ui.setEditorText(base); ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; } - if (!text) { ctx.ui.setEditorText(base); ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; } - if (submit) { ctx.ui.setEditorText(base); pi.sendUserMessage(text); } - else ctx.ui.setEditorText(joinText(base, text)); // identical to the last live paint — replace, not append - ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "prompt"}`, "info"); + if (rec.error) { stripTail(ctx, ourTail); ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; } + if (!text) { stripTail(ctx, ourTail); ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; } + if (submit) { stripTail(ctx, ourTail); pi.sendUserMessage(text); } + else paintTail(ctx, ourTail, text); // swap our live suffix for the finalised text; user's own text is untouched + ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "added"}`, "info"); } catch (e: any) { - rec.cancel(); ctx.ui.setEditorText(base); + rec.cancel(); stripTail(ctx, ourTail); ctx.ui.notify(`dictation failed: ${e?.message ?? e}`, "error"); } return; @@ -353,16 +376,17 @@ export default function asrExtension(pi: ExtensionAPI) { try { device = await resolveDevice(opts.device); } catch (e: any) { starting = false; ctx.ui.notify(`dictation: ${e?.message ?? e}`, "error"); return; } - const base = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, ""); + // No start-of-session snapshot: dictation appends to the *live* editor text (paintTail/stripTail), + // so any edits or deletions the user makes while we record are preserved and never revived. const client = new RealtimeAsrClient({ language, punctuation: true }); ctx.ui.setStatus("asr", "● listening …"); let rec: MicRecording; try { rec = await client.recordMic({ device, - // stream word-by-word into the composer: setEditorText REPLACES (never appends), and the client's - // liveText is a per-segment cumulative partial, so each paint just overwrites the growing tail — no dupes. - onLive: (live) => { if (active) { const tail = live.trim(); ctx.ui.setEditorText(joinText(base, tail)); ctx.ui.setStatus("asr", `● ${tail.slice(-70) || "…listening"}`); } }, + // stream word-by-word: paintTail owns only our appended suffix, so the user's draft is preserved + // and anything they delete stays deleted (no snapshot is ever written back). liveText is cumulative. + onLive: (live) => { if (active) { active.ourTail = paintTail(ctx, active.ourTail, live); ctx.ui.setStatus("asr", `● ${live.trim().slice(-70) || "…listening"}`); } }, }); } catch (e: any) { starting = false; ctx.ui.setStatus("asr", undefined); @@ -370,12 +394,12 @@ export default function asrExtension(pi: ExtensionAPI) { return; } starting = false; - active = { rec, submit: !!opts.submit, base }; - ctx.ui.setEditorText(joinText(base, rec.liveText())); // show current state immediately (deltas are cumulative, so nothing is lost) + active = { rec, submit: !!opts.submit, ourTail: "" }; + active.ourTail = paintTail(ctx, "", rec.liveText()); // reflect anything already recognised without overwriting the user's draft ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — speech streams into the prompt; ctrl+shift+d or /dictate to stop`, "info"); } - pi.registerShortcut("ctrl+shift+d", { + pi.registerShortcut("ctrl+shift+m", { description: "Toggle macOS live dictation (press to start, press again to stop and paste into the prompt)", handler: async (ctx) => { await toggleDictation(ctx); }, });