iulianp/pi-asr
1
0 Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

wip: incremental ASR

Browse Source
This commit is contained in:
Iulian committed 2026-09-09 20:31:03 +03:00
1 parent 0b44d7e1bf
commit 65855ace6c
2 files changed
+73 -19
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

extensions/asr.ts
+17 -14
View File
@@ -309,8 +309,10 @@ export default function asrExtension(pi: ExtensionAPI) {
* gives no key-release event for hold-to-talk without the Kitty keyboard protocol.
* Shared module state so the /dictate command and the ctrl+shift+d shortcut drive one session.
*/
let active: { rec: MicRecording; submit: boolean } | null = null;
let active: { rec: MicRecording; submit: boolean; base: string } | null = null;
let starting = false;
/** Compose the composer text from the user's pre-existing draft + the dictated tail (idempotent: never appended). */
const joinText = (base: string, t: string): string => { const tail = t.trim(); return base ? (tail ? `${base} ${tail}` : base) : tail; };
async function resolveDevice(d?: string | null): Promise<string | null> {
if (!d) return null;
@@ -325,23 +327,20 @@ export default function asrExtension(pi: ExtensionAPI) {
async function toggleDictation(ctx: ExtensionContext, opts: { device?: string | null; language?: string; submit?: boolean } = {}): Promise<void> {
if (!ctx.hasUI) { ctx.ui.notify("dictation needs the interactive TUI", "error"); return; }
// second press while live => stop, flush, paste the final transcript.
// second press while live => stop, flush, finalise the composer with the deduped transcript.
if (active) {
const { rec, submit } = active; active = null;
const { rec, submit, base } = active; active = null;
ctx.ui.setStatus("asr", undefined);
try {
const res = await rec.stop();
const text = res.text.trim();
if (rec.error) { ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; }
if (!text) { ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; }
if (submit) pi.sendUserMessage(text);
else {
const cur = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, "");
ctx.ui.setEditorText(cur ? `${cur} ${text}` : text);
}
if (rec.error) { ctx.ui.setEditorText(base); ctx.ui.notify(`dictation error: ${rec.error}`, "error"); return; }
if (!text) { ctx.ui.setEditorText(base); ctx.ui.notify("no speech detected (speak a little longer, or run asr_health)", "warning"); return; }
if (submit) { ctx.ui.setEditorText(base); pi.sendUserMessage(text); }
else ctx.ui.setEditorText(joinText(base, text)); // identical to the last live paint — replace, not append
ctx.ui.notify(`dictated ${wordCount(text)} words (${res.seconds}s) → ${submit ? "sent" : "prompt"}`, "info");
} catch (e: any) {
rec.cancel();
rec.cancel(); ctx.ui.setEditorText(base);
ctx.ui.notify(`dictation failed: ${e?.message ?? e}`, "error");
}
return;
@@ -354,13 +353,16 @@ export default function asrExtension(pi: ExtensionAPI) {
try { device = await resolveDevice(opts.device); }
catch (e: any) { starting = false; ctx.ui.notify(`dictation: ${e?.message ?? e}`, "error"); return; }
const base = (ctx.ui.getEditorText() ?? "").replace(/\s+$/, "");
const client = new RealtimeAsrClient({ language, punctuation: true });
ctx.ui.setStatus("asr", "● listening …");
let rec: MicRecording;
try {
rec = await client.recordMic({
device,
onDelta: (t) => { if (active) ctx.ui.setStatus("asr", `● ${t || "…listening"}`); },
// stream word-by-word into the composer: setEditorText REPLACES (never appends), and the client's
// liveText is a per-segment cumulative partial, so each paint just overwrites the growing tail — no dupes.
onLive: (live) => { if (active) { const tail = live.trim(); ctx.ui.setEditorText(joinText(base, tail)); ctx.ui.setStatus("asr", `● ${tail.slice(-70) || "…listening"}`); } },
});
} catch (e: any) {
starting = false; ctx.ui.setStatus("asr", undefined);
@@ -368,8 +370,9 @@ export default function asrExtension(pi: ExtensionAPI) {
return;
}
starting = false;
active = { rec, submit: !!opts.submit };
ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — press ctrl+shift+d or /dictate again to stop`, "info");
active = { rec, submit: !!opts.submit, base };
ctx.ui.setEditorText(joinText(base, rec.liveText())); // show current state immediately (deltas are cumulative, so nothing is lost)
ctx.ui.notify(`🎙 recording${device ? ` from “${device}”` : " (default mic)"} — speech streams into the prompt; ctrl+shift+d or /dictate to stop`, "info");
}
pi.registerShortcut("ctrl+shift+d", {