Setting the file. One moment. Hershey Script1
Asset Char Widths
scripts/transcribe.cjs
JavaScript·332 lines·12 KB
12const os = require("os");
13const cp = require("child_process");
14
15function hfRoot() {
16 const roots = [
17 process.env.HYPERFRAMES_ROOT,
18 path.resolve(__dirname, "..", "..", ".."),
19 path.join(os.homedir(), "Downloads", "hyperframes"),
20 ].filter(Boolean);
21 for (const r of roots)
22 if (fs.existsSync(path.join(r, "packages", "cli", "dist", "cli.js"))) return r;
23 console.error("[transcribe] hyperframes CLI not found — set HYPERFRAMES_ROOT");
24 process.exit(3);
25}
26function ensureSource(project) {
27 const src = path.join(project, "source.mp4");
28 if (fs.existsSync(src)) return src;
29 const EXCL = new Set(["final", "bg_plus_caps", "fg_caps", "audio"]);
30 let cands = fs
31 .readdirSync(project)
32 .filter(
33 (f) =>
34 ["mp4", "mov", "webm", "mkv", "m4v"].includes(path.extname(f).slice(1).toLowerCase()) &&
35 !EXCL.has(path.basename(f, path.extname(f))) &&
36 !f.startsWith("index"),
37 )
38 .map((f) => path.join(project, f));
39 let found = cands.sort((a, b) => fs.statSync(b).size - fs.statSync(a).size)[0];
40 if (found) {
41 try {
42 fs.symlinkSync(path.basename(found), src);
43 } catch {
44 fs.copyFileSync(found, src);
45 }
46 }
47 return src;
48}
49function _usableWords(d) {
50 return d && Array.isArray(d.words) && d.words.some((w) => w && "start" in w && "end" in w);
51}
52// Mean loudness of the audio, for the no-speech guard below. Silence → whisper
53// hallucinates (famously "Thank you."), and the decision gate refuses "no speech".
54function meanVolumeDb(audio) {
55 try {
56 // ffmpeg writes volumedetect stats to STDERR — capture it (spawnSync, no throw).
57 const r = cp.spawnSync(
58 "ffmpeg",
59 ["-hide_banner", "-nostats", "-i", audio, "-af", "volumedetect", "-f", "null", "-"],
60 { encoding: "utf8" },
61 );
62 const out = (r.stderr || "") + (r.stdout || "");
63 const m = out.match(/mean_volume:\s*(-?[\d.]+) dB/);
64 return m ? parseFloat(m[1]) : null;
65 } catch {
66 return null;
67 }
68}
69
70// Where does AUDIBLE content end? Whisper hallucinates trailing words over a silent
71// tail (observed: "I'm sorry." repeated over dead air at a clip's end). silencedetect
72// finds a terminal silence running to EOF; words "spoken" inside it are fabricated.
73// Conservative: applause/music read as non-silence, so this fires only on truly dead
74// tails. Returns {speechEnd, total} or null.
75function audibleEnd(audio) {
76 try {
77 const r = cp.spawnSync(
78 "ffmpeg",
79 [
80 "-hide_banner",
81 "-nostats",
82 "-i",
83 audio,
84 "-af",
85 "silencedetect=noise=-35dB:d=0.6",
86 "-f",
87 "null",
88 "-",
89 ],
90 { encoding: "utf8" },
91 );
92 const out = (r.stderr || "") + (r.stdout || "");
93 const durM = out.match(/Duration:\s*(\d+):(\d+):([\d.]+)/);
94 const total = durM ? +durM[1] * 3600 + +durM[2] * 60 + +durM[3] : null;
95 if (total == null) return null;
96 const starts = [...out.matchAll(/silence_start:\s*([\d.]+)/g)].map((x) => +x[1]);
97 const ends = [...out.matchAll(/silence_end:\s*([\d.]+)/g)].map((x) => +x[1]);
98 if (!starts.length) return { speechEnd: total, total };
99 const lastStart = starts[starts.length - 1];
100 const closed = ends.some((e) => e > lastStart); // silence re-broken before EOF?
101 return { speechEnd: closed ? total : lastStart, total };
102 } catch {
103 return null;
104 }
105}
106
107function main() {
108 const project = path.resolve(process.argv[2] || "");
109 if (!process.argv[2]) {
110 console.error("usage: transcribe.cjs <project-dir> [model] [language]");
111 process.exit(1);
112 }
113 // Default = multilingual `small`, NOT `small.en`. Per media-use: ".en models
114 // mistranslate non-English and mis-handle accented speech; default to small (auto-detects
115 // language)." We hardcoded small.en before — it hallucinated a wrong transcript on an
116 // accented speaker. Pass `small.en` only for known-clean-English; tough accents → a larger model.
117 const model = process.argv[3] || process.env.WHISPER_MODEL || "small";
118 const language = process.argv[4] || process.env.WHISPER_LANG || "";
119 const out = path.join(project, "transcript.json");
120
121 // already in our schema? skip — but validate the SHAPE, not just the keys:
122 // `hyperframes init` drops a whisper.cpp segment/token-format transcript.json
123 // (offsets-in-ms, nested tokens) that can carry a `words` key yet poison the
124 // compilers. Only a word-level {text,start,end} array counts as normalized.
125 try {
126 const d = JSON.parse(fs.readFileSync(out, "utf8"));
127 const wordShaped =
128 d &&
129 Array.isArray(d.words) &&
130 d.words.length > 0 &&
131 d.words.every(
132 (w) =>
133 w &&
134 typeof (w.text ?? w.word) === "string" &&
135 Number.isFinite(w.start) &&
136 Number.isFinite(w.end) &&
137 w.end < 36000, // ms-offset formats blow past any sane seconds value
138 );
139 if (wordShaped && d.language_code) {
140 console.log("[transcribe] already normalized, skipping");
141 return;
142 }
143 if (d && !wordShaped) {
144 console.log(
145 "[transcribe] existing transcript.json is NOT word-level (init stub / segment format) — regenerating",
146 );
147 }
148 } catch {}
149
150 const src = ensureSource(project);
151 if (!fs.existsSync(src)) {
152 console.error(`[transcribe] no source in ${project}`);
153 process.exit(2);
154 }
155 const audio = path.join(project, "audio.mp3");
156 if (!fs.existsSync(audio))
157 cp.execFileSync(
158 "ffmpeg",
159 ["-y", "-i", src, "-vn", "-acodec", "libmp3lame", "-q:a", "2", audio],
160 { stdio: "ignore" },
161 );
162
163 // ── engine: WhisperX (preferred — wav2vec2 forced alignment gives word timings far
164 // tighter than whisper.cpp's segment-interpolated ones; our gates are 80ms-strict) →
165 // fallback hyperframes whisper.cpp. Force with TRANSCRIBE_ENGINE=whisper|whisperx.
166 let words = null,
167 engine = null;
168 const wantWx = (process.env.TRANSCRIBE_ENGINE || "whisperx") === "whisperx";
169 if (wantWx) {
170 try {
171 const wav = path.join(project, "_wx_audio.wav");
172 cp.execFileSync("ffmpeg", ["-y", "-i", src, "-vn", "-ac", "1", "-ar", "16000", wav], {
173 stdio: "ignore",
174 });
175 const outDir = path.join(project, "_wx_out");
176 fs.mkdirSync(outDir, { recursive: true });
177 const wxModel = model.replace(/\.en$/, ""); // whisperx model names are multilingual ids
178 // Pin whisperx so `uvx` fetches a reproducible build instead of resolving
179 // "latest" on every run (a supply-chain + determinism foot-gun). Override
180 // with $WHISPERX_VERSION if you've validated a different release.
181 const whisperxSpec = `whisperx==${process.env.WHISPERX_VERSION || "3.8.6"}`;
182 const wxArgs = [
183 "--python",
184 "3.12",
185 "--from",
186 whisperxSpec,
187 "whisperx",
188 wav,
189 "--model",
190 wxModel,
191 "--device",
192 "cpu",
193 "--compute_type",
194 "int8",
195 "--output_dir",
196 outDir,
197 "--output_format",
198 "json",
199 "--no_align_deletes",
200 "--print_progress",
201 "False",
202 ];
203 if (language) wxArgs.push("--language", language);
204 // strip our flag if this whisperx build doesn't know it
205 let r = cp.spawnSync("uvx", wxArgs, { encoding: "utf8", timeout: 600000 });
206 if ((r.status || 0) !== 0 && /no_align_deletes/.test(r.stderr || "")) {
207 r = cp.spawnSync(
208 "uvx",
209 wxArgs.filter((a) => a !== "--no_align_deletes"),
210 { encoding: "utf8", timeout: 600000 },
211 );
212 }
213 if ((r.status || 0) !== 0)
214 throw new Error(
215 (r.stderr || "whisperx failed").split("\n").slice(-4).join(" ").slice(0, 300),
216 );
217 const wxJson = JSON.parse(fs.readFileSync(path.join(outDir, "_wx_audio.json"), "utf8"));
218 const wx = [];
219 for (const seg of wxJson.segments || [])
220 for (const w of seg.words || []) {
221 // alignment occasionally yields a word with no timing (OOV) — interpolate from neighbors later; mark null now
222 wx.push({ text: String(w.word || "").trim(), start: w.start, end: w.end, type: "word" });
223 }
224 // interpolate missing timings from neighbors (rare OOV/number cases)
225 for (let i = 0; i < wx.length; i++) {
226 if (wx[i].start == null || wx[i].end == null) {
227 const prevEnd = i > 0 ? wx[i - 1].end : 0;
228 const nextStart = wx.slice(i + 1).find((x) => x.start != null);
229 const ns = nextStart ? nextStart.start : prevEnd + 0.3;
230 wx[i].start = prevEnd;
231 wx[i].end = Math.max(prevEnd + 0.05, ns - 0.02);
232 }
233 }
234 if (wx.length) {
235 words = wx.filter((w) => w.text);
236 engine = `whisperx(${wxModel}+wav2vec2)`;
237 }
238 try {
239 fs.unlinkSync(wav);
240 } catch {}
241 } catch (e) {
242 console.error(
243 `[transcribe] whisperx unavailable (${String(e.message || e).slice(0, 160)}) — falling back to whisper.cpp`,
244 );
245 }
246 }
247
248 if (!words) {
249 // run hyperframes Whisper → writes a flat word array to <dir>/transcript.json
250 const cli = path.join(hfRoot(), "packages", "cli", "dist", "cli.js");
251 const args = ["transcribe", audio, "-d", project, "--json", "--model", model];
252 if (language) args.push("--language", language);
253 let info = {};
254 try {
255 const so = cp.execFileSync("node", [cli, ...args], { encoding: "utf8" });
256 const line = so.trim().split("\n").filter(Boolean).pop();
257 info = JSON.parse(line);
258 } catch (e) {
259 console.error("[transcribe] hyperframes whisper failed:", e.message);
260 process.exit(1);
261 }
262 const flatPath = info.transcriptPath || out;
263 const flat = JSON.parse(fs.readFileSync(flatPath, "utf8"));
264 const arr = Array.isArray(flat) ? flat : flat.words || [];
265 words = arr
266 .filter((w) => (w.text ?? w.word) != null)
267 .map((w) => ({
268 text: w.text ?? w.word,
269 start: w.start ?? w.t0,
270 end: w.end ?? w.t1,
271 type: "word",
272 }));
273 engine = `whisper.cpp(${model})`;
274 }
275
276 // Tail-hallucination guard: drop words whisper placed entirely inside a terminal
277 // silence (it fabricates e.g. repeated "I'm sorry." over dead air). Word START past
278 // the audible end (+0.4s slack) = fabricated; real final words start before it.
279 const ae = audibleEnd(audio);
280 let trimmedTail = 0;
281 if (ae && ae.speechEnd < ae.total - 0.8) {
282 const keep = words.filter((w) => w.start <= ae.speechEnd + 0.4);
283 trimmedTail = words.length - keep.length;
284 if (trimmedTail > 0) {
285 console.error(
286 `[transcribe] ⚠ trimmed ${trimmedTail} trailing word(s) starting after the audible end ` +
287 `(${ae.speechEnd.toFixed(2)}s; clip ${ae.total.toFixed(2)}s) — whisper hallucinates over silent tails.`,
288 );
289 words = keep;
290 }
291 }
292
293 const text = words
294 .map((w) => w.text)
295 .join(" ")
296 .replace(/\s+([,.!?;:])/g, "$1")
297 .trim();
298 fs.writeFileSync(
299 out,
300 JSON.stringify(
301 {
302 text,
303 language_code: language || "en",
304 engine,
305 words,
306 ...(trimmedTail ? { trimmed_tail_words: trimmedTail } : {}),
307 },
308 null,
309 2,
310 ),
311 );
312 console.log(`[transcribe] ${engine} ${words.length} words → ${out}`);
313 console.log(`[transcribe] text: ${text.slice(0, 160)}${text.length > 160 ? "…" : ""}`);
314
315 // No-speech guard: whisper returns confident hallucinations over silence (e.g. the
316 // whole clip as "Thank you."). The decision gate REFUSES "no speech" — operationalize
317 // it so an agent trusting the transcript can't sail past the gate.
318 const meanDb = meanVolumeDb(audio);
319 if (meanDb != null && meanDb < -45) {
320 console.error(
321 `\n[transcribe] ⚠ NEAR-SILENT AUDIO — mean ${meanDb.toFixed(1)} dB (real speech ≈ -16..-26 dB).`,
322 );
323 console.error(
324 ` This transcript is almost certainly a Whisper hallucination, NOT real speech.`,
325 );
326 console.error(
327 ` Per the decision gate, REFUSE "no speech" — confirm with \`ffmpeg -i <src> -af silencedetect\`;`,
328 );
329 console.error(` do NOT author captions from fabricated words.`);
330 }
331}
332main();
