Setting the file. One moment.
Matte · Embedded Captions · heygen-com/hyperframes · Skills Docs
Back to the top of the page Hershey Script1
Lines
280 scripts/ matte.cjs
JavaScript · 280 lines · 9 KB
14
* (products, phones) may drop out intermittently, letting captions pass in
15 * front. Never assume: sample frames_fg/ before placing the hero.
16 *
17 * Pipeline:
18 * source.mp4 → hyperframes remove-background → ProRes 4444 .mov (lossless
19 * alpha; temp, deleted) → ffmpeg fps=<matte.fps> → frames_fg/f_%04d.png.
20 * frames_bg/ is extracted at the same rate (preview tooling reads it).
21 *
22 * node matte.cjs <project-dir>
23 * Reads: <project>/source.mp4 (any video in the project dir is adopted)
24 * Writes: <project>/frames_fg/f_%04d.png (RGBA, subject opaque),
25 * <project>/frames_bg/f_%04d.png, <project>/matte.fps
26 * Env: HYPERFRAMES_ROOT — hyperframes checkout (default ~/Downloads/hyperframes)
27 */
28 const path = require ( "path" );
29 const fs = require ( "fs" );
30 const os = require ( "os" );
31 const cp = require ( "child_process" );
32
33 function hfCli () {
34 const roots = [
35 process.env. HYPERFRAMES_ROOT ,
36 path. resolve (__dirname, ".." , ".." , ".." ), // skills/embedded-captions/scripts → repo root if in-repo
37 path. join (os. homedir (), "Downloads" , "hyperframes" ),
38 ]. filter (Boolean);
39 for ( const root of roots) {
40 const cli = path. join (root, "packages" , "cli" , "dist" , "cli.js" );
41 if (fs. existsSync (cli)) return cli;
42 }
43 console. error ( "[matte] cannot find hyperframes cli — set HYPERFRAMES_ROOT to a built checkout" );
44 process. exit ( 3 );
45 }
46
47 function ensureSource ( project ) {
48 const src = path. join (project, "source.mp4" );
49 if ( ! fs. existsSync (src)) {
50 const found = fs
51 . readdirSync (project)
52 . filter (( f ) => / \. (mp4 | mov | webm | mkv) $ / i . test (f) && ! f. startsWith ( "_" ))
53 . map (( f ) => path. join (project, f))[ 0 ];
54 if ( ! found) return src;
55 try {
56 fs. symlinkSync (path. basename (found), src);
57 } catch {
58 fs. copyFileSync (found, src);
59 }
60 console. log ( `[matte] resolved source.mp4 -> ${ path . basename ( found ) }` );
61 }
62 return src;
63 }
64
65 function rateOf ( expr ) {
66 const [ n , d ] = String (expr || "" ). split ( "/" );
67 const f = parseFloat (n) / parseFloat (d || "1" );
68 return Number. isFinite (f) && f > 0 ? f : 0 ;
69 }
70
71 // Prefer avg_frame_rate (frames/duration — the truth) over r_frame_rate (the
72 // container's nominal tick rate, which lies on VFR sources: a 24fps screen
73 // recording can carry r_frame_rate=60 and would 2.5x-desync the matte).
74 function probeRates ( src ) {
75 try {
76 const out = cp
77 . execFileSync ( "ffprobe" , [
78 "-v" ,
79 "0" ,
80 "-select_streams" ,
81 "v:0" ,
82 "-show_entries" ,
83 "stream=r_frame_rate,avg_frame_rate" ,
84 "-of" ,
85 "default=nk=1:nw=1" ,
86 "--" ,
87 src,
88 ])
89 . toString ()
90 . trim ()
91 . split ( " \n " );
92 return { r: rateOf (out[ 0 ]), avg: rateOf (out[ 1 ]) };
93 } catch {
94 return { r: 0 , avg: 0 };
95 }
96 }
97
98 function probeFps ( src ) {
99 const { r , avg } = probeRates (src);
100 const f = avg || r;
101 return f > 0 ? Math. max ( 1 , Math. round (f)) : 24 ;
102 }
103
104 // VFR when nominal and actual disagree by >5% — the remove-background engine
105 // mishandles VFR timestamps (observed: 2251 fg frames vs 902 bg on one clip),
106 // so VFR sources get normalized to CFR before matting.
107 function isVfr ( src ) {
108 const { r , avg } = probeRates (src);
109 return r > 0 && avg > 0 && Math. abs (r - avg) / avg > 0.05 ;
110 }
111
112 function extractFrames ( src , dst , fps , extra = []) {
113 fs. mkdirSync (dst, { recursive: true });
114 if (fs. readdirSync (dst). some (( f ) => f. endsWith ( ".png" ))) return false ;
115 cp. execFileSync (
116 "ffmpeg" ,
117 [ "-y" , "-i" , src, "-vf" , `fps=${ fps }` , ... extra, path. join (dst, "f_%04d.png" )],
118 { stdio: "ignore" },
119 );
120 return true ;
121 }
122
123 function countPngs ( dir ) {
124 try {
125 return fs. readdirSync (dir). filter (( f ) => f. endsWith ( ".png" )). length ;
126 } catch {
127 return 0 ;
128 }
129 }
130
131 async function main () {
132 const project = path. resolve (process.argv[ 2 ] || "" );
133 if ( ! process.argv[ 2 ]) {
134 console. error ( "usage: matte.cjs <project-dir>" );
135 process. exit ( 1 );
136 }
137 const src = ensureSource (project);
138 if ( ! fs. existsSync (src)) {
139 console. error ( `[matte] no source video found in ${ project }` );
140 process. exit ( 2 );
141 }
142
143 const fpsFile = path. join (project, "matte.fps" );
144 // read-with-catch (no exists-then-read TOCTOU): a missing/corrupt fps file
145 // simply falls through to probing the source.
146 let fps = 0 ;
147 try {
148 fps = parseInt (fs. readFileSync (fpsFile, "utf8" ). replace ( / \D / g , "" ), 10 ) || 0 ;
149 } catch {
150 /* no cached fps — probe below */
151 }
152 if ( ! fps) fps = probeFps (src);
153 fs. writeFileSync (fpsFile, String (fps));
154
155 const framesBg = path. join (project, "frames_bg" );
156 const framesFg = path. join (project, "frames_fg" );
157 if ( extractFrames (src, framesBg, fps))
158 console. log ( `[matte] source fps=${ fps } → frames_bg extracted` );
159
160 const want = countPngs (framesBg);
161 if (want > 0 && countPngs (framesFg) >= want) {
162 console. log ( `[matte] frames_fg already complete (${ want } frames) — nothing to do` );
163 return ;
164 }
165
166 // 1) subject matte via hyperframes (ProRes 4444 keeps the alpha lossless).
167 // VFR sources are normalized to CFR first — remove-background trusts
168 // timestamps and emits a desynced frame count on VFR input (the ghost
169 // double-subject bug: the pasted matte runs at the wrong speed).
170 let matteSrc = src;
171 if ( isVfr (src)) {
172 const cfr = path. join (project, "_src_cfr.mp4" );
173 if ( ! fs. existsSync (cfr)) {
174 console. log (
175 `[matte] VFR source detected (nominal != actual fps) → normalizing to ${ fps }fps CFR for matting` ,
176 );
177 cp. execFileSync (
178 "ffmpeg" ,
179 [
180 "-y" ,
181 "-i" ,
182 src,
183 "-fps_mode" ,
184 "cfr" ,
185 "-r" ,
186 String (fps),
187 "-c:v" ,
188 "libx264" ,
189 "-crf" ,
190 "16" ,
191 "-preset" ,
192 "fast" ,
193 "-an" ,
194 cfr,
195 ],
196 { stdio: "ignore" },
197 );
198 }
199 matteSrc = cfr;
200 }
201 const mov = path. join (project, "_matte_tmp.mov" );
202 const t0 = Date. now ();
203 const cached = fs. existsSync (
204 path. join (
205 os. homedir (),
206 ".cache" ,
207 "hyperframes" ,
208 "background-removal" ,
209 "models" ,
210 "u2net_human_seg.onnx" ,
211 ),
212 );
213 console. log (
214 `[matte] hyperframes remove-background (u2net_human_seg${ cached ? "" : "; first run downloads ~168 MB"})… model load takes ~1-2 min with no output — not hung` ,
215 );
216 const r = cp. spawnSync ( "node" , [ hfCli (), "remove-background" , matteSrc, "-o" , mov], {
217 stdio: [ "ignore" , "pipe" , "pipe" ],
218 encoding: "utf8" ,
219 });
220 if (r.status !== 0 || ! fs. existsSync (mov)) {
221 console. error ( "[matte] remove-background FAILED:" );
222 console. error ((r.stderr || r.stdout || "" ). split ( " \n " ). slice ( - 8 ). join ( " \n " ));
223 process. exit ( 4 );
224 }
225
226 // 2) burst to RGBA pngs at the project rate
227 fs. mkdirSync (framesFg, { recursive: true });
228 cp. execFileSync (
229 "ffmpeg" ,
230 [ "-y" , "-i" , mov, "-vf" , `fps=${ fps }` , "-pix_fmt" , "rgba" , path. join (framesFg, "f_%04d.png" )],
231 { stdio: "ignore" },
232 );
233 fs. rmSync (mov, { force: true });
234
235 // 3) count parity with frames_bg. Composite overlays fg by INDEX at matte.fps,
236 // so any fg/bg count mismatch is a time desync (subject ghosting). Handle
237 // BOTH directions: pad when short, linearly remap when long — and shout
238 // when the mismatch is big enough to mean broken timestamps upstream.
239 let got = countPngs (framesFg);
240 const tol = Math. max ( 2 , Math. round (want * 0.01 ));
241 if (Math. abs (got - want) > tol) {
242 console. error (
243 `[matte] WARN frame-count desync: fg=${ got } vs bg=${ want } (tolerance ${ tol }). ` +
244 `Reconciling by remap — if the source is VFR this run predates the CFR fix; ` +
245 `delete frames_fg/ frames_bg/ matte.fps and re-run matte.cjs.` ,
246 );
247 }
248 if (got > want && want > 0 ) {
249 // keep want frames sampled evenly across got (re-times fg onto the bg timeline)
250 const keep = [];
251 for ( let j = 1 ; j <= want; j ++ )
252 keep. push (Math. min (got, Math. max ( 1 , Math. round ((j - 0.5 ) * (got / want)))));
253 const tmp = path. join (project, "_fg_remap" );
254 fs. mkdirSync (tmp, { recursive: true });
255 keep. forEach (( srcIdx , k ) => {
256 fs. copyFileSync (
257 path. join (framesFg, `f_${ String ( srcIdx ). padStart ( 4 , "0" ) }.png` ),
258 path. join (tmp, `f_${ String ( k + 1 ). padStart ( 4 , "0" ) }.png` ),
259 );
260 });
261 fs. rmSync (framesFg, { recursive: true , force: true });
262 fs. renameSync (tmp, framesFg);
263 got = countPngs (framesFg);
264 }
265 while (got < want && got > 0 ) {
266 fs. copyFileSync (
267 path. join (framesFg, `f_${ String ( got ). padStart ( 4 , "0" ) }.png` ),
268 path. join (framesFg, `f_${ String ( got + 1 ). padStart ( 4 , "0" ) }.png` ),
269 );
270 got ++ ;
271 }
272 console. log (
273 `[matte] done in ${ (( Date . now () - t0 ) / 1000 ). toFixed ( 1 ) }s → fg=${ got } bg=${ want } @ ${ fps }fps${ got === want ? " (parity ok)" : ""}` ,
274 );
275 }
276
277 main (). catch (( e ) => {
278 console. error ( "[matte]" , e.message);
279 process. exit ( 1 );
280 });
