Setting the file. One moment.
Eval · Media Use · heygen-com/hyperframes · Skills Docs
Back to the top of the page ⋯
scripts/11 files
Script Cutlist
scripts/ eval.mjs
JavaScript · 369 lines · 15 KB
;
17 import { join, basename, resolve, dirname } from "node:path" ;
18 import { execFileSync } from "node:child_process" ;
19 import { tmpdir } from "node:os" ;
20 import { fileURLToPath } from "node:url" ;
21
22 const SCRIPT_DIR = dirname ( fileURLToPath ( import . meta .url));
23 const REPO_ROOT = resolve ( SCRIPT_DIR , ".." , ".." , ".." );
24 const RESOLVE_SCRIPT = join ( SCRIPT_DIR , "resolve.mjs" );
25
26 const TEST_BLOCKS = [
27 "registry/blocks/nyc-paris-flight" ,
28 "registry/blocks/macos-tahoe-liquid-glass" ,
29 "registry/blocks/blue-sweater-intro-video" ,
30 "registry/blocks/vpn-youtube-spot" ,
31 "registry/blocks/apple-money-count" ,
32 "registry/blocks/liquid-glass-notification" ,
33 "registry/blocks/instagram-follow" ,
34 ];
35
36 // Run resolve.mjs with args as a literal argv array (no shell), so values
37 // interpolated from manifest metadata (--intent prompt, --type) can't inject
38 // shell. Mirrors the execFileSync fix in probe.mjs / heygen-search.mjs.
39 function run ( args , opts = {}) {
40 try {
41 return {
42 ok: true ,
43 output: execFileSync (process.execPath, [ RESOLVE_SCRIPT , ... args], {
44 encoding: "utf8" ,
45 timeout: 15000 ,
46 stdio: "pipe" ,
47 ... opts,
48 }). trim (),
49 };
50 } catch (err) {
51 return { ok: false , output: (err.stdout || "" ) + (err.stderr || "" ), code: err.status };
52 }
53 }
54
55 function countAssetFiles ( dir ) {
56 const assetsDir = join (dir, "assets" );
57 if ( ! existsSync (assetsDir)) return { count: 0 , files: [] };
58 const files = [];
59 function walk ( d , base = "" ) {
60 for ( const e of readdirSync (d, { withFileTypes: true })) {
61 const rel = base ? `${ base }/${ e . name }` : e.name;
62 if (e. isDirectory ()) walk ( join (d, e.name), rel);
63 else files. push (rel);
64 }
65 }
66 walk (assetsDir);
67 return { count: files. length , files };
68 }
69
70 function evalBlock ( blockPath ) {
71 const fullPath = join ( REPO_ROOT , blockPath);
72 if ( ! existsSync (fullPath)) return null ;
73
74 const name = basename (blockPath);
75 const tmp = mkdtempSync ( join ( tmpdir (), `mu-eval-${ name }-` ));
76
77 try {
78 cpSync (fullPath, tmp, { recursive: true });
79
80 // baseline: what the agent sees WITHOUT media-use
81 const baseline = countAssetFiles (tmp);
82 const htmlFiles = readdirSync (tmp). filter (( f ) => f. endsWith ( ".html" ));
83
84 // parse compositions for asset references
85 const assetRefs = [];
86 for ( const hf of htmlFiles) {
87 const html = readFileSync ( join (tmp, hf), "utf8" );
88 const srcMatches = html. matchAll ( /src= ["'] ( [ ^ "'] +? ) ["'] / g );
89 for ( const m of srcMatches) {
90 const ref = m[ 1 ];
91 if (ref. startsWith ( "data:" ) || ref. startsWith ( "http" )) continue ;
92 assetRefs. push ({ composition: hf, ref });
93 }
94 const urlMatches = html. matchAll ( /url \( ["'] ? ( [ ^ "')] +? ) ["'] ? \) / g );
95 for ( const m of urlMatches) {
96 const ref = m[ 1 ];
97 if (ref. startsWith ( "data:" ) || ref. startsWith ( "http" ) || ref. startsWith ( "#" )) continue ;
98 assetRefs. push ({ composition: hf, ref });
99 }
100 }
101
102 // with media-use: run --adopt
103 const adoptResult = run ([ "--adopt" , "--project" , tmp, "--json" ]);
104 let adopted = { ok: false , adopted: 0 , assets: [] };
105 if (adoptResult.ok) {
106 try {
107 adopted = JSON . parse (adoptResult.output);
108 } catch {
109 /* */
110 }
111 }
112
113 // read the generated index
114 const indexPath = join (tmp, ".media" , "index.md" );
115 const indexContent = existsSync (indexPath)
116 ? readFileSync (indexPath, "utf8" )
117 : "(no index generated)" ;
118
119 // read manifest for detail
120 const manifestPath = join (tmp, ".media" , "manifest.jsonl" );
121 const manifest = existsSync (manifestPath)
122 ? readFileSync (manifestPath, "utf8" )
123 . trim ()
124 . split ( " \n " )
125 . map (( l ) => {
126 try {
127 return JSON . parse (l);
128 } catch {
129 return null ;
130 }
131 })
132 . filter (Boolean)
133 : [];
134
135 // test resolve cache hit: try resolving something that was adopted
136 let resolveTest = null ;
137 if (manifest. length > 0 ) {
138 const first = manifest[ 0 ];
139 const prompt = first.provenance?.prompt || first.description;
140 const r = run ([ "--type" , first.type, "--intent" , prompt, "--project" , tmp, "--json" ]);
141 if (r.ok) {
142 try {
143 resolveTest = JSON . parse (r.output);
144 } catch {
145 /* */
146 }
147 }
148 }
149
150 // test resolve miss: try resolving something that doesn't exist
151 const missResult = run ([
152 "--type" ,
153 "bgm" ,
154 "--intent" ,
155 "nonexistent query xyz" ,
156 "--project" ,
157 tmp,
158 "--json" ,
159 ]);
160 let resolveMiss = null ;
161 if ( ! missResult.ok) {
162 try {
163 resolveMiss = JSON . parse (missResult.output);
164 } catch {
165 /* */
166 }
167 }
168
169 // coverage: which composition refs are covered by the manifest
170 const manifestPaths = new Set (manifest. map (( m ) => m.path));
171 const coverage = assetRefs. map (( r ) => ({
172 ... r,
173 covered: manifestPaths. has (r.ref),
174 }));
175
176 return {
177 name,
178 baseline: { fileCount: baseline.count, files: baseline.files, htmlCount: htmlFiles. length },
179 compositions: htmlFiles,
180 assetRefs: coverage,
181 adopted: { count: adopted.adopted, assets: adopted.assets || [] },
182 index: indexContent,
183 manifest,
184 resolveTest,
185 resolveMiss,
186 };
187 } finally {
188 rmSync (tmp, { recursive: true , force: true });
189 }
190 }
191
192 function generateReport ( results ) {
193 const all = results. filter (Boolean);
194 const passed = all. filter (( r ) => r.adopted.count > 0 );
195
196 const rows = results
197 . filter (Boolean)
198 . map (( r ) => {
199 const hasMetadata = r.manifest. some (( m ) => m.duration || m.width);
200 const cacheHit = r.resolveTest?._source === "cached" ;
201 const missHandled = r.resolveMiss?.ok === false ;
202
203 return `<tr>
204 <td><strong>${ r . name }</strong></td>
205 <td>${ r . baseline . fileCount } files, ${ r . baseline . htmlCount } comp${ r . baseline . htmlCount === 1 ? "" : "s"}</td>
206 <td>${ r . adopted . count } adopted</td>
207 <td>${ hasMetadata ? "<span class='pass'>with metadata</span>" : "<span class='warn'>no metadata</span>"}</td>
208 <td>${ cacheHit ? "<span class='pass'>cache hit</span>" : "<span class='warn'>no hit</span>"}</td>
209 <td>${ missHandled ? "<span class='pass'>handled</span>" : "<span class='fail'>unexpected</span>"}</td>
210 </tr>` ;
211 })
212 . join ( " \n " );
213
214 const details = results
215 . filter (Boolean)
216 . filter (( r ) => r.adopted.count > 0 )
217 . map (( r ) => {
218 const assetRows = r.manifest
219 . map (( m ) => {
220 const dur = m.duration != null ? `${ m . duration }s` : "—" ;
221 const dims = m.width && m.height ? `${ m . width }×${ m . height }` : "—" ;
222 return `<tr><td>${ m . id }</td><td>${ m . type }</td><td>${ dur }</td><td>${ dims }</td><td class="path">${ m . path }</td><td>${ m . description || ""}</td></tr>` ;
223 })
224 . join ( " \n " );
225
226 const coveredCount = r.assetRefs. filter (( c ) => c.covered). length ;
227 const totalRefs = r.assetRefs. length ;
228 const coveragePct = totalRefs > 0 ? Math. round ((coveredCount / totalRefs) * 100 ) : 100 ;
229
230 const refRows = r.assetRefs
231 . map (
232 ( c ) =>
233 `<tr><td class="path">${ c . composition }</td><td class="path">${ c . ref }</td><td>${ c . covered ? "<span class='pass'>covered</span>" : "<span class='warn'>not in manifest</span>"}</td></tr>` ,
234 )
235 . join ( " \n " );
236
237 return `<div class="block-detail">
238 <h3>${ r . name }</h3>
239 <p style="font-size:13px;color:var(--muted)">${ r . compositions . length } composition${ r . compositions . length === 1 ? "" : "s"}: ${ r . compositions . join ( ", " ) }</p>
240
241 <div class="comparison">
242 <div class="col">
243 <h4>Baseline (no media-use)</h4>
244 <p>Agent sees: ${ r . baseline . fileCount } raw files in assets/<br>No metadata, no type info, no relationship to compositions.</p>
245 <pre class="file-list">${ r . baseline . files . join ( " \n " ) || "(no assets)"}</pre>
246 </div>
247 <div class="col">
248 <h4>With media-use (after --adopt)</h4>
249 <p>Agent reads index.md — structured, typed, with metadata:</p>
250 <pre class="index">${ escapeHtml ( r . index ) }</pre>
251 </div>
252 </div>
253
254 ${
255 totalRefs > 0
256 ? `<h4>Composition → asset coverage <span class="${ coveragePct === 100 ? "pass" : "warn"}">${ coveragePct }%</span> (${ coveredCount }/${ totalRefs } refs)</h4>
257 <table class="manifest">
258 <thead><tr><th>composition</th><th>asset reference</th><th>in manifest?</th></tr></thead>
259 <tbody>${ refRows }</tbody>
260 </table>`
261 : ""
262 }
263
264 <h4>Manifest records</h4>
265 <table class="manifest">
266 <thead><tr><th>id</th><th>type</th><th>dur</th><th>dims</th><th>path</th><th>description</th></tr></thead>
267 <tbody>${ assetRows }</tbody>
268 </table>
269 </div>` ;
270 })
271 . join ( " \n " );
272
273 return `<title>media-use eval report</title>
274 <style>
275 :root { --bg: #fafaf7; --text: #1b1b18; --muted: #7a756a; --accent: #0d7377; --good: #1a7a3a; --warn: #b45309; --fail: #dc2626; --border: #e8e5df; --surface: #fff; --mono: ui-monospace, 'SF Mono', Menlo, Consolas, monospace; --sans: system-ui, -apple-system, sans-serif; --serif: Georgia, serif }
276 * { box-sizing: border-box; margin: 0 } body { background: var(--bg); color: var(--text); font-family: var(--serif); line-height: 1.6; font-size: 15px; padding: 40px 24px }
277 .wrap { max-width: 1100px; margin: 0 auto }
278 h1 { font-family: var(--sans); font-size: 28px; font-weight: 700; margin-bottom: 8px; letter-spacing: -.02em }
279 h2 { font-family: var(--sans); font-size: 20px; font-weight: 650; margin: 32px 0 12px; letter-spacing: -.01em }
280 h3 { font-family: var(--sans); font-size: 17px; font-weight: 650; margin: 24px 0 8px }
281 h4 { font-family: var(--sans); font-size: 14px; font-weight: 600; margin: 16px 0 6px; color: var(--muted) }
282 p { margin-bottom: 10px }
283 .meta { font-family: var(--mono); font-size: 12px; color: var(--muted); margin-bottom: 24px }
284 .summary { display: flex; gap: 16px; margin: 16px 0; flex-wrap: wrap }
285 .stat { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 18px; flex: 1; min-width: 140px }
286 .stat .num { font-family: var(--sans); font-size: 28px; font-weight: 700; color: var(--accent) }
287 .stat .label { font-family: var(--mono); font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: .1em }
288 table { width: 100%; border-collapse: collapse; font-size: 13px; font-family: var(--sans); margin: 8px 0 }
289 th { text-align: left; font-family: var(--mono); font-size: 10px; letter-spacing: .08em; text-transform: uppercase; color: var(--muted); border-bottom: 2px solid var(--border); padding: 6px 8px; font-weight: 700 }
290 td { border-bottom: 1px solid var(--border); padding: 7px 8px; vertical-align: top }
291 td.path { font-family: var(--mono); font-size: 12px; color: var(--muted); max-width: 300px; overflow: hidden; text-overflow: ellipsis }
292 .pass { color: var(--good); font-weight: 600 } .warn { color: var(--warn); font-weight: 600 } .fail { color: var(--fail); font-weight: 600 }
293 .comparison { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin: 12px 0 }
294 @media(max-width:700px) { .comparison { grid-template-columns: 1fr } }
295 .col { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 16px }
296 .col h4 { margin-top: 0 }
297 pre { font-family: var(--mono); font-size: 12px; background: #1b1b18; color: #d4d0c8; border-radius: 6px; padding: 12px 14px; overflow-x: auto; margin: 6px 0; line-height: 1.5 }
298 pre.file-list { background: var(--bg); color: var(--muted); border: 1px solid var(--border) }
299 pre.index { white-space: pre; }
300 .block-detail { border-top: 1px solid var(--border); padding-top: 20px; margin-top: 20px }
301 .verdict { margin-top: 24px; padding: 16px 20px; border-radius: 8px; font-family: var(--sans); font-size: 15px }
302 .verdict.ship { background: #edfbf0; border: 1px solid #1a7a3a; color: #1a7a3a }
303 .verdict.wait { background: #fff3ec; border: 1px solid #d94f04; color: #d94f04 }
304 </style>
305 <div class="wrap">
306 <h1>media-use eval report</h1>
307 <p class="meta">${ new Date (). toISOString (). slice ( 0 , 10 ) } · ${ all . length } blocks evaluated · baseline vs. media-use --adopt</p>
308
309 <div class="summary">
310 <div class="stat"><div class="num">${ all . length }</div><div class="label">blocks tested</div></div>
311 <div class="stat"><div class="num">${ passed . length }</div><div class="label">with assets</div></div>
312 <div class="stat"><div class="num">${ all . reduce (( s , r ) => s + r . adopted . count , 0 ) }</div><div class="label">assets adopted</div></div>
313 <div class="stat"><div class="num">${ all . filter (( r ) => r . manifest . some (( m ) => m . duration || m . width )). length }</div><div class="label">with ffprobe metadata</div></div>
314 <div class="stat"><div class="num">${ (() => {
315 const refs = all. flatMap (( r ) => r.assetRefs);
316 const covered = refs. filter (( c ) => c.covered). length ;
317 return refs. length > 0 ? Math. round ((covered / refs. length ) * 100 ) + "%" : "—" ;
318 } )() }</div><div class="label">composition coverage</div></div>
319 </div>
320
321 <h2>Results matrix</h2>
322 <table>
323 <thead><tr><th>Block</th><th>Baseline</th><th>Adopted</th><th>Metadata</th><th>Cache hit</th><th>Miss handling</th></tr></thead>
324 <tbody>${ rows }</tbody>
325 </table>
326
327 <h2>Before / after comparisons</h2>
328 ${ details }
329
330 <div class="verdict ${ passed . length >= 3 ? "ship" : "wait"}">
331 ${
332 passed . length >= 3
333 ? `<strong>Ship it.</strong> ${ passed . length }/${ all . length } blocks adopted successfully with metadata. Resolve cache hits work. Miss handling is clean.`
334 : `<strong>Needs work.</strong> Only ${ passed . length } blocks adopted. Check the failures above.`
335 }
336 </div>
337 </div>` ;
338 }
339
340 function escapeHtml ( str ) {
341 return str. replace ( /&/ g , "&" ). replace ( /</ g , "<" ). replace ( />/ g , ">" );
342 }
343
344 console. log ( "media-use eval · running against registry blocks... \n " );
345
346 const results = [];
347 for ( const block of TEST_BLOCKS ) {
348 const fullPath = join ( REPO_ROOT , block);
349 if ( ! existsSync (fullPath)) {
350 console. log ( ` skip ${ basename ( block ) } (not found)` );
351 results. push ( null );
352 continue ;
353 }
354 process.stdout. write ( ` ${ basename ( block ) }...` );
355 const result = evalBlock (block);
356 if (result) {
357 console. log (
358 ` ${ result . adopted . count } adopted, ${ result . manifest . filter (( m ) => m . duration || m . width ). length } with metadata` ,
359 );
360 } else {
361 console. log ( " failed" );
362 }
363 results. push (result);
364 }
365
366 const report = generateReport (results);
367 const outPath = join ( SCRIPT_DIR , ".." , "eval-report.html" );
368 writeFileSync (outPath, report);
369 console. log ( ` \n Report: ${ outPath }` );