Setting the file. One moment.
Verify And Regen · Vercel Optimize · vercel-labs/agent-skills · Skills Docs
Back to the top of the page scripts/verify-and-regen.mjs
scripts/ verify-and-regen.mjs
JavaScript · 346 lines · 15 KB
;
11 import { extractClaims, summarizeClaimResults } from '../lib/extract-claims.mjs' ;
12 import { gradeRecommendation, applyQualityFloor } from '../lib/grade-recommendation.mjs' ;
13 import { deriveProjectFacts } from '../lib/project-facts.mjs' ;
14 import { resolveRepoRoot } from '../lib/repo-root.mjs' ;
15 import { applySanitizers } from '../lib/sanitizers/index.mjs' ;
16
17 const SCHEMA_VERSION = '1.0' ;
18 const REGEN_PASS_RATE_THRESHOLD = 0.8 ;
19 // 1/1 failed is as broken as 1/5; below 2 claims is below the noise floor.
20 const REGEN_MIN_CLAIMS = 2 ;
21 // The Poor/Fair grade boundary — Poor recs erode trust faster than recall helps.
22 const QUALITY_FLOOR = 0.55 ;
23
24 const log = ( ... a ) => console. error ( '[verify-and-regen]' , ... a);
25
26 async function main () {
27 const args = parseArgs (process.argv. slice ( 2 ));
28 if ( ! args.recsPath) {
29 console. error ( 'usage: node scripts/verify-and-regen.mjs <recommendations.json> [--signals merged.json] [--repo-root DIR] [--out FILE]' );
30 process. exit ( 1 );
31 }
32
33 const recs = JSON . parse ( await readFile (args.recsPath, 'utf-8' ));
34 if ( ! Array. isArray (recs)) {
35 console. error ( '[verify-and-regen] FATAL: recommendations.json must be an array of rec objects' );
36 process. exit ( 2 );
37 }
38
39 let framework, version, cacheComponents, knownFindings = [], projectFacts = [], signals = null ;
40 if (args.signalsPath) {
41 signals = JSON . parse ( await readFile (args.signalsPath, 'utf-8' ));
42 const stack = signals.stack ?? signals.codebase?.stack ?? {};
43 framework = stack.framework;
44 version = stack.frameworkVersion;
45 cacheComponents = stack.cacheComponents;
46 knownFindings = (signals.codebase?.findings ?? signals.findings ?? [])
47 . filter (( f ) => f.file && (f.line != null ))
48 . map (( f ) => ({ file: f.file, line: f.line }));
49 projectFacts = deriveProjectFacts (signals);
50 if (projectFacts. length > 0 ) {
51 log ( `project facts in play: ${ projectFacts . map (( f ) => f . id ). join ( ', ' ) }` );
52 }
53 }
54
55 // Repo-root priority: (1) signals.project.rootDirectory from Vercel API
56 // (authoritative — returns "apps/<name>" so cwd can be back-mapped without
57 // filesystem probing), (2) supplied --repo-root, (3) walk-up from cwd.
58 const rootResult = await resolveRepoRoot (recs, args.repoRoot, process. cwd (), signals);
59 const repoRoot = rootResult.root;
60 if (rootResult.source === 'api' ) {
61 log ( `repo-root from Vercel API: '${ repoRoot }' (rootDirectory='${ rootResult . apiOffset }')` );
62 } else if (rootResult.source === 'auto-detected' ) {
63 log ( `repo-root auto-detected: '${ repoRoot }' (probe: ${ rootResult . probe })` );
64 } else if (rootResult.source === 'corrected' ) {
65 log ( `repo-root auto-corrected: '${ args . repoRoot }' → '${ repoRoot }' (sub-agent paths resolve there)` );
66 }
67 log ( `verifying ${ recs . length } rec(s) — framework=${ framework ?? '?'}@${ version ?? '?'} repoRoot=${ repoRoot }` );
68
69 // knownFindings MUST combine scanner findings + sub-agent's verified
70 // findingRefs — scanner-only grounding would miss every metric-gate rec.
71 // Abstentions are first-class outputs ({abstain:true, candidateRef, reason})
72 // and MUST NOT be graded; the abstention IS the answer.
73 const recsGraded = [];
74 const abstentions = [];
75 const observations = [];
76 const sanitizerDropped = [];
77 for ( let i = 0 ; i < recs. length ; i ++ ) {
78 const rec = recs[i];
79
80 if (rec?.abstain === true ) {
81 abstentions. push ({
82 index: i,
83 candidateRef: rec.candidateRef ?? null ,
84 reason: rec.reason ?? '(no reason recorded)' ,
85 });
86 // Observation: real non-perf signal worth surfacing (regression, error storm).
87 if (rec.observation && typeof rec.observation === 'object' && rec.observation.summary) {
88 observations. push ({
89 index: i,
90 candidateRef: rec.candidateRef ?? null ,
91 summary: String (rec.observation.summary),
92 evidence: rec.observation.evidence ?? null ,
93 suggestedAction: rec.observation.suggestedAction ?? null ,
94 kind: rec.observation.kind ?? 'other' ,
95 });
96 }
97 continue ;
98 }
99
100 const baseClaimCtx = {
101 framework,
102 version,
103 repoRoot,
104 projectFacts,
105 projectRootDirectory: signals?.project?.rootDirectory ?? null ,
106 cacheComponents,
107 signals,
108 };
109 const initialClaims = extractClaims (rec, baseClaimCtx);
110 const initialVerifyResults = await Promise . all (initialClaims. map (( c ) => verifyClaim (c)));
111 const initialClaimsWithResults = initialVerifyResults. map (( r , j ) => ({
112 ... r,
113 type: initialClaims[j]?.type,
114 claimType: initialClaims[j]?.type,
115 claim: initialClaims[j],
116 }));
117 const sanitizerResult = await applySanitizers (rec, {
118 framework,
119 version,
120 signals,
121 verifyResults: initialClaimsWithResults,
122 });
123 if ( ! sanitizerResult.kept) {
124 sanitizerDropped. push ({
125 index: i,
126 candidateRef: rec.candidateRef ?? null ,
127 what: rec.what ?? null ,
128 reason: sanitizerResult.dropReason ?? 'automated-check' ,
129 });
130 continue ;
131 }
132
133 const sanitizedRec = sanitizerResult.rec;
134 const claims = extractClaims (sanitizedRec, baseClaimCtx);
135 const verifyResults = await Promise . all (claims. map (( c ) => verifyClaim (c)));
136 const verification = summarizeClaimResults (verifyResults);
137
138 // A findingRef whose file_exists claim verified counts as grounding evidence.
139 const verifiedRefs = [];
140 for ( let j = 0 ; j < claims. length ; j ++ ) {
141 const c = claims[j];
142 const r = verifyResults[j];
143 if (r?.disposition !== 'verified' ) continue ;
144 if (c.sourceField === 'findingRefs' && c.type === 'file_exists' ) {
145 const ref = (rec.findingRefs ?? []). find (( x ) => String (x). startsWith (c.file + ':' ));
146 if (ref) {
147 const m = String (ref). match ( / ^ ( . +? ):( \d + ) $ / );
148 if (m) verifiedRefs. push ({ file: m[ 1 ], line: Number (m[ 2 ]) });
149 }
150 }
151 }
152 const recKnownFindings = [ ... knownFindings, ... verifiedRefs];
153 const quality = gradeRecommendation (sanitizedRec, { knownFindings: recKnownFindings });
154
155 recsGraded. push ({
156 index: i,
157 rec: { ... sanitizedRec, verification, verifyResults, quality },
158 claims,
159 verifyResults,
160 verification,
161 quality,
162 });
163 }
164
165 // Project-config contradictions are a HARD trigger: a "turn on Fluid" rec on
166 // a project where Fluid is already on passes 8/9 claims but is the wrong rec.
167 // passRate alone won't catch this.
168 const regenPlan = [];
169 for ( const g of recsGraded) {
170 const { passRate , verifiable } = g.verification;
171 const claimsWithResults = g.verifyResults. map (( r , j ) => ({ ... r, claim: g.claims[j] }));
172 const contradictions = claimsWithResults. filter (
173 ( r ) => r.disposition === 'failed' && r.claim?.type === 'does_not_contradict_project_config'
174 );
175 const triggeredByPassRate = verifiable >= REGEN_MIN_CLAIMS && passRate < REGEN_PASS_RATE_THRESHOLD ;
176 const cacheSafetyFailures = claimsWithResults. filter (
177 ( r ) => r.disposition === 'failed' && (
178 r.claim?.type === 'cache_vary_matches_dynamic_inputs' ||
179 r.claim?.type === 'cache_vary_cardinality_safe'
180 )
181 );
182 const semanticSafetyFailures = claimsWithResults. filter (
183 ( r ) => r.disposition === 'failed' && (
184 r.claim?.type === 'next_cached_not_found_causal_support' ||
185 r.claim?.type === 'next_stable_cache_api_for_version' ||
186 r.claim?.type === 'next_runtime_cache_api_for_version' ||
187 r.claim?.type === 'next_cache_life_single_execution' ||
188 r.claim?.type === 'next_cache_lifetime_freshness_supported' ||
189 r.claim?.type === 'next_cache_components_route_chain_file' ||
190 r.claim?.type === 'next_cache_life_cdn_header_semantics' ||
191 r.claim?.type === 'image_response_headers_citation' ||
192 r.claim?.type === 'next_image_priority_api_for_version' ||
193 r.claim?.type === 'next_cache_components_route_segment_config' ||
194 r.claim?.type === 'next_route_revalidate_static_prereq' ||
195 r.claim?.type === 'next_cache_tag_invalidation_supported' ||
196 r.claim?.type === 'cache_rec_not_error_dominated_or_acknowledged' ||
197 r.claim?.type === 'cache_control_header_syntax' ||
198 r.claim?.type === 'cache_control_headers_citation' ||
199 r.claim?.type === 'cache_404_long_ttl_safety' ||
200 r.claim?.type === 'route_error_not_found_status_and_scope' ||
201 r.claim?.type === 'immutable_dynamic_route_safety' ||
202 r.claim?.type === 'auth_guard_parallelization_safety' ||
203 r.claim?.type === 'parallelization_impact_not_overclaimed' ||
204 r.claim?.type === 'parallelization_not_cpu_bound_work' ||
205 r.claim?.type === 'runtime_error_cause_supported' ||
206 r.claim?.type === 'vercel_ignore_command_project_state'
207 )
208 );
209 const triggeredByContradiction = contradictions. length > 0 ;
210 const triggeredByCacheSafety = cacheSafetyFailures. length > 0 ;
211 const triggeredBySemanticSafety = semanticSafetyFailures. length > 0 ;
212 if ( ! triggeredByPassRate && ! triggeredByContradiction && ! triggeredByCacheSafety && ! triggeredBySemanticSafety) continue ;
213
214 const failures = claimsWithResults
215 . filter (( r ) => r.disposition === 'failed' )
216 . slice ( 0 , 5 );
217 regenPlan. push ({
218 index: g.index,
219 candidateRef: g.rec.candidateRef ?? null ,
220 what: g.rec.what ?? null ,
221 verifiableClaimCount: verifiable,
222 passRate,
223 regenTrigger: triggeredByContradiction
224 ? 'project_config_contradiction'
225 : triggeredByCacheSafety
226 ? 'cache_vary_safety'
227 : triggeredBySemanticSafety
228 ? 'semantic_safety'
229 : 'pass_rate_below_threshold' ,
230 topFailures: failures. map (( f ) => ({
231 claimType: f.claim?.type,
232 field: f.claim?.sourceField,
233 url: f.claim?.url,
234 file: f.claim?.file,
235 pattern: f.claim?.pattern,
236 reason: f.reason,
237 })),
238 regenBriefHint: triggeredByContradiction
239 ? 'Sub-agent recommended toggling on a project setting that is already enabled. Re-spawn with the project-config Strengths block highlighted; the rec must drop the contradictory step and keep only the actionable parts.'
240 : triggeredByCacheSafety
241 ? 'Sub-agent recommended CDN caching with unsafe or missing Vary behavior. Re-spawn with the cache safety failure highlighted; the rec must use a low-cardinality Vary header that matches the dynamic inputs, or abstain.'
242 : triggeredBySemanticSafety
243 ? 'Sub-agent made a framework-semantic claim that failed deterministic checks. Re-spawn with the failure highlighted; the rec must either add version-correct code/citations/runtime evidence or abstain.'
244 : 'Re-spawn the sub-agent with this rec \' s topFailures injected as feedback. Re-emit the rec only if regenPassRate >= originalPassRate AND citation count not gutted.' ,
245 });
246 }
247
248 const qualityCheck = applyQualityFloor (recsGraded. map (( g ) => g.rec), QUALITY_FLOOR );
249 const hardRegenIndexes = new Set (regenPlan. map (( p ) => p.index));
250 const qualityDroppedIndexes = new Set (
251 qualityCheck.dropped
252 . map (( d ) => recsGraded. findIndex (( g ) => g.rec === d.rec))
253 . filter (( i ) => i >= 0 )
254 );
255 const needsReviewIndexes = new Set (
256 recsGraded
257 . filter (( g ) => g.rec.needsReview === true )
258 . map (( g ) => g.index)
259 );
260 const verifiedRecommendations = recsGraded
261 . filter (( g ) => ! hardRegenIndexes. has (g.index) && ! qualityDroppedIndexes. has (g.index) && ! needsReviewIndexes. has (g.index))
262 . map (( g ) => g.rec);
263 const withheldRecommendations = recsGraded
264 . filter (( g ) => hardRegenIndexes. has (g.index) || qualityDroppedIndexes. has (g.index) || needsReviewIndexes. has (g.index))
265 . map (( g ) => ({
266 index: g.index,
267 candidateRef: g.rec.candidateRef ?? null ,
268 what: g.rec.what ?? null ,
269 reason: hardRegenIndexes. has (g.index)
270 ? (regenPlan. find (( p ) => p.index === g.index)?.regenTrigger ?? 'verification' )
271 : qualityDroppedIndexes. has (g.index)
272 ? 'quality_floor'
273 : 'needs_review' ,
274 }));
275
276 const summary = {
277 totalRecs: recs. length ,
278 abstentions: abstentions. length ,
279 observations: observations. length ,
280 sanitizerDropped: sanitizerDropped. length ,
281 needsRegen: regenPlan. length ,
282 qualityDropped: qualityCheck.dropped. length ,
283 needsReview: needsReviewIndexes.size,
284 verifiedRecommendations: verifiedRecommendations. length ,
285 withheldRecommendations: withheldRecommendations. length ,
286 averagePassRate: recsGraded. length > 0
287 ? round4 (recsGraded. reduce (( s , g ) => s + g.verification.passRate, 0 ) / recsGraded. length )
288 : null ,
289 averageQuality: recsGraded. length > 0
290 ? round4 (recsGraded. reduce (( s , g ) => s + g.quality.overall, 0 ) / recsGraded. length )
291 : null ,
292 };
293
294 const output = {
295 schemaVersion: SCHEMA_VERSION ,
296 summary,
297 recsGraded: recsGraded. map (( g ) => g.rec),
298 verifiedRecommendations,
299 renderableRecommendations: verifiedRecommendations,
300 withheldRecommendations,
301 abstentions,
302 observations,
303 sanitizerDropped,
304 regenPlan,
305 qualityDropped: qualityCheck.dropped. map (( d ) => ({
306 index: recsGraded. findIndex (( g ) => g.rec === d.rec),
307 candidateRef: d.rec.candidateRef ?? null ,
308 quality: d.rec.quality,
309 reason: d.reason,
310 })),
311 };
312
313 const serialized = JSON . stringify (output, null , 2 ) + ' \n ' ;
314 if (args.outPath) {
315 await mkdir ( dirname (args.outPath), { recursive: true });
316 await writeFile (args.outPath, serialized, 'utf-8' );
317 log ( `wrote ${ serialized . length }B → ${ args . outPath }` );
318 } else {
319 process.stdout. write (serialized);
320 }
321 log ( `done: ${ summary . totalRecs } records checked; ${ summary . verifiedRecommendations } ready, ${ summary . withheldRecommendations } held back, ${ summary . abstentions } found no supported change, ${ summary . sanitizerDropped } dropped by safety checks` );
322 }
323
324 function parseArgs ( argv ) {
325 const out = { positional: [] };
326 for ( let i = 0 ; i < argv. length ; i ++ ) {
327 const a = argv[i];
328 if (a === '--signals' ) out.signalsPath = argv[ ++ i];
329 else if (a. startsWith ( '--signals=' )) out.signalsPath = a. slice ( '--signals=' . length );
330 else if (a === '--repo-root' ) out.repoRoot = argv[ ++ i];
331 else if (a. startsWith ( '--repo-root=' )) out.repoRoot = a. slice ( '--repo-root=' . length );
332 else if (a === '--out' ) out.outPath = resolve (argv[ ++ i]);
333 else if (a. startsWith ( '--out=' )) out.outPath = resolve (a. slice ( '--out=' . length ));
334 else out.positional. push (a);
335 }
336 out.recsPath = out.positional[ 0 ];
337 return out;
338 }
339
340 function round4 ( n ) { return Math. round (n * 10000 ) / 10000 ; }
341
342 main (). catch (( err ) => {
343 console. error ( '[verify-and-regen] FAILED:' , err.message);
344 console. error (err.stack);
345 process. exit ( 1 );
346 });