Setting the file. One moment.
Collect Signals · Vercel Optimize · vercel-labs/agent-skills · Skills Docs
Back to the top of the page
Position
66 of 155 scripts/ collect-signals.mjs
JavaScript · 576 lines · 21 KB
16
getContract,
17 getUsage,
18 filterUsageByProject,
19 inferPlan,
20 queryMetric,
21 detectStack,
22 redactSensitiveText,
23 } from '../lib/vercel.mjs' ;
24 import { classifyFrameworkSupport } from '../lib/framework-support.mjs' ;
25 import { QUERIES, TIME_WINDOW, normalizerFor } from '../lib/queries.mjs' ;
26
27 const SCHEMA_VERSION = '1.2' ;
28
29 const log = ( ... args ) => console. error ( '[collect-signals]' , ... args);
30
31 function parseArgs ( argv ) {
32 let explicitProjectId = null ;
33 let continueWithoutObservability = process.env. VERCEL_OPTIMIZE_CONTINUE_WITHOUT_OBSERVABILITY === '1' ;
34 let continueUnsupportedFramework = process.env. VERCEL_OPTIMIZE_CONTINUE_UNSUPPORTED_FRAMEWORK === '1' ;
35
36 for ( const arg of argv) {
37 if (arg === '--continue-without-observability' ) {
38 continueWithoutObservability = true ;
39 continue ;
40 }
41 if (arg === '--continue-unsupported-framework' ) {
42 continueUnsupportedFramework = true ;
43 continue ;
44 }
45 if (arg. startsWith ( '--' )) {
46 throw new Error ( `UNKNOWN_ARG: ${ arg }` );
47 }
48 if ( ! explicitProjectId) {
49 explicitProjectId = arg;
50 continue ;
51 }
52 throw new Error ( `UNKNOWN_ARG: ${ arg }` );
53 }
54
55 return { explicitProjectId, continueWithoutObservability, continueUnsupportedFramework };
56 }
57
58 async function main () {
59 const { explicitProjectId , continueWithoutObservability , continueUnsupportedFramework } = parseArgs (process.argv. slice ( 2 ));
60
61 log ( 'checking Vercel CLI version…' );
62 const cli = await checkCliVersion ();
63 log ( `vercel CLI v${ cli . join ( '.' ) } OK` );
64
65 log ( 'checking auth…' );
66 await checkAuth ();
67 log ( 'auth OK' );
68
69 log ( 'resolving project id…' );
70 const project = await resolveProjectId (explicitProjectId);
71 if ( ! project) {
72 throw new Error (
73 'NO_PROJECT_ID: pass one as argv, set VERCEL_PROJECT_ID, or run `vercel link` in this directory.'
74 );
75 }
76 log ( `project link resolved (source=${ project . source }; teamScope=${ project . orgId ? 'yes' : 'no'})` );
77
78 if ( ! project.orgId) {
79 throw new Error ( 'PROJECT_SCOPE_UNRESOLVED: the project was resolved without an owner account. Ask the user which Vercel team or personal scope owns the project, then rerun from a linked app directory or set VERCEL_PROJECT_ID with VERCEL_ORG_ID for that scope.' );
80 }
81
82 log ( 'checking framework support…' );
83 const stack = await detectStack ();
84 const frameworkSupport = classifyFrameworkSupport (stack);
85 log ( `framework=${ stack . framework }@${ stack . frameworkVersion ?? '?'} support=${ frameworkSupport . status }` );
86
87 if ( ! frameworkSupport.ok && ! continueUnsupportedFramework) {
88 writeOutput ({
89 schemaVersion: SCHEMA_VERSION ,
90 collectedAt: new Date (). toISOString (),
91 timeWindow: TIME_WINDOW ,
92 projectId: project.projectId,
93 orgId: project.orgId,
94 projectIdSource: project.source,
95 commandScope: null ,
96 frameworkSupport,
97 frameworkSupportBlocker: frameworkSupport.blocker,
98 frameworkSupportDetail: frameworkSupport.detail,
99 observabilityPlus: null ,
100 observabilityPlusPreflight: null ,
101 observabilityPlusUsable: null ,
102 observabilityPlusBlocker: null ,
103 observabilityPlusBlockerDetail: null ,
104 plan: {
105 plan: 'uncertain' ,
106 reason: 'not collected before unsupported-framework confirmation' ,
107 },
108 project: null ,
109 contract: null ,
110 usage: null ,
111 usageScope: null ,
112 usageTeamTotal: null ,
113 usageError: 'NOT_COLLECTED_UNSUPPORTED_FRAMEWORK' ,
114 stack,
115 metrics: {},
116 metricsSchema: null ,
117 }, { usable: true , blocker: null , detail: 'Observability Plus was not checked.' }, frameworkSupport);
118 return ;
119 }
120
121 if ( ! frameworkSupport.ok && continueUnsupportedFramework) {
122 log ( 'continuing after unsupported framework blocker because --continue-unsupported-framework was set' );
123 }
124
125 log ( 'resolving Vercel CLI command scope…' );
126 const commandScope = await resolveCommandScope (project);
127 if ( ! commandScope.ok) {
128 throw new Error ( `SCOPE_UNRESOLVED: ${ commandScope . detail } Run \` vercel switch <team> \` or re-link with \` vercel link --yes --project <project-name-or-id> --team <team-slug> \` .` );
129 }
130 const scope = commandScope.cliScope || undefined ;
131 log ( `command scope resolved (source=${ commandScope . source }; scoped=${ scope ? 'yes' : 'no'})` );
132
133 log ( 'validating linked project belongs to the resolved scope…' );
134 const projectCfg = await getProjectConfig (project.projectId, project.orgId);
135 const projectScope = validateProjectScope (projectCfg, project);
136 if ( ! projectScope.ok) {
137 throw new Error ( `PROJECT_SCOPE_MISMATCH: ${ projectScope . detail } Ask the user to confirm the exact Vercel project and team/personal scope, then rerun after \` vercel link --yes --project <project-name-or-id> --team <team-slug> \` or after setting both VERCEL_PROJECT_ID and VERCEL_ORG_ID for the intended scope.` );
138 }
139 log ( `project scope verified (source=${ projectScope . source })` );
140
141 log ( 'checking Observability Plus configuration…' );
142 const observabilityPlusConfig = await checkObservabilityPlusConfiguration ({
143 orgId: project.orgId,
144 projectId: project.projectId,
145 });
146 log ( `observabilityPlusPreflight=${ observabilityPlusConfig . access === true ? 'enabled' : observabilityPlusConfig . blocker ?? 'unknown'} (${ observabilityPlusConfig . source })` );
147
148 let oplus = observabilityPlusConfig.access === true ;
149 if (observabilityPlusConfig.access == null ) {
150 log ( 'Observability Plus configuration preflight inconclusive; falling back to metrics schema probe…' );
151 oplus = await hasObservabilityPlus (scope);
152 }
153 log ( `observabilityPlus=${ oplus }` );
154
155 const schema = oplus ? await getMetricsSchema (scope) : null ;
156 if (oplus && schema) {
157 const count = Array. isArray (schema) ? schema. length : (schema.metrics?. length ?? 0 );
158 log ( `metric catalog: ${ count } metrics available` );
159 }
160
161 // Check one cheap metric before pulling slower project context. If this fails,
162 // the orchestrator can ask the user immediately instead of waiting on billing.
163 let metrics = {};
164 let metricsCanaryOk = false ;
165 if (oplus) {
166 log ( `checking Observability Plus metrics access (window=${ TIME_WINDOW })…` );
167 const t0 = Date. now ();
168 const canary = await queryMetric ( 'vercel.request.count' , {
169 aggregation: 'sum' ,
170 since: TIME_WINDOW ,
171 limit: 1 ,
172 scope,
173 });
174 metricsCanaryOk = !! canary?.ok;
175 if ( ! metricsCanaryOk) {
176 metrics = {
177 observabilityPlusCanary: {
178 ... canary,
179 metricId: 'vercel.request.count' ,
180 aggregation: 'sum' ,
181 },
182 };
183 log ( `metrics access check failed: ${ canary ?. code ?? 'unknown'} — skipping full metrics fan-out` );
184 } else {
185 log ( `metrics access check passed in ${ Date . now () - t0 }ms` );
186 }
187 } else {
188 log ( 'skipping metric queries (Observability Plus preflight did not confirm access)' );
189 }
190
191 let oplusDiag = observabilityPlusConfig.access === false
192 ? {
193 usable: false ,
194 blocker: observabilityPlusConfig.blocker,
195 detail: observabilityPlusConfig.detail,
196 }
197 : (metricsCanaryOk
198 ? { usable: true , blocker: null , detail: 'Observability Plus metrics access check passed.' }
199 : diagnoseObservabilityPlus (metrics, oplus));
200
201 if ( ! oplusDiag.usable && ! continueWithoutObservability) {
202 writeOutput ({
203 schemaVersion: SCHEMA_VERSION ,
204 collectedAt: new Date (). toISOString (),
205 timeWindow: TIME_WINDOW ,
206 projectId: project.projectId,
207 orgId: project.orgId,
208 projectIdSource: project.source,
209 commandScope,
210 observabilityPlus: oplus,
211 observabilityPlusPreflight: observabilityPlusConfig,
212 observabilityPlusUsable: oplusDiag.usable,
213 observabilityPlusBlocker: oplusDiag.blocker,
214 observabilityPlusBlockerDetail: oplusDiag.detail,
215 frameworkSupport,
216 frameworkSupportBlocker: frameworkSupport.blocker,
217 frameworkSupportDetail: frameworkSupport.detail,
218 plan: {
219 plan: 'uncertain' ,
220 reason: 'not collected before Observability Plus blocker confirmation' ,
221 },
222 project: projectCfg,
223 contract: null ,
224 usage: null ,
225 usageScope: null ,
226 usageTeamTotal: null ,
227 usageError: 'NOT_COLLECTED_OBSERVABILITY_BLOCKED' ,
228 stack: null ,
229 metrics,
230 metricsSchema: schema,
231 }, oplusDiag);
232 return ;
233 }
234
235 if ( ! oplusDiag.usable && continueWithoutObservability) {
236 log ( 'continuing after Observability Plus blocker because --continue-without-observability was set' );
237 }
238
239 log ( 'pulling account plan + contract + usage in parallel…' );
240 const [ accountPlan , contract , usageResult ] = await Promise . all ([
241 getAccountPlan (project.orgId || scope),
242 getContract (scope),
243 getUsage ({ days: 14 , scope }),
244 ]);
245
246 let usage = null ;
247 let usageContextMismatch = false ;
248 let usageTotalCost = null ;
249 let usageScope = 'team' ;
250 let usageTeamTotal = null ;
251 if (usageResult?.ok) {
252 usage = usageResult.data;
253 const contractContext = contract?.context;
254 if (usage?.context && contractContext && usage.context !== contractContext) {
255 usageContextMismatch = true ;
256 log ( `usage: WARNING context mismatch — returned context=${ usage . context } but project team=${ contractContext }; treating usage as unavailable for this project` );
257 usage = null ;
258 } else {
259 // Capture team total pre-filter so the report can label "this project vs team-wide" honestly.
260 usageTeamTotal = sumUsageCosts (usage);
261 const filterResult = filterUsageByProject (usage, project.projectId, projectCfg?.name);
262 if (filterResult.matched) {
263 usage = filterResult.filtered;
264 usageScope = 'project' ;
265 usageTotalCost = sumUsageCosts (usage);
266 log ( `usage: filtered to project — ~$${ usageTotalCost . toFixed ( 2 ) } (team-wide ~$${ usageTeamTotal . toFixed ( 2 ) }; unattributed ~$${ filterResult . unattributedTotal . toFixed ( 2 ) })` );
267 } else {
268 usageTotalCost = usageTeamTotal;
269 log ( `usage: ~$${ usageTotalCost . toFixed ( 2 ) } billed across services (team-wide — no per-project usage rows matched the linked project; report will label this team-wide)` );
270 }
271 }
272 } else {
273 log ( `usage: unavailable (${ usageResult ?. code ?? 'unknown'}) — degrading to scanner+metrics-only mode` );
274 }
275
276 const planInfo = inferPlan (contract, { accountPlan, usageTotalCost });
277 log ( `plan=${ planInfo . plan } (${ planInfo . reason })` );
278
279 if (projectCfg?.error) {
280 log ( `project config: failed (${ projectCfg . error }) — gates that need it will skip` );
281 }
282
283 log ( `stack: ${ stack . framework }@${ stack . frameworkVersion ?? '?'} ${ stack . hasAppRouter ? 'app-router' : ''}${ stack . hasPagesRouter ? ' pages-router' : ''}${ stack . orm !== 'none' ? ` orm=${ stack . orm }` : ''}` );
284
285 // Each query is wrapped; one failure degrades only that metric.
286 if (oplus && metricsCanaryOk) {
287 log ( `querying observability metrics (${ QUERIES . length } metrics in parallel)…` );
288 const t0 = Date. now ();
289 metrics = await collectMetrics (scope);
290 const wallMs = Date. now () - t0;
291 const counts = Object. fromEntries (
292 Object. entries (metrics). map (([ k , v ]) => {
293 if ( ! v) return [k, 'null' ];
294 if ( ! v.ok) return [k, `err:${ v . code }` ];
295 const rows = Array. isArray (v.rows) ? v.rows. length : 0 ;
296 return [k, `${ rows } rows` ];
297 })
298 );
299 log ( `metrics collected in ${ wallMs }ms: ${ JSON . stringify ( counts ) }` );
300 }
301
302 // The `vercel metrics schema` probe alone is NOT a reliable usability signal:
303 // it can return OK while per-route queries fail with payment_required (metrics
304 // unavailable for the team) or FORBIDDEN (auth-scope mismatch). Diagnose AFTER
305 // running queries by counting failure codes so the orchestrator can PAUSE and
306 // surface the choice before falling back to scanner-only mode.
307 oplusDiag = observabilityPlusConfig.access === false
308 ? {
309 usable: false ,
310 blocker: observabilityPlusConfig.blocker,
311 detail: observabilityPlusConfig.detail,
312 }
313 : diagnoseObservabilityPlus (metrics, oplus);
314
315
316 const output = {
317 schemaVersion: SCHEMA_VERSION ,
318 collectedAt: new Date (). toISOString (),
319 timeWindow: TIME_WINDOW ,
320 projectId: project.projectId,
321 orgId: project.orgId,
322 projectIdSource: project.source,
323 commandScope,
324 observabilityPlus: oplus,
325 observabilityPlusPreflight: observabilityPlusConfig,
326 observabilityPlusUsable: oplusDiag.usable,
327 observabilityPlusBlocker: oplusDiag.blocker,
328 observabilityPlusBlockerDetail: oplusDiag.detail,
329 frameworkSupport,
330 frameworkSupportBlocker: frameworkSupport.blocker,
331 frameworkSupportDetail: frameworkSupport.detail,
332 plan: planInfo,
333 project: projectCfg,
334 contract,
335 usage,
336 usageScope,
337 usageTeamTotal,
338 usageError: usageResult?.ok
339 ? (usageContextMismatch ? 'USAGE_CONTEXT_MISMATCH' : null )
340 : (usageResult?.code ?? 'UNKNOWN' ),
341 stack,
342 metrics,
343 metricsSchema: schema,
344 };
345
346 writeOutput (output, oplusDiag);
347 }
348
349 function writeOutput ( output , oplusDiag , frameworkSupport = output.frameworkSupport) {
350 if (frameworkSupport?.blocker) {
351 log ( `⚠ Framework is not supported for metric-backed route-to-file optimization: ${ frameworkSupport . detail }` );
352 log ( ' The orchestrator should PAUSE and ask whether to continue with a limited platform/scanner audit.' );
353 }
354 if ( ! oplusDiag.usable) {
355 log ( `⚠ Observability Plus is NOT usable on this project: blocker=${ oplusDiag . blocker } (${ oplusDiag . detail })` );
356 log ( ' The orchestrator should PAUSE and follow the blocker-specific remediation before proceeding.' );
357 }
358
359 process.stdout. write ( JSON . stringify (output, null , 2 ) + ' \n ' );
360 log ( 'done' );
361 }
362
363 function validateProjectScope ( projectCfg , project ) {
364 if ( ! projectCfg || projectCfg.error) {
365 return {
366 ok: false ,
367 source: 'project-api' ,
368 detail: `The resolved account could not read the resolved project (project API error=${ projectCfg ?. error ?? 'unknown'}).` ,
369 };
370 }
371
372 if (projectCfg.id && String (projectCfg.id) !== String (project.projectId)) {
373 return {
374 ok: false ,
375 source: 'project-api' ,
376 detail: 'The project API returned a different project than the collector resolved from the link or environment.' ,
377 };
378 }
379
380 const ownerId = firstString (
381 projectCfg.accountId,
382 projectCfg.orgId,
383 projectCfg.ownerId,
384 projectCfg.teamId,
385 projectCfg.team?.id,
386 projectCfg.account?.id,
387 projectCfg.owner?.id,
388 );
389 if (ownerId && project.orgId && String (ownerId) !== String (project.orgId)) {
390 return {
391 ok: false ,
392 source: 'project-api' ,
393 detail: 'The project API returned an owner account that differs from the collector-resolved account.' ,
394 };
395 }
396
397 return {
398 ok: true ,
399 source: ownerId ? 'project-api-owner' : 'project-api-readable' ,
400 };
401 }
402
403 function firstString ( ... values ) {
404 return values. find (( value ) => typeof value === 'string' && value. trim () !== '' ) ?? null ;
405 }
406
407 async function collectMetrics ( scope ) {
408 const results = await Promise . all (
409 QUERIES . map ( async ( entry ) => {
410 const r = await queryMetric (entry.metricId, {
411 aggregation: entry.aggregation,
412 groupBy: entry.groupBy,
413 filter: entry.filter,
414 since: TIME_WINDOW ,
415 limit: entry.limit,
416 scope,
417 });
418 return [entry, r];
419 })
420 );
421
422 const out = {};
423 for ( const [ entry , result ] of results) {
424 out[entry.id] = enrichEntry (entry, result);
425 }
426 return out;
427 }
428
429 function enrichEntry ( entry , result ) {
430 if ( ! result?.ok) {
431 return {
432 ... result,
433 metricId: entry.metricId,
434 aggregation: entry.aggregation,
435 groupBy: entry.groupBy,
436 };
437 }
438 const normalize = normalizerFor (entry);
439 const { rows } = normalize (result.data);
440 return {
441 ... result,
442 rows,
443 metricId: entry.metricId,
444 aggregation: entry.aggregation,
445 groupBy: entry.groupBy,
446 };
447 }
448
449 // `vercel usage --format json` shape is documented but not stable across CLI
450 // versions; try several roots, return null if none match.
451 function sumUsageCosts ( usage ) {
452 if ( ! usage) return null ;
453 if ( typeof usage.totalCost === 'number' ) return usage.totalCost;
454 if ( typeof usage.totals?.billedCost === 'number' ) return usage.totals.billedCost;
455 if (Array. isArray (usage.services)) {
456 return usage.services. reduce (( s , x ) => s + (x.billedCost ?? x.cost ?? 0 ), 0 );
457 }
458 if (Array. isArray (usage.breakdown?.data)) {
459 return usage.breakdown.data. reduce (( s , d ) => {
460 if (Array. isArray (d.services)) {
461 return s + d.services. reduce (( ss , x ) => ss + (x.billedCost ?? x.cost ?? 0 ), 0 );
462 }
463 return s + (d.billedCost ?? d.cost ?? 0 );
464 }, 0 );
465 }
466 return null ;
467 }
468
469 // Returns { usable, blocker, detail }. `blocker` enum:
470 // null | 'no_oplus_probe' | 'project_disabled' | 'payment_required' |
471 // 'forbidden' | 'daily_quota_exceeded' | 'project_not_found' |
472 // 'not_linked' | 'all_failed_other' | 'no_traffic'
473 export function diagnoseObservabilityPlus ( metrics , oplusProbe ) {
474 if ( ! oplusProbe) {
475 return {
476 usable: false ,
477 blocker: 'no_oplus_probe' ,
478 detail: 'vercel metrics schema returned non-OK; the team does not have Observability Plus enabled.' ,
479 };
480 }
481
482 const entries = Object. values (metrics);
483 if (entries. length === 0 ) {
484 return { usable: false , blocker: 'no_oplus_probe' , detail: 'No metrics were attempted.' };
485 }
486
487 const failures = entries. filter (( m ) => m && m.ok === false );
488 const successes = entries. filter (( m ) => m && m.ok !== false );
489
490 if (successes. length === 0 ) {
491 const codeCounts = new Map ();
492 for ( const f of failures) {
493 const code = String (f.code ?? 'unknown' ). toLowerCase ();
494 codeCounts. set (code, (codeCounts. get (code) ?? 0 ) + 1 );
495 }
496 const top = [ ... codeCounts. entries ()]. sort (( a , b ) => b[ 1 ] - a[ 1 ])[ 0 ];
497 const topCode = top?.[ 0 ] ?? 'unknown' ;
498 if ( /daily_quota_exceeded/ . test (topCode)) {
499 return {
500 usable: false ,
501 blocker: 'daily_quota_exceeded' ,
502 detail: `${ top [ 1 ] }/${ entries . length } metric queries hit the daily Observability query limit. Retry after the next UTC midnight reset.` ,
503 };
504 }
505 if ( /payment_required/ . test (topCode)) {
506 const text = failures
507 . map (( f ) => `${ f . message ?? ''} \n ${ f . stderr ?? ''}` )
508 . join ( ' \n ' )
509 . toLowerCase ();
510 if (
511 /subscription to observability plus [\s\S] {0,160} required/ . test (text) ||
512 /observability plus [\s\S] {0,160} not enabled/ . test (text)
513 ) {
514 return {
515 usable: false ,
516 blocker: 'no_oplus_probe' ,
517 detail: `${ top [ 1 ] }/${ entries . length } metric queries need route-level Observability Plus data. Enable Observability Plus, then re-run the metric-backed audit.` ,
518 };
519 }
520 return {
521 usable: false ,
522 blocker: 'payment_required' ,
523 detail: `${ top [ 1 ] }/${ entries . length } metric queries returned payment_required. Route-level metrics were recognized for this team, but these queries are not usable. Check the team's Observability Plus subscription or event quota.` ,
524 };
525 }
526 if ( /forbidden | not_authorized | 403/ . test (topCode)) {
527 return {
528 usable: false ,
529 blocker: 'forbidden' ,
530 detail: `${ top [ 1 ] }/${ entries . length } metric queries returned FORBIDDEN. Auth-scope mismatch — likely logged in to the wrong team (run \` vercel switch \` ).` ,
531 };
532 }
533 if ( /project_not_found/ . test (topCode)) {
534 return {
535 usable: false ,
536 blocker: 'project_not_found' ,
537 detail: `Project ID not visible to the auth'd team. Run \` vercel switch \` or verify the project ID.` ,
538 };
539 }
540 if ( /not_linked/ . test (topCode)) {
541 return {
542 usable: false ,
543 blocker: 'not_linked' ,
544 detail: `${ top [ 1 ] }/${ entries . length } metric queries returned NOT_LINKED. Link the app directory first: \` vercel link --yes --project <project-name-or-id> --cwd <project-dir> \` ; add \` --team <team-id-or-slug> \` when the team is known.` ,
545 };
546 }
547 return {
548 usable: false ,
549 blocker: 'all_failed_other' ,
550 detail: `Every metric query failed; top error code was \` ${ topCode } \` (${ top ?.[ 1 ] }/${ entries . length }).` ,
551 };
552 }
553
554 // Some queries succeeded; zero rows across the board = "no traffic in window",
555 // NOT an Observability Plus billing issue.
556 const totalRows = successes. reduce (( s , m ) => s + (Array. isArray (m.rows) ? m.rows. length : 0 ), 0 );
557 if (totalRows === 0 ) {
558 return {
559 usable: true ,
560 blocker: 'no_traffic' ,
561 detail: 'Observability Plus queries succeeded but every metric returned 0 rows. Either the project has no traffic in the 14-day window, or Observability Plus retention is limited (free tier = 1 day on Pro).' ,
562 };
563 }
564
565 return { usable: true , blocker: null , detail: 'Observability Plus is usable; queries returned data.' };
566 }
567
568 // Run main() only as a CLI; the test suite imports diagnoseObservabilityPlus directly.
569 import { fileURLToPath } from 'node:url' ;
570 import { realpathSync } from 'node:fs' ;
571 if (process.argv[ 1 ] && realpathSync (process.argv[ 1 ]) === realpathSync ( fileURLToPath ( import . meta .url))) {
572 main (). catch (( err ) => {
573 console. error ( '[collect-signals] FAILED:' , redactSensitiveText (err.message));
574 process. exit ( 1 );
575 });
576 }