Setting the file. One moment.
Assemble Extraction · Wix Headless Replatform · wix/skills · Skills Docs
Back to the top of the page 28.10
Workflow
This file
Number 28.11
Position 11 of 89
Type JavaScript
Size 36 KB
Lines 844 scripts/ assemble-extraction.mjs
JavaScript · 844 lines · 36 KB
12 createGap,
13 freezeSpec,
14 semanticHash,
15 sha256,
16 writeFrozenExtraction,
17 } from "./lib/extraction-contract.mjs" ;
18 import { loadApprovedRegistry, selectRegistryItem } from "./lib/component-registry.mjs" ;
19 import { classifyObservedUnit, planRecursiveUnitJobs } from "./lib/extraction-detectors.mjs" ;
20 import {
21 CONTENT_EXTRACTION_SEMANTICS_VERSION,
22 inspectContentObject,
23 removeContaminatedContent,
24 } from "./lib/content-boundary.mjs" ;
25
26 const LEGACY_OBSERVATION_FILES = [
27 "assets.json" ,
28 "control-state-contract.json" ,
29 "interaction-map.json" ,
30 "layout-blueprint.json" ,
31 "scene-contract.json" ,
32 "seo.json" ,
33 "tokens.json" ,
34 "ui-normalization.json" ,
35 "visual-assets.json" ,
36 ];
37
38 export async function assembleExtraction ({
39 outputDir ,
40 requestedUrl ,
41 resolvedUrl ,
42 canonicalUrl ,
43 page ,
44 assets = {},
45 seo = {},
46 tokens = {},
47 interactionMap = {},
48 sceneContract = {},
49 layoutBlueprint = {},
50 uiNormalization = {},
51 controlStateContract = {},
52 visualAssets = {},
53 sourceFingerprint ,
54 sourceFingerprintIncludesContentSemantics = false ,
55 captureId : suppliedCaptureId ,
56 decisionPatches = [],
57 pageResolutionArtifact = null ,
58 }) {
59 const docs = docsDir (outputDir);
60 const observedSourceFingerprint = sourceFingerprint || sha256 ({
61 resolvedUrl,
62 title: page?.title || "" ,
63 sections: (page?.sections || []). map (( section ) => ({ id: section.id, heading: section.heading, text: section.text })),
64 });
65 const fingerprint = sourceFingerprintIncludesContentSemantics
66 ? observedSourceFingerprint
67 : sha256 ({
68 sourceFingerprint: observedSourceFingerprint,
69 contentExtractionSemantics: CONTENT_EXTRACTION_SEMANTICS_VERSION ,
70 });
71 const captureId = suppliedCaptureId || `home-${ fingerprint . slice ( 0 , 12 ) }` ;
72 const extractionDir = path. join (docs, "extraction" , captureId);
73 const gaps = [];
74 const decisionRequests = [];
75 const appliedDecisions = [];
76 const patchesByRequest = new Map (decisionPatches. map (( patch ) => [patch.requestId, patch]));
77 const rawEvidenceEstimate = Buffer. byteLength ( JSON . stringify ({ page, assets, seo, tokens, interactionMap, sceneContract, layoutBlueprint, uiNormalization, controlStateContract, visualAssets }));
78 const recursiveJobs = planRecursiveUnitJobs (page?.sections || []);
79 const pageCaptureGapRefs = [];
80 if (rawEvidenceEstimate > EXTRACTION_BUDGETS .maxRawEvidenceBytes) {
81 page = { ... page, rawDomSnapshots: {}, normalizedDomSnapshots: {} };
82 const gap = createGap ({
83 id: "gap:page-capture:evidence-budget" ,
84 ownerUnitId: "page-capture:home" ,
85 missingFields: [ "rawDomSnapshots" , "normalizedDomSnapshots" ],
86 reason: `Raw evidence was estimated at ${ rawEvidenceEstimate } bytes and exceeded the ${ EXTRACTION_BUDGETS . maxRawEvidenceBytes }-byte retention budget. DOM snapshots were omitted by priority while structured observations were retained.` ,
87 evidenceRefs: [],
88 dependencyClosure: (page?.sections || []). map (( section , index ) => section.id || `section-${ String ( index + 1 ). padStart ( 3 , "0" ) }` ),
89 affectedAcceptance: [ "packet-replay" ],
90 assumptions: [ "Structured page/unit evidence remains sufficient to assemble a viable home-page shell." ],
91 omissions: [ "DOM-only replay until a targeted recapture is authorized." ],
92 });
93 gaps. push (gap);
94 pageCaptureGapRefs. push (gap.id);
95 }
96
97 const pageResolution = pageResolutionArtifact || freezeSpec ({
98 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
99 kind: "page-resolution" ,
100 id: "page-resolution:home" ,
101 pageKey: "home" ,
102 destinationRoute: "/" ,
103 status: "accepted" ,
104 source: {
105 requestedUrl,
106 resolvedUrl,
107 canonicalUrl: canonicalUrl || resolvedUrl,
108 },
109 evidence: [{ kind: "navigation" , ref: "observations/page.json" }],
110 gapRefs: [],
111 });
112
113 const pageCapture = freezeSpec ({
114 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
115 kind: "page-capture" ,
116 id: "page-capture:home" ,
117 pageKey: "home" ,
118 status: pageCaptureGapRefs. length ? "provisional" : "accepted" ,
119 sourceFingerprint: fingerprint,
120 viewportSet: Object. entries (page?.screenshots || {}). map (([ name , screenshotRef ]) => ({ name, screenshotRef: `observations/screenshots/${ path . basename ( String ( screenshotRef )) }` })),
121 sectionOrder: recursiveJobs.jobs. filter (( job ) => job.unitKind === "section" ). map (( job ) => job.id),
122 ignoredSurfaces: page?.ignoredSurfaces || [],
123 normalizationActions: page?.normalizationActions || [],
124 observationRefs: LEGACY_OBSERVATION_FILES . map (( name ) => `observations/${ name }` ),
125 extensions: {
126 "wix.replatform.content-boundary" : { semanticsVersion: CONTENT_EXTRACTION_SEMANTICS_VERSION },
127 },
128 gapRefs: pageCaptureGapRefs,
129 }, { dependencyHashes: [pageResolution.hash], gapIds: pageCaptureGapRefs });
130
131 const foundationGapRefs = [];
132 if ( ! hasFoundationEvidence (tokens)) {
133 const gap = createGap ({
134 id: "gap:foundation:tokens" ,
135 ownerUnitId: "foundation:home" ,
136 missingFields: [ "tokens" ],
137 reason: "No reliable page-wide token evidence was projected from the capture." ,
138 evidenceRefs: [ "observations/tokens.json" ],
139 dependencyClosure: [ "shared-chrome:home" , ... (page?.sections || []). map (( section , index ) => section.id || `section-${ String ( index + 1 ). padStart ( 3 , "0" ) }` )],
140 assumptions: [ "Use only directly observed computed values local to each unit." ],
141 omissions: [ "Unobserved global token aliases." ],
142 });
143 gaps. push (gap);
144 foundationGapRefs. push (gap.id);
145 }
146 const foundation = freezeSpec ({
147 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
148 kind: "foundation" ,
149 id: "foundation:home" ,
150 pageKey: "home" ,
151 status: foundationGapRefs. length ? "provisional" : "accepted" ,
152 tokens: sanitizeEvidence (tokens),
153 fonts: sanitizeEvidence (assets?.fonts || {}),
154 primitives: collectFoundationPrimitives (page),
155 globalCssIntent: collectGlobalCssIntent (tokens),
156 evidenceRefs: [ "observations/tokens.json" , "observations/assets.json" , "observations/page.json" ],
157 acceptance: { required: [ "fonts-load" , "direction-matches" , "page-background-matches" , "token-references-resolve" ] },
158 gapRefs: foundationGapRefs,
159 }, { dependencyHashes: [pageCapture.hash], gapIds: foundationGapRefs });
160
161 const metadata = freezeSpec ({
162 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
163 kind: "metadata" ,
164 id: "metadata:home" ,
165 pageKey: "home" ,
166 status: "accepted" ,
167 document: sanitizeEvidence (seo),
168 evidenceRefs: [ "observations/seo.json" , "observations/page.json" ],
169 acceptance: { required: [ "title" , "description-when-observed" , "canonical-when-observed" , "lang-and-dir" ] },
170 gapRefs: [],
171 }, { dependencyHashes: [pageCapture.hash] });
172
173 const sharedChromeGapRefs = [];
174 if ( ! page?.chrome && ! page?.footer) {
175 const gap = createGap ({
176 id: "gap:shared-chrome:unobserved" ,
177 ownerUnitId: "shared-chrome:home" ,
178 missingFields: [ "header" , "footer" ],
179 reason: "The capture did not identify a reliable header or footer boundary." ,
180 evidenceRefs: [ "observations/page.json" ],
181 assumptions: [ "Do not invent navigation or footer content." ],
182 omissions: [ "Unobserved shared chrome." ],
183 });
184 gaps. push (gap);
185 sharedChromeGapRefs. push (gap.id);
186 }
187 const sharedChromeContamination = inspectContentObject ({ header: page?.chrome || null , footer: page?.footer || null });
188 if (sharedChromeContamination.contaminated) {
189 const gap = contentContaminationGap ( "shared-chrome:home" , "observations/page.json" , sharedChromeContamination);
190 gaps. push (gap);
191 sharedChromeGapRefs. push (gap.id);
192 }
193 const safeSharedChrome = removeContaminatedContent ({ header: page?.chrome || null , footer: page?.footer || null });
194 const sharedChrome = freezeSpec ({
195 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
196 kind: "shared-chrome" ,
197 id: "shared-chrome:home" ,
198 pageKey: "home" ,
199 status: sharedChromeGapRefs. length ? "provisional" : "accepted" ,
200 header: sanitizeEvidence (safeSharedChrome.header),
201 footer: sanitizeEvidence (safeSharedChrome.footer),
202 controls: sanitizeEvidence (controlStateContract?.controls || []),
203 visualAssets: selectChromeAssets (visualAssets),
204 behavior: selectOwnedInteractions (interactionMap, "shared-chrome" ),
205 evidenceRefs: [ "observations/page.json" , "observations/control-state-contract.json" , "observations/visual-assets.json" ],
206 acceptance: { required: [ "visible-content" , "navigation-hierarchy" , "logo-identity" , "responsive-chrome" , "keyboard-navigation" ] },
207 gapRefs: sharedChromeGapRefs,
208 }, { dependencyHashes: [foundation.hash, metadata.hash], gapIds: sharedChromeGapRefs });
209
210 const specs = [foundation, metadata, sharedChrome];
211 const specPaths = [
212 { spec: foundation, relativePath: "foundation.spec.json" },
213 { spec: metadata, relativePath: "metadata.spec.json" },
214 { spec: sharedChrome, relativePath: "shared-chrome.spec.json" },
215 ];
216
217 const layouts = new Map ((layoutBlueprint?.pages || []). flatMap (( entry ) => entry.sections || []). map (( entry ) => [entry.sectionId, entry]));
218 const normalizations = new Map ((uiNormalization?.sections || []). map (( entry ) => [entry.sectionId, entry]));
219 const scenes = new Map ((sceneContract?.scenes || []). map (( entry ) => [entry.sectionId, entry]));
220 const recursiveSpecById = new Map ();
221 const jobsByDepth = [ ... recursiveJobs.jobs]. sort (( a , b ) => b.depth - a.depth || a.id. localeCompare (b.id));
222 for ( const job of jobsByDepth) {
223 const decision = resolveClassification ({
224 value: job.value,
225 unitId: job.id,
226 evidenceRef: job.evidenceRef,
227 patchesByRequest,
228 decisionRequests,
229 appliedDecisions,
230 });
231 const diagnosticGaps = job.unitKind === "section"
232 ? gapsForSection (job.value, job.id, page, decision.classification)
233 : [];
234 const contentOptions = { allowVisibleCode: Array. isArray (job.value?.visibleCodeTexts) && job.value.visibleCodeTexts. length > 0 };
235 const contamination = inspectContentObject (job.value, contentOptions);
236 const contaminationGaps = contamination.contaminated
237 ? [ contentContaminationGap (job.id, job.evidenceRef, contamination)]
238 : [];
239 const unitGaps = [ ... decision.gaps, ... diagnosticGaps, ... contaminationGaps];
240 gaps. push ( ... unitGaps);
241 const gapRefs = unitGaps. map (( gap ) => gap.id);
242 const isSection = job.unitKind === "section" ;
243 const behavior = isSection
244 ? scenes. get (job.id) || selectOwnedInteractions (interactionMap, job.id)
245 : job.value.behavior || selectOwnedInteractions (interactionMap, job.id);
246 const unit = freezeSpec ({
247 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
248 kind: "unit" ,
249 id: job.id,
250 unitKind: job.unitKind,
251 pageKey: "home" ,
252 parentUnitId: job.parentUnitId,
253 order: job.order,
254 status: gapRefs. length ? "provisional" : "accepted" ,
255 classification: {
256 ... decision.classification,
257 capabilityRequirements: capabilityRequirementsFor (job.value, decision.classification),
258 },
259 scope: {
260 evidenceRef: job.evidenceRef,
261 ... (isSection ? { screenshotRefs: Object. values (page?.screenshots || {}). map (( item ) => `observations/screenshots/${ path . basename ( String ( item )) }` ) } : {}),
262 },
263 content: contentFor (job.value, contentOptions),
264 assets: sanitizeEvidence (job.value.images || job.value.assets || []),
265 layers: sanitizeEvidence (job.value.layers || (isSection ? layouts. get (job.id)?.background?.layers : []) || []),
266 layout: sanitizeEvidence ((isSection ? layouts. get (job.id) : null ) || job.value.layout || {}),
267 styleables: sanitizeEvidence (job.value.styleables || (job.value.layoutEvidence?.regions || []). map (( region , index ) => ({
268 id: `${ job . id }:styleable-${ String ( index + 1 ). padStart ( 3 , "0" ) }` ,
269 role: region.role,
270 categories: region.styleableCategories || [],
271 computedByViewport: { desktop: region.computedStyle || {} },
272 authoredCss: region.authoredCss || [],
273 }))),
274 behavior: sanitizeEvidence (behavior || {}),
275 children: job.childIds,
276 reconstruction: isSection ? {
277 strategy: knownClassification (job.value) ? "registry-or-deterministic" : "bounded-custom" ,
278 foundationRef: foundation.id,
279 sharedChromeRef: sharedChrome.id,
280 normalization: sanitizeEvidence (normalizations. get (job.id) || null ),
281 } : {
282 strategy: "registry-or-custom" ,
283 foundationRef: foundation.id,
284 },
285 acceptance: acceptanceFor (job.value),
286 diagnostics: unitGaps. map (( gap ) => ({ gapRef: gap.id, reason: gap.reason })),
287 provenance: {
288 sourceRef: job.evidenceRef,
289 sourceUrl: resolvedUrl,
290 captureFingerprint: fingerprint,
291 detectorVersions: decision.classification.detectorVersions,
292 recursiveJobVersion: recursiveJobs.version,
293 },
294 extensions: {
295 "wix.replatform.content-boundary" : {
296 semanticsVersion: CONTENT_EXTRACTION_SEMANTICS_VERSION ,
297 visibleCode: contentOptions.allowVisibleCode,
298 },
299 },
300 gapRefs,
301 }, {
302 dependencyHashes: [foundation.hash, ... job.childIds. map (( id ) => recursiveSpecById. get (id)?.hash). filter (Boolean)],
303 gapIds: gapRefs,
304 });
305 recursiveSpecById. set (job.id, unit);
306 specs. push (unit);
307 const sectionRoot = job.id. split ( ":component-" )[ 0 ];
308 specPaths. push ({
309 spec: unit,
310 relativePath: isSection
311 ? `sections/${ job . id }/section.spec.json`
312 : `sections/${ sectionRoot }/components/${ pathSafe ( job . id ) }.spec.json` ,
313 });
314 }
315
316 const observationRefs = await writeObservations (extractionDir, {
317 page,
318 assets,
319 seo,
320 tokens,
321 interactionMap,
322 sceneContract,
323 layoutBlueprint,
324 uiNormalization,
325 controlStateContract,
326 visualAssets,
327 recursiveJobs: {
328 version: recursiveJobs.version,
329 jobs: recursiveJobs.jobs. map (({ value , ... job }) => job),
330 ownership: recursiveJobs.ownership,
331 conflicts: recursiveJobs.conflicts,
332 hash: recursiveJobs.hash,
333 },
334 }, outputDir, fingerprint);
335 if (decisionRequests. length > EXTRACTION_BUDGETS .maxAgentDecisions) {
336 throw new Error ( `Typed decision budget exceeded: ${ decisionRequests . length } > ${ EXTRACTION_BUDGETS . maxAgentDecisions }` );
337 }
338 const decisionRequestsArtifact = {
339 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
340 requests: decisionRequests,
341 };
342 const appliedDecisionsArtifact = {
343 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
344 decisions: appliedDecisions,
345 };
346 await writeJson (path. join (extractionDir, "decisions" , "requests.json" ), decisionRequestsArtifact);
347 await writeJson (path. join (extractionDir, "decisions" , "applied.json" ), appliedDecisionsArtifact);
348 observationRefs. push (
349 { ref: "decisions/requests.json" , hash: semanticHash (decisionRequestsArtifact) },
350 { ref: "decisions/applied.json" , hash: semanticHash (appliedDecisionsArtifact) },
351 );
352 const manifest = createExtractionManifest ({ captureId, pageResolution, pageCapture, specs, gaps, observationRefs });
353 await writeFrozenExtraction ({ extractionDir, pageResolution, pageCapture, specs, gaps, manifest, specPaths });
354 const buildPlan = createBuildPlan (manifest, specs);
355 await ensureDir (path. join (docs, "build" ));
356 await writeJson (path. join (docs, "build" , "build-plan.json" ), buildPlan);
357 const approvedRegistry = await loadApprovedRegistry ();
358 const componentSelection = {
359 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
360 manifestHash: manifest.manifestHash,
361 selections: specs. filter (( spec ) => spec.kind === "unit" ). map (( spec ) => ({
362 unitId: spec.id,
363 contract: spec.classification,
364 ... selectRegistryItem ({
365 kind: spec.classification.kind,
366 requiresClientRuntime: requiresClientRuntime (spec),
367 hardRequirements: [ "license" , "accessibility" , "responsive" , "ssr" , "rtl" , "reducedMotion" ],
368 capabilityRequirements: spec.classification.capabilityRequirements,
369 }, approvedRegistry),
370 })),
371 rejectedRegistryItems: approvedRegistry.rejected,
372 };
373 await writeJson (path. join (docs, "build" , "component-selection.json" ), componentSelection);
374 await ensureDir (path. join (docs, "qa" ));
375 await writeJson (path. join (docs, "qa" , "registry-opportunities.json" ), {
376 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
377 manifestHash: manifest.manifestHash,
378 opportunities: componentSelection.selections
379 . filter (( selection ) => selection.strategy === "bounded-custom" )
380 . map (( selection ) => ({
381 unitId: selection.unitId,
382 contractKind: selection.contract.kind,
383 variant: selection.contract.variant,
384 reason: "No human-approved local registry item met every hard requirement; bounded custom building remains available." ,
385 requiredCapabilities: [ "license" , "accessibility" , "responsive" , "ssr" , "rtl" , "reducedMotion" ],
386 capabilityRequirements: selection.contract.capabilityRequirements || {},
387 registryRejections: selection.rejected || [],
388 sourceNeutral: true ,
389 })),
390 });
391 await writeJson (path. join (docs, "extraction" , "latest.json" ), { captureId, manifestRef: `${ captureId }/extraction-manifest.json` , manifestHash: manifest.manifestHash });
392 await enforceArtifactBudgets (extractionDir, manifest);
393 return { captureId, extractionDir, pageResolution, pageCapture, specs, gaps, manifest, buildPlan };
394 }
395
396 export function createBuildPlan ( manifest , specs ) {
397 const byId = new Map (specs. map (( spec ) => [spec.id, spec]));
398 const ordered = [ ... specs]. sort (( a , b ) => buildRank (a) - buildRank (b) || (a.order || 0 ) - (b.order || 0 ) || a.id. localeCompare (b.id));
399 const plan = {
400 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
401 kind: "build-plan" ,
402 pageKey: "home" ,
403 route: "/" ,
404 manifestHash: manifest.manifestHash,
405 status: manifest.status,
406 units: ordered. map (( spec ) => ({
407 id: spec.id,
408 kind: spec.kind,
409 unitKind: spec.unitKind || spec.kind,
410 status: spec.status,
411 hash: spec.hash,
412 gapRefs: spec.gapRefs || [],
413 dependencies: dependencyIds (spec, byId),
414 })),
415 };
416 assertValidArtifact (plan, "build-plan" );
417 return plan;
418 }
419
420 function buildRank ( spec ) {
421 if (spec.kind === "foundation" ) return 0 ;
422 if (spec.kind === "metadata" ) return 1 ;
423 if (spec.kind === "shared-chrome" ) return 2 ;
424 if (spec.unitKind === "component" ) return 3 ;
425 return 4 ;
426 }
427
428 function dependencyIds ( spec , byId ) {
429 const ids = [spec.reconstruction?.foundationRef, spec.reconstruction?.sharedChromeRef, ... (spec.children || [])]. filter (Boolean);
430 return ids. filter (( id ) => byId. has (id));
431 }
432
433 async function writeObservations ( extractionDir , values , outputDir , sourceFingerprint ) {
434 const observationsDir = path. join (extractionDir, "observations" );
435 await ensureDir (observationsDir);
436 const page = structuredClone (values.page || {});
437 const screenshotDir = path. join (observationsDir, "screenshots" );
438 for ( const [ name , sourceRef ] of Object. entries (page.screenshots || {})) {
439 const sourcePath = path. isAbsolute (sourceRef) ? sourceRef : path. resolve (outputDir, sourceRef);
440 const filename = path. basename (sourceRef);
441 try {
442 await ensureDir (screenshotDir);
443 await copyFile (sourcePath, path. join (screenshotDir, filename));
444 page.screenshots[name] = `screenshots/${ filename }` ;
445 } catch {
446 page.screenshots[name] = `screenshots/${ filename }` ;
447 }
448 }
449 const mapping = {
450 "page.json" : page,
451 "assets.json" : values.assets,
452 "seo.json" : values.seo,
453 "tokens.json" : values.tokens,
454 "interaction-map.json" : values.interactionMap,
455 "scene-contract.json" : values.sceneContract,
456 "layout-blueprint.json" : values.layoutBlueprint,
457 "ui-normalization.json" : values.uiNormalization,
458 "control-state-contract.json" : values.controlStateContract,
459 "visual-assets.json" : values.visualAssets,
460 "recursive-jobs.json" : values.recursiveJobs,
461 };
462 const refs = [];
463 const packetDir = path. join (observationsDir, "packets" );
464 await ensureDir (packetDir);
465 const writePacket = async ( packetName , packet ) => {
466 assertValidArtifact (packet, "observation-packet" );
467 await writeJson (path. join (packetDir, packetName), packet);
468 refs. push ({ ref: `observations/packets/${ packetName }` , hash: semanticHash (packet) });
469 };
470 for ( const [ name , value ] of Object. entries (mapping)) {
471 const sanitized = sanitizeEvidence (value);
472 await writeJson (path. join (observationsDir, name), sanitized);
473 const packetName = name. replace ( / \. json $ / , ".packet.json" );
474 const packet = {
475 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
476 kind: "observation-packet" ,
477 packetType: name. replace ( / \. json $ / , "" ),
478 sourceFingerprint,
479 viewport: null ,
480 state: "settled" ,
481 payload: sanitized,
482 };
483 await writePacket (packetName, packet);
484 }
485 for ( const [ name , screenshotRef ] of Object. entries (page.screenshots || {})) {
486 const packetName = `page.viewport-${ pathSafe ( name ) }.packet.json` ;
487 await writePacket (packetName, {
488 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
489 kind: "observation-packet" ,
490 packetType: "page-viewport" ,
491 sourceFingerprint,
492 viewport: { name, screenshotRef },
493 state: "settled" ,
494 payload: sanitizeEvidence ({
495 responsiveTextGeometry: page.responsiveTextGeometry?.[name] || null ,
496 viewportEvidence: page.viewportEvidence?.[name] || null ,
497 }),
498 });
499 }
500 let statePacketIndex = 0 ;
501 for ( const scene of values.sceneContract?.scenes || []) {
502 for ( const interaction of scene.states || []) {
503 const observedStates = Array. isArray (interaction.states) && interaction.states. length ? interaction.states : [ null ];
504 for ( const observedState of observedStates) {
505 statePacketIndex += 1 ;
506 const stateName = typeof observedState === "string"
507 ? observedState
508 : observedState?.id || observedState?.name || observedState?.phase || interaction.kind || "observed" ;
509 await writePacket ( `behavior-state-${ String ( statePacketIndex ). padStart ( 4 , "0" ) }.packet.json` , {
510 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
511 kind: "observation-packet" ,
512 packetType: "behavior-state" ,
513 sourceFingerprint,
514 viewport: null ,
515 state: String (stateName),
516 payload: sanitizeEvidence ({
517 sceneId: scene.id,
518 sectionId: scene.sectionId,
519 interactionId: interaction.id,
520 interactionKind: interaction.kind,
521 observation: observedState,
522 }),
523 });
524 }
525 }
526 }
527 return refs;
528 }
529
530 export function sanitizeEvidence ( value , key = "" ) {
531 if (Array. isArray (value)) return value. map (( item ) => sanitizeEvidence (item, key));
532 if ( ! value || typeof value !== "object" ) {
533 if ( typeof value === "string" && /authorization | cookie | set-cookie | password | token | secret | formvalue/ i . test (key)) return "[redacted]" ;
534 if ( typeof value === "string" && /domsnapshots ? / i . test (key)) return redactDomSecrets (value);
535 if ( typeof value === "string" && /url $ / i . test (key)) return redactUrl (value);
536 return value;
537 }
538 const result = {};
539 for ( const [ childKey , childValue ] of Object. entries (value)) {
540 if ( / ^ (authorization | cookie | set-cookie | password | token | secret | formValue) $ / i . test (childKey)) {
541 result[childKey] = "[redacted]" ;
542 continue ;
543 }
544 result[childKey] = sanitizeEvidence (childValue, childKey);
545 }
546 return result;
547 }
548
549 function redactDomSecrets ( html ) {
550 return String (html)
551 . replace ( /(<input \b [ ^ >] *\b value \s * = \s * ["'] ) [ ^ "'] * ( ["'][ ^ >] * >)/ gi , "$1[redacted]$2" )
552 . replace ( /( ["'] ? (?:authorization | access [_-] ? token | refresh [_-] ? token | api [_-] ? key | client [_-] ? secret | password) ["'] ? \s * [:=]\s * ["'] ) [ ^ "'] + ( ["'] )/ gi , "$1[redacted]$2" )
553 . replace ( /(<meta \b [ ^ >] * (?:csrf | token | secret) [ ^ >] *\b content \s * = \s * ["'] ) [ ^ "'] * ( ["'][ ^ >] * >)/ gi , "$1[redacted]$2" );
554 }
555
556 function redactUrl ( value ) {
557 try {
558 const url = new URL (value);
559 for ( const key of [ ... url.searchParams. keys ()]) {
560 if ( /token | key | secret | auth | password | session | email | code/ i . test (key)) url.searchParams. set (key, "[redacted]" );
561 }
562 return url. toString ();
563 } catch {
564 return value;
565 }
566 }
567
568 function hasFoundationEvidence ( tokens ) {
569 return Boolean (tokens && typeof tokens === "object" && Object. keys (tokens). some (( key ) => {
570 const value = tokens[key];
571 return Array. isArray (value) ? value. length : value && typeof value === "object" ? Object. keys (value). length : Boolean (value);
572 }));
573 }
574
575 function collectFoundationPrimitives ( page ) {
576 const kinds = new Set ();
577 for ( const section of page?.sections || []) {
578 const classification = classifyObservedUnit (section, { unitKind: "section" , evidenceRef: "observations/page.json" });
579 if ([ "button-group" , "card" , "accordion" , "tabs" , "gallery" , "carousel" , "navigation" ]. includes (classification.kind)) kinds. add (classification.kind);
580 }
581 return [ ... kinds]. sort (). map (( kind ) => ({ kind, evidence: "observations/page.json" }));
582 }
583
584 function collectGlobalCssIntent ( tokens ) {
585 return {
586 direction: tokens?.direction || tokens?.typography?.direction || "unknown" ,
587 colorRoles: tokens?.colors || tokens?.colorRoles || {},
588 typography: tokens?.typography || {},
589 spacing: tokens?.spacing || {},
590 breakpoints: tokens?.breakpoints || {},
591 };
592 }
593
594 function selectChromeAssets ( visualAssets ) {
595 return sanitizeEvidence ({ logos: visualAssets?.logos || [], icons: visualAssets?.icons || [] });
596 }
597
598 function selectOwnedInteractions ( interactionMap , owner ) {
599 const interactions = interactionMap?.interactions || [];
600 if (owner === "shared-chrome" ) return interactions. filter (( item ) => /header | footer | nav | chrome/ i . test ( `${ item . owner || ""} ${ item . scope || ""} ${ item . target ?. selector || ""}` ));
601 return interactions. filter (( item ) => item.sectionId === owner || item.ownerUnitId === owner);
602 }
603
604 function resolveClassification ({ value , unitId , evidenceRef , patchesByRequest , decisionRequests , appliedDecisions }) {
605 const classification = classifyObservedUnit (value, { unitKind: unitId. includes ( ":component-" ) ? "component" : "section" , evidenceRef });
606 if (classification.confidence === "high" ) return { classification, gaps: [] };
607 if (classification.confidence === "low" ) {
608 return {
609 classification,
610 gaps: [ createGap ({
611 id: `gap:${ unitId }:classification` ,
612 ownerUnitId: unitId,
613 missingFields: [ "classification" ],
614 reason: "Classification confidence is below the calibrated decision band." ,
615 evidenceRefs: [evidenceRef],
616 confidence: "low" ,
617 affectedAcceptance: [ "classification-specific-behavior" , "visual-geometry" ],
618 assumptions: [ "Use the bounded custom builder and preserve observed content/order/layout evidence." ],
619 omissions: [ "Unverified classification-specific behavior." ],
620 })],
621 };
622 }
623 const choices = [ ...new Set ([classification.kind, value?.variant, "static-content" , "bounded-custom" ]. filter (Boolean))];
624 const request = createDecisionRequest ({
625 id: `decision:${ unitId }:classification` ,
626 decisionType: "classification" ,
627 allowedChoices: choices,
628 evidenceRefs: [evidenceRef],
629 detectorVersions: { sectionClassification: "0083.1" },
630 responseSchema: { type: "string" , enum: choices },
631 });
632 decisionRequests. push (request);
633 const patch = patchesByRequest. get (request.id);
634 if (patch) {
635 try {
636 const applied = applyDecisionPatch (request, patch);
637 appliedDecisions. push (applied);
638 return { classification: { ... classification, kind: applied.choice, confidence: applied.confidence }, gaps: [] };
639 } catch (error) {
640 return {
641 classification,
642 gaps: [ createGap ({
643 id: `gap:${ unitId }:classification` ,
644 ownerUnitId: unitId,
645 missingFields: [ "classification" ],
646 reason: `The typed classification patch was invalid: ${ error . message }` ,
647 evidenceRefs: [evidenceRef, `decisions/requests.json#/${ request . id }` ],
648 confidence: "medium" ,
649 affectedAcceptance: [ "classification-specific-behavior" , "visual-geometry" ],
650 assumptions: [ "Use the bounded custom builder until a valid typed decision is supplied." ],
651 omissions: [ "Unverified classification-specific behavior." ],
652 })],
653 };
654 }
655 }
656 return {
657 classification,
658 gaps: [ createGap ({
659 id: `gap:${ unitId }:classification` ,
660 ownerUnitId: unitId,
661 missingFields: [ "classification" ],
662 reason: "Classification requires a bounded typed orchestrating-agent decision." ,
663 evidenceRefs: [evidenceRef, `decisions/requests.json#/${ request . id }` ],
664 confidence: "medium" ,
665 affectedAcceptance: [ "classification-specific-behavior" , "visual-geometry" ],
666 assumptions: [ "Use the bounded custom builder if the decision is not supplied before assembly." ],
667 omissions: [ "Unverified classification-specific behavior." ],
668 })],
669 };
670 }
671
672 function gapsForSection ( section , sectionId , page , classification ) {
673 const gaps = [];
674 const warnings = [
675 ... (section?.diagnostics?.warnings || []),
676 ... (page?.sectionDiagnostics?.warnings || []). filter (( warning ) => String (warning). includes (sectionId)),
677 ];
678 if (warnings. length ) {
679 gaps. push ( createGap ({
680 id: `gap:${ sectionId }:diagnostics` ,
681 ownerUnitId: sectionId,
682 missingFields: [ "diagnostics" ],
683 reason: warnings. join ( " " ),
684 evidenceRefs: [ `observations/page.json#/${ sectionId }` ],
685 confidence: classification.confidence,
686 affectedAcceptance: [ "classification-specific-behavior" , "visual-geometry" ],
687 assumptions: [ "Use the bounded custom builder and preserve observed content/order/layout evidence." ],
688 omissions: [ "Unverified classification-specific behavior." ],
689 }));
690 }
691 if ( ! section?.layoutEvidence && ! section?.layout) {
692 gaps. push ( createGap ({
693 id: `gap:${ sectionId }:layout-evidence` ,
694 ownerUnitId: sectionId,
695 missingFields: [ "layout" , "styleables" ],
696 reason: "Viewport layout and styleable-category evidence was not captured for this visible section." ,
697 evidenceRefs: [ `observations/page.json#/${ sectionId }` ],
698 confidence: classification.confidence,
699 affectedAcceptance: [ "responsive-layout" , "visible-layers" , "visual-geometry" ],
700 assumptions: [ "Preserve observed content and order in a bounded fluid layout." ],
701 omissions: [ "Unverified authored CSS context and viewport-specific property unions." ],
702 }));
703 }
704 return gaps;
705 }
706
707 function contentFor ( value , options = {}) {
708 return sanitizeEvidence ( removeContaminatedContent ({
709 heading: value?.heading || "" ,
710 text: value?.text || "" ,
711 label: value?.label || "" ,
712 links: value?.links || [],
713 items: value?.items || [],
714 accessibleName: value?.accessibleName || value?.ariaLabel || value?.a11y?.accessibleName || "" ,
715 icon: value?.icon || null ,
716 }, options));
717 }
718
719 function contentContaminationGap ( unitId , evidenceRef , contamination ) {
720 const fields = contamination.findings. map (( finding ) => finding.path). filter (Boolean);
721 return createGap ({
722 id: `gap:${ unitId }:content-contamination` ,
723 ownerUnitId: unitId,
724 missingFields: fields. length ? fields : [ "content.text" ],
725 reason: `Content-bearing DOM evidence matched non-rendered source-code signals (${ contamination . reasons . join ( ", " ) }). The affected strings were omitted instead of being emitted as page copy.` ,
726 evidenceRefs: [evidenceRef],
727 affectedAcceptance: [ "visible-content" , "content-fidelity" ],
728 assumptions: [ "Reliable structure, layout, assets, and uncontaminated copy can still be implemented for this unit." ],
729 omissions: [ "Strings that appear to originate from executable script, stylesheet, template, or embedded source markup." ],
730 });
731 }
732
733 function capabilityRequirementsFor ( value , classification ) {
734 if (value?.capabilityRequirements && typeof value.capabilityRequirements === "object" ) {
735 return sanitizeEvidence (value.capabilityRequirements);
736 }
737 if (classification.kind !== "button" ) return {};
738 const label = value?.label || value?.heading || value?.text || "" ;
739 const href = (value?.links || []). find (( link ) => link?.href)?.href || value?.href || "" ;
740 const icon = value?.icon || (value?.images || value?.assets || []). find (( asset ) => /icon/ i . test ( `${ asset ?. role || ""} ${ asset ?. type || ""}` ));
741 const accessibleName = value?.accessibleName || value?.ariaLabel || value?.a11y?.accessibleName || "" ;
742 const availableSlots = [
743 label ? "label" : null ,
744 href ? "href" : null ,
745 icon ? "icon" : null ,
746 accessibleName ? "accessibleName" : null ,
747 ]. filter (Boolean);
748 const composition = href
749 ? "as-link"
750 : icon && ! label
751 ? "icon-only"
752 : icon && value?.iconPosition === "end"
753 ? "trailing-icon"
754 : icon
755 ? "leading-icon"
756 : "text" ;
757 const axes = {};
758 if (value?.componentVariant || value?.visualVariant) axes.variant = value.componentVariant || value.visualVariant;
759 if (value?.componentSize || value?.controlSize) axes.size = value.componentSize || value.controlSize;
760 return {
761 axes,
762 composition,
763 availableSlots,
764 requiredSlots: availableSlots,
765 requiredStates: [ "focus-visible" , ... (value?.disabled ? [ "disabled" ] : [])],
766 };
767 }
768
769 function requiresClientRuntime ( spec ) {
770 const behavior = JSON . stringify (spec.behavior || {}). toLowerCase ();
771 return /accordion | disclosure | tablist | tabpanel | carousel | menuitem | dropdown | toggle | clientstate | client-state | hydration/ . test (behavior);
772 }
773
774 function acceptanceFor ( value ) {
775 return {
776 required: [ "content-order" , "responsive-layout" , "visible-layers" , "keyboard-when-interactive" ],
777 sourceAssertions: sanitizeEvidence (value?.acceptance || value?.assertions || []),
778 };
779 }
780
781 function knownClassification ( section ) {
782 return /hero | navigation | menu | button | accordion | tabs | gallery | carousel | card | repeater | form | banner | footer | content | cta | promo | reviews/ i . test ( `${ section ?. kind || ""} ${ section ?. variant || ""}` );
783 }
784
785 function pathSafe ( value ) {
786 return value. replace ( / [ ^ a-z0-9._-] + / gi , "-" );
787 }
788
789 async function enforceArtifactBudgets ( extractionDir , manifest ) {
790 const total = await directorySize (extractionDir);
791 if (total > EXTRACTION_BUDGETS .maxRawEvidenceBytes) {
792 const error = new Error ( `Extraction evidence exceeds ${ EXTRACTION_BUDGETS . maxRawEvidenceBytes } bytes (${ total })` );
793 error.code = "EXTRACTION_EVIDENCE_BUDGET" ;
794 throw error;
795 }
796 const specsBytes = Buffer. byteLength ( JSON . stringify (manifest));
797 if (specsBytes > EXTRACTION_BUDGETS .maxFrozenSpecBytes) {
798 const error = new Error ( `Frozen manifest exceeds ${ EXTRACTION_BUDGETS . maxFrozenSpecBytes } bytes (${ specsBytes })` );
799 error.code = "EXTRACTION_SPEC_BUDGET" ;
800 throw error;
801 }
802 }
803
804 async function directorySize ( dir ) {
805 let total = 0 ;
806 for ( const entry of await readdir (dir, { withFileTypes: true })) {
807 const target = path. join (dir, entry.name);
808 total += entry. isDirectory () ? await directorySize (target) : ( await stat (target)).size;
809 }
810 return total;
811 }
812
813 async function main () {
814 const args = parseArgs ();
815 if ( ! args.out) throw new Error ( "--out is required" );
816 const outputDir = path. resolve ( String (args.out));
817 const docs = docsDir (outputDir);
818 const pagePath = args.page || path. join (docs, "pages" , "home-home.json" );
819 const readOptional = async ( name , fallback ) => {
820 try { return await readJson (path. join (docs, name)); } catch { return fallback; }
821 };
822 const page = await readJson (pagePath);
823 const decisionPatchArtifact = args[ "decision-patches" ] ? await readJson (path. resolve ( String (args[ "decision-patches" ]))) : [];
824 const result = await assembleExtraction ({
825 outputDir,
826 requestedUrl: args.url || page.url,
827 resolvedUrl: page.url,
828 canonicalUrl: page.seo?.canonical || page.url,
829 page,
830 assets: await readOptional ( "assets.json" , {}),
831 seo: await readOptional ( "seo.json" , page.seo || {}),
832 tokens: await readOptional ( "tokens.json" , {}),
833 interactionMap: await readOptional ( "interaction-map.json" , {}),
834 sceneContract: await readOptional ( "scene-contract.json" , {}),
835 layoutBlueprint: await readOptional ( "layout-blueprint.json" , {}),
836 uiNormalization: await readOptional ( "ui-normalization.json" , {}),
837 controlStateContract: await readOptional ( "control-state-contract.json" , {}),
838 visualAssets: await readOptional ( "visual-assets.json" , {}),
839 decisionPatches: Array. isArray (decisionPatchArtifact) ? decisionPatchArtifact : decisionPatchArtifact.patches || [],
840 });
841 console. log ( JSON . stringify ({ captureId: result.captureId, manifestHash: result.manifest.manifestHash, status: result.manifest.status }, null , 2 ));
842 }
843
844 if ( import . meta .url === `file://${ process . argv [ 1 ] }` ) main (). catch (( error ) => { console. error (error.stack || error.message); process. exit ( 1 ); });