Setting the file. One moment.
Extraction Contract · Wix Headless Replatform · wix/skills · Skills Docs
Back to the top of the page 28.10
Workflow
This file
Number 28.42
Position 42 of 89
Type JavaScript
Size 19 KB
Lines 370 scripts/lib/ extraction-contract.mjs
JavaScript · 370 lines · 19 KB
8
export
const
MAX_GAP_RECOVERY_ATTEMPTS
=
2
;
9 export const EXTRACTION_BUDGETS = Object. freeze ({
10 maxRawEvidenceBytes: 250 * 1024 * 1024 ,
11 maxFrozenSpecBytes: 10 * 1024 * 1024 ,
12 maxAgentDecisions: 30 ,
13 maxAgentInputTokens: 50_000 ,
14 maxExtractionMs: 10 * 60 * 1000 ,
15 });
16
17 const ARTIFACT_KINDS = new Set ([
18 "page-resolution" ,
19 "observation-packet" ,
20 "page-capture" ,
21 "foundation" ,
22 "metadata" ,
23 "shared-chrome" ,
24 "unit" ,
25 "extraction-gap" ,
26 "extraction-manifest" ,
27 "build-plan" ,
28 "final-report" ,
29 "decision-request" ,
30 "decision-patch" ,
31 ]);
32
33 const NON_SEMANTIC_KEYS = new Set ([
34 "createdAt" ,
35 "updatedAt" ,
36 "startedAt" ,
37 "completedAt" ,
38 "generatedAt" ,
39 "capturedAt" ,
40 "timestamp" ,
41 "logs" ,
42 "log" ,
43 ]);
44
45 const ALLOWED_FIELDS = Object. freeze ({
46 "page-resolution" : new Set ([ "schemaVersion" , "kind" , "id" , "pageKey" , "destinationRoute" , "status" , "source" , "evidence" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
47 "page-capture" : new Set ([ "schemaVersion" , "kind" , "id" , "pageKey" , "status" , "sourceFingerprint" , "viewportSet" , "sectionOrder" , "ignoredSurfaces" , "normalizationActions" , "observationRefs" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
48 foundation: new Set ([ "schemaVersion" , "kind" , "id" , "pageKey" , "status" , "tokens" , "fonts" , "primitives" , "globalCssIntent" , "evidenceRefs" , "acceptance" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
49 metadata: new Set ([ "schemaVersion" , "kind" , "id" , "pageKey" , "status" , "document" , "evidenceRefs" , "acceptance" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
50 "shared-chrome" : new Set ([ "schemaVersion" , "kind" , "id" , "pageKey" , "status" , "header" , "footer" , "controls" , "visualAssets" , "behavior" , "evidenceRefs" , "acceptance" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
51 unit: new Set ([ "schemaVersion" , "kind" , "id" , "unitKind" , "pageKey" , "parentUnitId" , "order" , "status" , "classification" , "scope" , "content" , "assets" , "layers" , "layout" , "styleables" , "behavior" , "children" , "reconstruction" , "acceptance" , "diagnostics" , "provenance" , "gapRefs" , "dependencyHashes" , "hash" , "extensions" ]),
52 "extraction-gap" : new Set ([ "schemaVersion" , "kind" , "id" , "ownerUnitId" , "scope" , "status" , "missingFields" , "reason" , "evidenceRefs" , "confidence" , "dependencyClosure" , "affectedAcceptance" , "attempts" , "remainingAttempts" , "assumptions" , "omissions" , "unblockAction" , "userDecision" , "extensions" ]),
53 "extraction-manifest" : new Set ([ "schemaVersion" , "kind" , "captureId" , "pageKey" , "pageResolution" , "pageCapture" , "observationRefs" , "specs" , "gaps" , "budgets" , "status" , "manifestHash" , "extensions" ]),
54 "build-plan" : new Set ([ "schemaVersion" , "kind" , "pageKey" , "route" , "manifestHash" , "status" , "units" , "extensions" ]),
55 "final-report" : new Set ([ "schemaVersion" , "kind" , "captureId" , "manifestHash" , "status" , "summary" , "gaps" , "qa" , "extensions" ]),
56 "decision-request" : new Set ([ "schemaVersion" , "kind" , "id" , "decisionType" , "allowedChoices" , "evidenceRefs" , "detectorVersions" , "responseSchema" , "cacheKey" , "extensions" ]),
57 "decision-patch" : new Set ([ "schemaVersion" , "kind" , "requestId" , "choice" , "confidence" , "reasonCodes" , "extensions" ]),
58 });
59
60 export function canonicalizeJson ( value ) {
61 if (value === null || typeof value === "boolean" || typeof value === "string" ) {
62 return JSON . stringify (value);
63 }
64 if ( typeof value === "number" ) {
65 if ( ! Number. isFinite (value)) throw new TypeError ( "Canonical JSON cannot contain non-finite numbers" );
66 return JSON . stringify (value);
67 }
68 if (Array. isArray (value)) return `[${ value . map ( canonicalizeJson ). join ( "," ) }]` ;
69 if (value && typeof value === "object" ) {
70 return `{${ Object . keys ( value ). sort (). map (( key ) => `${ JSON . stringify ( key ) }:${ canonicalizeJson ( value [ key ]) }` ). join ( "," ) }}` ;
71 }
72 throw new TypeError ( `Canonical JSON cannot contain ${ typeof value }` );
73 }
74
75 export function sha256 ( value ) {
76 const input = typeof value === "string" || Buffer. isBuffer (value) ? value : canonicalizeJson (value);
77 return createHash ( "sha256" ). update (input). digest ( "hex" );
78 }
79
80 export function semanticContent ( value , key = "" ) {
81 if (Array. isArray (value)) return value. map (( item ) => semanticContent (item));
82 if ( ! value || typeof value !== "object" ) {
83 if ( typeof value === "string" && (key === "absolutePath" || key. endsWith ( "AbsolutePath" )) && path. isAbsolute (value)) {
84 return path. basename (value);
85 }
86 return value;
87 }
88 const result = {};
89 for ( const childKey of Object. keys (value). sort ()) {
90 if ( NON_SEMANTIC_KEYS . has (childKey)) continue ;
91 if (childKey === "hash" || childKey === "manifestHash" || childKey === "cacheKey" ) continue ;
92 result[childKey] = semanticContent (value[childKey], childKey);
93 }
94 return result;
95 }
96
97 export function semanticHash ( artifact , dependencyHashes = []) {
98 return sha256 ({
99 artifact: semanticContent (artifact),
100 dependencyHashes: [ ... dependencyHashes]. sort (),
101 });
102 }
103
104 export function createDecisionRequest ({ id , decisionType , allowedChoices , evidenceRefs = [], detectorVersions = {}, responseSchema = {} }) {
105 if ( ! id || ! decisionType || ! allowedChoices?. length ) throw new Error ( "A decision request requires id, decisionType, and allowedChoices" );
106 const request = {
107 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
108 kind: "decision-request" ,
109 id,
110 decisionType,
111 allowedChoices,
112 evidenceRefs,
113 detectorVersions,
114 responseSchema,
115 cacheKey: "" ,
116 };
117 request.cacheKey = semanticHash (request);
118 assertValidArtifact (request, "decision-request" );
119 return request;
120 }
121
122 export function applyDecisionPatch ( request , patch ) {
123 assertValidArtifact (request, "decision-request" );
124 assertValidArtifact (patch, "decision-patch" );
125 if (patch.requestId !== request.id) throw new Error ( `Decision patch ${ patch . requestId } does not match ${ request . id }` );
126 if ( ! request.allowedChoices. some (( choice ) => canonicalizeJson (choice) === canonicalizeJson (patch.choice))) {
127 throw new Error ( "Decision patch choice is outside the allowed typed choices" );
128 }
129 return { requestId: request.id, cacheKey: request.cacheKey, choice: patch.choice, confidence: patch.confidence, reasonCodes: patch.reasonCodes };
130 }
131
132 export function createGap ({
133 id ,
134 ownerUnitId ,
135 scope = "local" ,
136 missingFields = [],
137 reason ,
138 evidenceRefs = [],
139 confidence = "low" ,
140 dependencyClosure = [],
141 affectedAcceptance = [],
142 assumptions = [],
143 omissions = [],
144 unblockAction = null ,
145 }) {
146 if ( ! id || ! ownerUnitId || ! reason) throw new Error ( "A gap requires id, ownerUnitId, and reason" );
147 if ( !new Set ([ "local" , "global" ]). has (scope)) throw new Error ( `Invalid gap scope: ${ scope }` );
148 return {
149 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
150 kind: "extraction-gap" ,
151 id,
152 ownerUnitId,
153 scope,
154 status: scope === "global" ? "global-blocker" : "recoverable" ,
155 missingFields,
156 reason,
157 evidenceRefs,
158 confidence,
159 dependencyClosure: [ ...new Set ([ownerUnitId, ... dependencyClosure])],
160 affectedAcceptance,
161 attempts: [],
162 remainingAttempts: scope === "global" ? 0 : MAX_GAP_RECOVERY_ATTEMPTS ,
163 assumptions,
164 omissions,
165 unblockAction,
166 userDecision: null ,
167 };
168 }
169
170 export function recordGapAttempt ( gap , { tactic , evidenceRefs = [], outcome = "failed" , producedNewEvidence = false , note = "" }) {
171 if (gap.scope === "global" ) return gap;
172 if ([ "provisional" , "resolved" , "needs-user-decision" ]. includes (gap.status) || gap.remainingAttempts === 0 ) return gap;
173 if ( ! tactic) throw new Error ( "A gap recovery attempt requires a tactic" );
174 const previousSameTactic = gap.attempts. some (( attempt ) => attempt.tactic === tactic && ! attempt.producedNewEvidence);
175 if (previousSameTactic && ! producedNewEvidence) return gap;
176 const countsAgainstBudget = outcome !== "resolved" ;
177 const attempts = [ ... gap.attempts, { tactic, evidenceRefs, outcome, producedNewEvidence, countsAgainstBudget, note }];
178 const consumed = attempts. filter (( attempt ) => attempt.countsAgainstBudget && attempt.outcome !== "resolved" ). length ;
179 const resolved = outcome === "resolved" ;
180 const remainingAttempts = Math. max ( 0 , MAX_GAP_RECOVERY_ATTEMPTS - consumed);
181 return {
182 ... gap,
183 attempts,
184 remainingAttempts,
185 status: resolved ? "resolved" : remainingAttempts === 0 ? "provisional" : "recoverable" ,
186 };
187 }
188
189 export function validateArtifact ( artifact , expectedKind = artifact?.kind) {
190 const errors = [];
191 if ( ! artifact || typeof artifact !== "object" || Array. isArray (artifact)) errors. push ( "artifact must be an object" );
192 if ( ! ARTIFACT_KINDS . has (expectedKind)) errors. push ( `unsupported artifact kind: ${ expectedKind }` );
193 if (artifact?.kind !== expectedKind) errors. push ( `kind must be ${ expectedKind }` );
194 if (artifact?.schemaVersion !== EXTRACTION_SCHEMA_VERSION ) errors. push ( `schemaVersion must be ${ EXTRACTION_SCHEMA_VERSION }` );
195 if ( ARTIFACT_KINDS . has (expectedKind)) {
196 const schemaInput = [ "page-resolution" , "page-capture" , "foundation" , "metadata" , "shared-chrome" , "unit" ]. includes (expectedKind) && ! artifact?.hash
197 ? { ... artifact, hash: "0" . repeat ( 64 ) }
198 : artifact;
199 const generated = validateGeneratedArtifact (schemaInput, expectedKind);
200 errors. push ( ... generated.errors. map (( error ) => `schema ${ error }` ));
201 }
202 const allowed = ALLOWED_FIELDS [expectedKind];
203 if (allowed && artifact && typeof artifact === "object" ) {
204 for ( const key of Object. keys (artifact)) if ( ! allowed. has (key)) errors. push ( `unknown field: ${ key }` );
205 }
206 if (expectedKind === "page-resolution" ) {
207 if (artifact.pageKey !== "home" ) errors. push ( "pageKey must be home" );
208 if ( ! artifact.source?.requestedUrl || ! artifact.source?.resolvedUrl) errors. push ( "requestedUrl and resolvedUrl are required" );
209 if (artifact.destinationRoute !== "/" ) errors. push ( "destinationRoute must be /" );
210 }
211 if ([ "foundation" , "metadata" , "shared-chrome" , "unit" ]. includes (expectedKind)) {
212 if ( ! artifact.id) errors. push ( "id is required" );
213 if ( ! [ "accepted" , "provisional" ]. includes (artifact.status)) errors. push ( "status must be accepted or provisional" );
214 if ( ! Array. isArray (artifact.gapRefs)) errors. push ( "gapRefs must be an array" );
215 if (artifact.status === "provisional" && artifact.gapRefs?. length === 0 ) errors. push ( "provisional artifacts require gapRefs" );
216 }
217 if (expectedKind === "extraction-gap" ) {
218 if ( ! artifact.id || ! artifact.ownerUnitId || ! artifact.reason) errors. push ( "gap id, ownerUnitId, and reason are required" );
219 if ( ! [ "local" , "global" ]. includes (artifact.scope)) errors. push ( "gap scope must be local or global" );
220 if ( ! Array. isArray (artifact.attempts)) errors. push ( "gap attempts must be an array" );
221 }
222 if (expectedKind === "extraction-manifest" ) {
223 if ( ! artifact.captureId || ! Array. isArray (artifact.specs) || ! Array. isArray (artifact.gaps)) errors. push ( "manifest captureId, specs, and gaps are required" );
224 if (artifact.gaps?. some (( gap ) => gap.scope === "global" && gap.status !== "resolved" )) errors. push ( "manifest cannot freeze with an unresolved global blocker" );
225 }
226 if (expectedKind === "decision-request" ) {
227 if ( ! artifact.id || ! artifact.decisionType || ! artifact.allowedChoices?. length || ! artifact.cacheKey) errors. push ( "decision request is incomplete" );
228 }
229 if (expectedKind === "decision-patch" ) {
230 if ( ! artifact.requestId || ! artifact.reasonCodes?. length || ! [ "high" , "medium" , "low" ]. includes (artifact.confidence)) errors. push ( "decision patch is incomplete" );
231 }
232 return { ok: errors. length === 0 , errors };
233 }
234
235 export function assertValidArtifact ( artifact , expectedKind = artifact?.kind) {
236 const result = validateArtifact (artifact, expectedKind);
237 if ( ! result.ok) throw new Error ( `${ expectedKind } validation failed: ${ result . errors . join ( "; " ) }` );
238 return artifact;
239 }
240
241 export function freezeSpec ( spec , { dependencyHashes = [], gapIds = spec.gapRefs || [] } = {}) {
242 assertValidArtifact (spec, spec.kind);
243 const frozen = {
244 ... spec,
245 gapRefs: [ ...new Set (gapIds)]. sort (),
246 dependencyHashes: [ ... dependencyHashes]. sort (),
247 };
248 frozen.hash = semanticHash (frozen, frozen.dependencyHashes);
249 return Object. freeze (frozen);
250 }
251
252 export function createExtractionManifest ({ captureId , pageResolution , pageCapture , specs , gaps = [], observationRefs = [], budgets = {} }) {
253 const unresolvedGlobal = gaps. filter (( gap ) => gap.scope === "global" && gap.status !== "resolved" );
254 if (unresolvedGlobal. length ) {
255 const error = new Error ( `Cannot freeze extraction manifest: ${ unresolvedGlobal . map (( gap ) => gap . id ). join ( ", " ) }` );
256 error.code = "EXTRACTION_GLOBAL_BLOCKER" ;
257 error.gaps = unresolvedGlobal;
258 throw error;
259 }
260 const specEntries = [ ... specs]
261 . map (( spec ) => ({ id: spec.id, kind: spec.kind, status: spec.status, hash: spec.hash, gapRefs: spec.gapRefs || [] }))
262 . sort (( a , b ) => a.id. localeCompare (b.id));
263 const manifest = {
264 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
265 kind: "extraction-manifest" ,
266 captureId,
267 pageKey: "home" ,
268 pageResolution: { id: pageResolution.id, hash: pageResolution.hash },
269 pageCapture: { id: pageCapture.id, hash: pageCapture.hash },
270 observationRefs: [ ... observationRefs]. sort (( a , b ) => a.ref. localeCompare (b.ref)),
271 specs: specEntries,
272 gaps: gaps. map (( gap ) => ({ id: gap.id, ownerUnitId: gap.ownerUnitId, scope: gap.scope, status: gap.status })). sort (( a , b ) => a.id. localeCompare (b.id)),
273 budgets: { ... EXTRACTION_BUDGETS , ... budgets },
274 status: pageCapture.status === "provisional" || specEntries. some (( spec ) => spec.status === "provisional" ) ? "done_with_gaps" : "accepted" ,
275 };
276 manifest.manifestHash = semanticHash (manifest, specEntries. map (( entry ) => entry.hash));
277 assertValidArtifact (manifest, "extraction-manifest" );
278 return Object. freeze (manifest);
279 }
280
281 export async function verifyFrozenManifest ( extractionDir ) {
282 const manifest = JSON . parse ( await readFile (path. join (extractionDir, "extraction-manifest.json" ), "utf8" ));
283 assertValidArtifact (manifest, "extraction-manifest" );
284 const failures = [];
285 const pageResolution = JSON . parse ( await readFile (path. join (extractionDir, "page-resolution.spec.json" ), "utf8" ));
286 const pageCapture = JSON . parse ( await readFile (path. join (extractionDir, "page-capture.spec.json" ), "utf8" ));
287 const gapArtifact = JSON . parse ( await readFile (path. join (extractionDir, "extraction-gaps.json" ), "utf8" ));
288 const gaps = gapArtifact.gaps || [];
289 const gapIds = new Set ();
290 for ( const gap of gaps) {
291 try { assertValidArtifact (gap, "extraction-gap" ); }
292 catch (error) { failures. push ( `${ gap . id || "unknown gap"}: ${ error . message }` ); }
293 if (gapIds. has (gap.id)) failures. push ( `duplicate gap id: ${ gap . id }` );
294 gapIds. add (gap.id);
295 }
296 const manifestGapProjection = gaps
297 . map (( gap ) => ({ id: gap.id, ownerUnitId: gap.ownerUnitId, scope: gap.scope, status: gap.status }))
298 . sort (( a , b ) => a.id. localeCompare (b.id));
299 if ( canonicalizeJson (manifestGapProjection) !== canonicalizeJson (manifest.gaps)) failures. push ( "manifest gap projection does not match extraction-gaps.json" );
300 const resolutionHash = semanticHash (pageResolution, pageResolution.dependencyHashes || []);
301 const captureHash = semanticHash (pageCapture, pageCapture.dependencyHashes || []);
302 if (resolutionHash !== manifest.pageResolution.hash) failures. push ( `page resolution: expected ${ manifest . pageResolution . hash }, got ${ resolutionHash }` );
303 if (captureHash !== manifest.pageCapture.hash) failures. push ( `page capture: expected ${ manifest . pageCapture . hash }, got ${ captureHash }` );
304 for ( const entry of manifest.observationRefs) {
305 const absolute = path. resolve (extractionDir, entry.ref);
306 const extractionRoot = `${ path . resolve ( extractionDir ) }${ path . sep }` ;
307 if ( ! absolute. startsWith (extractionRoot)) {
308 failures. push ( `observation ${ entry . ref }: path escapes extraction directory` );
309 continue ;
310 }
311 try {
312 const observation = JSON . parse ( await readFile (absolute, "utf8" ));
313 const actual = semanticHash (observation);
314 if (actual !== entry.hash) failures. push ( `observation ${ entry . ref }: expected ${ entry . hash }, got ${ actual }` );
315 } catch (error) {
316 failures. push ( `observation ${ entry . ref }: ${ error . code === "ENOENT" ? "file missing" : error . message }` );
317 }
318 }
319 const knownHashes = new Set ([resolutionHash, captureHash, ... manifest.specs. map (( entry ) => entry.hash)]);
320 for ( const gapRef of [ ... (pageResolution.gapRefs || []), ... (pageCapture.gapRefs || [])]) {
321 if ( ! gapIds. has (gapRef)) failures. push ( `page artifact: unknown gap ref ${ gapRef }` );
322 }
323 for ( const entry of manifest.specs) {
324 const file = await findSpecFile (extractionDir, entry.id);
325 if ( ! file) {
326 failures. push ( `${ entry . id }: file missing` );
327 continue ;
328 }
329 const spec = JSON . parse ( await readFile (file, "utf8" ));
330 const actual = semanticHash (spec, spec.dependencyHashes || []);
331 if (actual !== entry.hash) failures. push ( `${ entry . id }: expected ${ entry . hash }, got ${ actual }` );
332 if ( canonicalizeJson ([ ... (spec.gapRefs || [])]. sort ()) !== canonicalizeJson ([ ... (entry.gapRefs || [])]. sort ())) failures. push ( `${ entry . id }: manifest gap refs do not match spec` );
333 for ( const gapRef of spec.gapRefs || []) if ( ! gapIds. has (gapRef)) failures. push ( `${ entry . id }: unknown gap ref ${ gapRef }` );
334 if (spec.status === "provisional" && ! (spec.gapRefs || []). length ) failures. push ( `${ entry . id }: provisional spec is not gap-backed` );
335 for ( const dependencyHash of spec.dependencyHashes || []) {
336 if ( ! knownHashes. has (dependencyHash)) failures. push ( `${ entry . id }: unknown dependency hash ${ dependencyHash }` );
337 }
338 }
339 const manifestHash = semanticHash (manifest, manifest.specs. map (( entry ) => entry.hash));
340 if (manifestHash !== manifest.manifestHash) failures. push ( `manifest: expected ${ manifest . manifestHash }, got ${ manifestHash }` );
341 return { ok: failures. length === 0 , failures, manifest };
342 }
343
344 async function findSpecFile ( extractionDir , id ) {
345 const indexPath = path. join (extractionDir, "spec-index.json" );
346 try {
347 const index = JSON . parse ( await readFile (indexPath, "utf8" ));
348 return index[id] ? path. join (extractionDir, index[id]) : null ;
349 } catch {
350 return null ;
351 }
352 }
353
354 export async function writeFrozenExtraction ({ extractionDir , pageResolution , pageCapture , specs , gaps , manifest , specPaths }) {
355 await ensureDir (extractionDir);
356 await writeJson (path. join (extractionDir, "page-resolution.spec.json" ), pageResolution);
357 await writeJson (path. join (extractionDir, "page-capture.spec.json" ), pageCapture);
358 for ( const gap of gaps) assertValidArtifact (gap, "extraction-gap" );
359 await writeJson (path. join (extractionDir, "extraction-gaps.json" ), {
360 schemaVersion: EXTRACTION_SCHEMA_VERSION ,
361 gaps,
362 });
363 const index = {};
364 for ( const { spec , relativePath } of specPaths) {
365 await writeJson (path. join (extractionDir, relativePath), spec);
366 index[spec.id] = relativePath;
367 }
368 await writeJson (path. join (extractionDir, "spec-index.json" ), index);
369 await writeJson (path. join (extractionDir, "extraction-manifest.json" ), manifest);
370 }