Setting the file. One moment.
Dependency Readiness · Wix Replatform · wix/skills · Skills Docs
Back to the top of the page — line 74
This file
Number 45.62
Position 62 of 182
Type JavaScript
Size 111 KB
Lines 1,219 lib/ dependency-readiness.js
JavaScript · 1,219 lines · 111 KB
(
'./execution-capability'
);
8 const local = require ( './local-state' );
9 const setup = require ( './setup-verification' );
10 const { SETUP_CHECK_METHODS } = require ( './setup-check-methods.js' );
11 const census = require ( './census-reconciliation' );
12 const recovery = require ( './dependency-recovery' );
13 const proofs = require ( './dependency-proofs' );
14 const assessments = require ( './operation-assessments' );
15 const { readEntityMappings } = require ( './website-handoff' );
16 const { requiredAppsFor } = require ( './app-requirements' );
17 const evaluations = new WeakMap ();
18 const evaluationGenerations = new WeakMap ();
19 const grants = new WeakMap ();
20 const indexes = new Map ();
21 const subscriptions = new Map ();
22 const PASS_FILE = 'state/readiness/task-passes.ndjson' ;
23 const SELECTIVE = new Set ([ 'full' , 'entity' , 'source-type' , 'missing-only' , 'failed-only' , 'deferred-only' , 'unresolved-only' ]);
24 // A finished producer releases its dependents; only records naming a parent that cannot land stop.
25 const SETTLED = new Set ([ 'complete' , 'complete-with-holds' , 'finished-partial' , 'empty' , 'setup-passed' , 'excluded-approved' , 'proven' , 'inherited' ]);
26 const PARENT_NOT_FINAL = new Set ([ 'failed_retryable' , 'started' , 'deferred' ]);
27 function linkChoice ( entityRefs , ref ) {
28 return entityRefs. every ( entityRef => domain (entityRef).dependencyFailureActions?.[ref] === 'offer-omit' ) ? 'offer-omit' : 'fix-only' ;
29 }
30 function linkRequired ( entityRefs , ref ) {
31 return entityRefs. some ( entityRef => ( domain (entityRef).dependencyLinkRequired || []). includes (ref));
32 }
33 const refPattern = / ^ [a-z0-9-] + \/ [a-z0-9-] +$ / ;
34 const hash = bytes => crypto. createHash ( 'sha256' ). update (bytes). digest ( 'hex' );
35 const equal = ( a , b ) => JSON . stringify (a) === JSON . stringify (b);
36 function evidenceReason ( error ) { return / ^ (?:dependency_ | readiness_ | mapping_plan_ | entity_ref_ | task_depends_on_ | site_id_ | plan_digest_) [a-zA-Z0-9_:\-/;,] +$ / . test (error.message || '' ) ? error.message : error.code === 'ENOENT' || error.code === 'JOURNAL_MISSING' ? 'dependency-evidence-missing' : 'dependency-evidence-invalid' ; }
37 function read ( projectDir , file , fallback ) {
38 try { return JSON . parse (fs. readFileSync ( inside (projectDir, file), 'utf8' )); }
39 catch (error) { if (error.code === 'ENOENT' && arguments . length === 3 ) return fallback; throw error; }
40 }
41 // Read-only inputs parsed once per on-disk identity (inode, size, mtime, ctime): a rewritten
42 // file re-parses, so the digests below still see every change.
43 function readCached ( projectDir , file , fallback ) { return require ( './cached-json' ). readJson ( inside (projectDir, file), fallback); }
44 function inside ( projectDir , file ) {
45 const root = path. resolve (projectDir), absolute = path. resolve (root, file);
46 if ( ! absolute. startsWith (root + path.sep)) throw new Error ( 'DEPENDENCY_PATH_OUTSIDE_PROJECT' );
47 return absolute;
48 }
49 const digestCache = new Map (), snapshotCache = new Map ();
50 function fileDigest ( projectDir , file ) { const absolute = inside (projectDir, file), stamp = statKey (absolute), prior = digestCache. get (absolute); if (prior?.stamp === stamp) return prior.digest; const digest = hash (fs. readFileSync (absolute)); digestCache. set (absolute, { stamp, digest }); return digest; }
51 function domain ( ref ) {
52 const result = proofs. domain (ref), plan = assessments. current ()?.plan;
53 if ( ! plan?.operationAssessments?. length ) return result;
54 const operations = [ ...new Set ( readEntityMappings (plan). filter ( r => r.targetRef === ref). map (proofs.resolveRowOperation). map ( r => r.operation). filter (Boolean))];
55 if (operations. length !== 1 ) return result;
56 const supplement = proofs. operationContract (ref,operations[ 0 ])?.contract.assessmentReferenceFields;
57 if ( ! supplement) return result;
58 result.dependsOn = proofs. applicableDependsOn (ref,operations[ 0 ]);
59 result.dependencyReferenceFields ||= {};
60 for ( const [ dependency , paths ] of Object. entries (supplement)) result.dependencyReferenceFields[dependency] = [ ...new Set ([ ... (result.dependencyReferenceFields[dependency] || []), ... paths])];
61 return result;
62 }
63 function edgeMode ( task , ref , producer ) {
64 if ([ 'setup-step' , 'pending-setup' ]. includes (producer?.kind)) return 'task-complete' ;
65 const modes = task.entityRefs. map ( entity => domain (entity).dependencyModes?.[ref] || 'task-complete' );
66 return modes. every ( mode => mode === 'per-record' ) ? 'per-record' : 'task-complete' ;
67 }
68 // Target knowledge's `legacyTaskCompleteEdges`: refs that became per-record after manifests had
69 // shipped them as task-complete. Such a manifest keeps its edge (from an import task, a disposition
70 // or a proof): it waits for at least as much, so the import it already ran is never refused. Any
71 // other stale mode is refused and regenerated. Secondary and child identities resolve
72 // only per record, so those edges always match exactly.
73 const PER_RECORD_ONLY = new Set ([ 'secondary-output' , 'child-output' ]);
74 function keptTaskEdge ( task , ref , emitted , derived , producer ) {
75 return emitted === 'task-complete' && derived === 'per-record' && ! PER_RECORD_ONLY . has (producer?.kind) && task.entityRefs. every ( entity => ( domain (entity).legacyTaskCompleteEdges || []). includes (ref));
76 }
77 function edgeModeAccepted ( task , ref , emitted , derived , producer ) { return emitted === derived || keptTaskEdge (task, ref, emitted, derived, producer); }
78 // The reference paths of such a kept task-complete edge were declared after its manifest was
79 // emitted, so its payloads are checked as they were then: not at those paths.
80 function checkedRefs ( task ) {
81 return (task.dependencyRefs || []). filter ( ref => { const edge = (task.dependencyEdges || []). find ( row => row.ref === ref); return ! (edge && keptTaskEdge (task, ref, edge.mode, edgeMode (task, ref, edge.producer), edge.producer)); });
82 }
83 // Producer-independent: a stale emitted mode fails before producers or dispositions load.
84 // Run with the project's mapping plan in scope: plan assessments can set dependencyModes.
85 function dependencyModeErrors ( projectDir , manifest ) {
86 const errors = [];
87 for ( const task of (manifest.tasks || []). filter ( row => row.kind === 'import' )) for ( const edge of task.dependencyEdges || []) {
88 try { if ( ! edgeModeAccepted (task, edge.ref, edge.mode, edge.producer?.kind === 'inherited' ? 'task-complete' : edgeMode (task, edge.ref, edge.producer), edge.producer)) errors. push ( `dependency_edge_mode_mismatch:${ task . id }:${ edge . ref }` ); } catch (e) { errors. push (e.message); }
89 }
90 return errors;
91 }
92 // Every edge has exactly one producer: an import task, all bound setup steps, an approved census
93 // disposition, a validated noProducer proof for the exact dependent operation, a
94 // member task's verified secondary identities, a product task's verified child identities (its
95 // variants), or -- for a transitive ref, or a child ref with no child producer -- the dependency that
96 // carries it (`inherited`). The graph is resolved by the shared pure resolver the
97 // mapper preflight also uses; this adapter supplies the manifest's real tasks and setup steps.
98 function bindDependencyProducers ({ manifest , setupPlan , dispositions = {}, proofs : validProofs = new Map (), scopeRefs = [], adoptOnlyRefs = [], adoptOnlyMixed = [] }) {
99 const result = {}, errors = [], available = new Set (scopeRefs);
100 for ( const task of manifest.tasks || []) for ( const ref of task.entityRefs || []) { available. add (ref); try { domain (ref); } catch (e) { errors. push (e.message); } }
101 const imports = (manifest.tasks || []). filter ( task => task.kind === 'import' );
102 for ( const task of imports) if (task.writeOperations !== undefined && ( ! task.writeOperations || typeof task.writeOperations !== 'object' || Object. entries (task.writeOperations). some (([ ref , op ]) => ! (task.entityRefs || []). includes (ref) || typeof op !== 'string' || ! op))) errors. push ( `task_write_operations_invalid:${ task . id }` );
103 const graph = resolveManifestGraph ({ manifest, setupPlan, dispositions, proofs: validProofs, adoptOnlyRefs, adoptOnlyMixed });
104 errors. push ( ... graph.errors. filter ( error => ! error. startsWith ( 'entity_ref_unknown:' )));
105 for ( const task of imports) {
106 const resolved = graph.tasks[task.id] || { refs: [], edges: [], dependsOn: [] };
107 if ( ! equal ([ ... (task.dependencyRefs || [])]. sort (), resolved.refs)) errors. push ( `dependency_refs_mismatch:${ task . id }` );
108 for ( const { ref , mode , producer } of resolved.edges) {
109 result[ref] = producer;
110 const emitted = (task.dependencyEdges || []). find ( edge => edge.ref === ref);
111 if ( ! emitted) errors. push ( `dependency_edge_missing:${ task . id }:${ ref }` );
112 else if ( ! edgeModeAccepted (task, ref, emitted.mode, mode, producer)) errors. push ( `dependency_edge_mode_mismatch:${ task . id }:${ ref }` );
113 else if ( ! equal (emitted.producer, producer)) errors. push ( `dependency_edge_producer_mismatch:${ task . id }:${ ref }` );
114 }
115 if ((task.dependencyEdges || []). length !== resolved.refs. length ) errors. push ( `dependency_edge_coverage:${ task . id }` );
116 if ( ! equal ([ ... (task.dependsOn || [])]. sort (), resolved.dependsOn)) errors. push ( `task_depends_on_mismatch:${ task . id }` );
117 }
118 return { ok: errors. length === 0 , errors: [ ...new Set (errors)], producers: result };
119 }
120 function resolveManifestGraph ({ manifest , setupPlan , dispositions = {}, proofs : validProofs = new Map (), adoptOnlyRefs = [], adoptOnlyMixed = [] }) {
121 const dependents = (manifest.tasks || []). filter ( task => task.kind === 'import' ). map ( task => ({ id: task.id, entityRefs: task.entityRefs || [], operations: task.writeOperations || {}, ... (task.adoptOnly === true ? { adoptOnly: true } : {}) }));
122 // An entity an import task writes is owned by that import: a setup step may not also claim it.
123 const importRefs = new Set (dependents. flatMap ( task => task.entityRefs));
124 const claims = (setupPlan.steps || []). flatMap ( step => (step.entityRefs || []). filter ( ref => importRefs. has (ref)). map ( ref => `setup_step_claims_import_entity:${ step . id }:${ ref }` ));
125 const setupFor = ref => { if (importRefs. has (ref)) return null ; const stepIds = (setupPlan.steps || []). filter ( step => (step.entityRefs || []). includes (ref)). map ( step => step.id). sort (); return stepIds. length ? { kind: 'setup-step' , stepIds } : null ; };
126 const graph = proofs. resolveDependencyGraph ({ dependents, imports: dependents, setupFor, dispositions, proofs: validProofs, adoptOnlyRefs, adoptOnlyMixed });
127 return claims. length ? { ... graph, ok: false , errors: [ ... claims, ... graph.errors] } : graph;
128 }
129 // For codegen: the dependencyRefs/dependencyEdges/dependsOn each import task must emit, from the
130 // same resolver the manifest validator uses, so a generated manifest never hand-derives a producer.
131 async function deriveDependencyEdges ( projectDir , manifest ) {
132 const adapter = await loadDependencyDispositions (projectDir);
133 const graph = resolveManifestGraph ({ manifest, setupPlan: read (projectDir, 'setup/setup-plan.json' ), dispositions: adapter.dispositions, proofs: adapter.proofs, adoptOnlyRefs: adapter.adoptOnlyRefs, adoptOnlyMixed: adapter.adoptOnlyMixed });
134 return { ok: graph.ok, errors: graph.errors, tasks: Object. fromEntries (Object. entries (graph.tasks). map (([ id , task ]) => [id, { dependencyRefs: task.refs, dependencyEdges: task.edges, dependsOn: task.dependsOn }])) };
135 }
136 function validateAssociations ({ plan , reconciliation }) {
137 const errors = [], bindings = new Map (), rows = new Map ((reconciliation?.rows || []). map ( row => [row.censusId, row]));
138 for ( const binding of plan?.dependencyBindings || []) {
139 if ( ! refPattern. test (binding.ref) || bindings. has (binding.ref) || ! Array. isArray (binding.censusIds) || ! binding.censusIds. length || new Set (binding.censusIds).size !== binding.censusIds. length ) {
140 errors. push ( `dependency_binding_invalid:${ binding . ref }` ); continue ;
141 }
142 try { domain (binding.ref); } catch (e) { errors. push (e.message); }
143 for ( const id of binding.censusIds) if ( ! rows. has (id) || ! rows. get (id).decisionRow) errors. push ( `dependency_binding_census_unknown:${ id }` );
144 bindings. set (binding.ref, binding.censusIds);
145 }
146 for ( const row of readEntityMappings (plan)) {
147 const { ids } = census. planCensusIds (row, [ ... rows. values ()]);
148 if ( ! row.targetRef || ! refPattern. test (row.targetRef)) { errors. push ( 'dependency_target_ref_missing' ); continue ; }
149 if ( ! bindings. has (row.targetRef) || ids. some ( id => ! bindings. get (row.targetRef). includes (id))) errors. push ( `dependency_binding_incomplete:${ row . targetRef }` );
150 }
151 const rules = [];
152 for ( const row of rows. values ()) for ( const rule of row.dependencyFieldRules || []) {
153 const key = `${ row . censusId }:${ rule . dependentRef }:${ rule . dependencyRef }:${ rule . path }` ;
154 const valid = [ 'omit' , 'defer' ]. includes (rule.action) && typeof rule.path === 'string' && / ^ \w + (?: \[\] ) ? (?: \. \w + (?: \[\] ) ? ) *$ / . test (rule.path);
155 let known = false ;
156 try { const d = domain (rule.dependentRef); known = (d.dependsOn || []). includes (rule.dependencyRef) && (d.dependencyReferenceFields?.[rule.dependencyRef] || []). includes (rule.path); } catch {}
157 if ( ! valid || ! known || ! bindings. get (rule.dependencyRef)?. includes (row.censusId) || rules. some ( prior => prior.key. startsWith ( `${ row . censusId }:${ rule . dependentRef }:${ rule . dependencyRef }:` ) && pathsOverlap (prior.path, rule.path))) errors. push ( `dependency_field_rule_invalid:${ key }` );
158 rules. push ({ ... rule, censusId: row.censusId, key });
159 }
160 return { ok: errors. length === 0 , errors, bindings, rules };
161 }
162 function pathsOverlap ( a , b ) { return a === b || a. startsWith (b + '.' ) || b. startsWith (a + '.' ); }
163 function pathMatches ( template , concrete ) {
164 const escaped = template. replace ( / [.*+?^${}()|[ \]\\ ] / g , ' \\ $&' ). replace ( / \\\[\\\] / g , ' \\ [ \\ d+ \\ ]' );
165 return new RegExp ( '^' + escaped + '$' ). test (concrete);
166 }
167 async function loadDependencyDispositions ( projectDir , { stage = 'codegen' } = {}) {
168 const plan = readRequired (projectDir, 'mapping/mapping-plan.json' );
169 const reconciliation = await census. readReconciliation (projectDir);
170 if ( ! reconciliation || reconciliation.reviewOnly || reconciliation.errors?. length || ! ( await census. censusReconciliationFreshness (projectDir, reconciliation)).ok) throw new Error ( 'dependency_reconciliation_invalid' );
171 const validated = validateAssociations ({ plan, reconciliation });
172 if ( ! validated.ok) throw new Error (validated.errors. join ( ';' ));
173 const approved = await census. ownerApprovedCensusIds (projectDir), rows = new Map (reconciliation.rows. map ( row => [row.censusId, row]));
174 const reconciliationSha256 = fileDigest (projectDir, 'mapping/review/census-reconciliation.json' );
175 const dispositions = {};
176 for ( const [ ref , ids ] of validated.bindings) if (ids. every ( id => rows. get (id).state === 'excluded' )) dispositions[ref] = {
177 censusIds: ids, reconciliationSha256, approved: ids. every ( id => approved. has (id)),
178 rules: validated.rules. filter ( rule => ids. includes (rule.censusId) && approved. has (rule.censusId)),
179 };
180 const proven = proofs. validateProofs (plan.dependencyProofs, { rows, plan, readEntityMappings, projectDir, stage });
181 if (proven.errors. length ) throw new Error (proven.errors. join ( ';' ));
182 const adoption = proofs. adoptOnlyRefsOf (plan, readEntityMappings);
183 return { dispositions, proofs: proven.proofs, ... validated, rows, approved, reconciliationSha256, adoptOnlyRefs: adoption.refs, adoptOnlyMixed: adoption.mixed };
184 }
185 // A plugin entity with a current read plan is read in full before the mapping checkpoint, so its
186 // row cannot stay pending for want of data. Pending stays legitimate only when a full read was
187 // attempted and failed: its typed extraction task carries the reason and operator action.
188 // Rows are matched by any identity they carry (a short entityKey/sourceEntity name may sit beside
189 // the canonical plugin reference), so every mapping-plan representation is covered.
190 async function pluginInventoryErrors ( projectDir , plan ) {
191 const plans = require ( '../resources/rp-source-wordpress/lib/plugin-entity-read-plan.js' );
192 const planned = new Map (plans. plannedPluginEntities (projectDir). map ( entity => [entity.sourceEntityRef, entity.planRef]));
193 const pending = new Set ();
194 for ( const row of readEntityMappings (plan)) {
195 if (row?.fieldMappingStatus !== 'pending' ) continue ;
196 const ref = [row.sourceEntityRef, row.sourceEntity, row.entityKey, row.ref]. find ( id => planned. has (id));
197 if (ref) pending. add (ref);
198 }
199 const errors = [];
200 for ( const ref of [ ... pending]. sort ()) {
201 const { state } = await plans. pluginFullReadState (projectDir, ref, planned. get (ref));
202 if (state === 'not-attempted' || state === 'stale' ) errors. push ({ code: 'plugin_full_read_missing' , detail: ref, next: `${ state === 'stale' ? 'Its full-read evidence no longer verifies. ' : ''}Read the entity in full before the mapping checkpoint: node resources/rp-source-wordpress/scripts/plugin-full-read.js --project-dir <dir> --entity ${ ref }. Then map its exact fields and re-run the preflight.` });
203 else if (state === 'complete' ) errors. push ({ code: 'plugin_inventory_unmapped' , detail: ref, next: 'A complete full read exists; map the exact fields from data/source-extract/<entity>.ndjson and set fieldMappingStatus: resolved, then re-run the preflight.' });
204 else if (state === 'components-complete' ) errors. push ({ code: 'plugin_inventory_unmapped' , detail: ref, next: 'Every component was read in full (data/source-extract/<entity>.components.json); its project-code assembly runs only after approval. Map the exact fields from those component outputs and set fieldMappingStatus: resolved, then re-run the preflight.' });
205 }
206 return errors;
207 }
208 // The same dependency rules, run by the mapper BEFORE mapping approval on proposed
209 // operations, census and source evidence -- no setup plan, execution manifest or payload capture is
210 // needed. Proposed exclusions count as dispositions here (approval binds them); setup-producible
211 // refs are `pending setup`, which can pass structural mapping readiness but never execution
212 // readiness. Written to mapping/review/dependency-preflight.json for the mapping review, without
213 // the `work` timing and counters: an approval request pins that file, so it must be byte-stable.
214 async function preflightMappingDependencies ( projectDir , { write = true } = {}) {
215 const plan = read (projectDir, 'mapping/mapping-plan.json' );
216 const reconciliation = await census. readReconciliation (projectDir);
217 const errors = [];
218 const validated = validateAssociations ({ plan, reconciliation });
219 errors. push ( ... validated.errors);
220 const rows = new Map ((reconciliation?.rows || []). map ( row => [row.censusId, row]));
221 const dispositions = {};
222 for ( const [ ref , ids ] of validated.bindings) if (ids. every ( id => rows. get (id)?.state === 'excluded' )) dispositions[ref] = { censusIds: ids, reconciliationSha256: null };
223 const report = proofs. preflightMapping ({ plan, reconciliation, dispositions, readEntityMappings, projectDir });
224 report.errors. push ( ... ( await pluginInventoryErrors (projectDir, plan)));
225 if (report.errors. length ) report.structuralReady = false ;
226 if (errors. length ) { report.errors. unshift ( ... errors. map ( code => ({ code: code. split ( ':' )[ 0 ], detail: code. split ( ':' ). slice ( 1 ). join ( ':' ), next: 'Correct the mapping dependency bindings, then re-run the preflight.' }))); report.structuralReady = false ; }
227 report.mappingPlanDigest = fileDigest (projectDir, 'mapping/mapping-plan.json' );
228 if (write) { const { work , ... persisted } = report; await require ( './orchestration-state' ). writeJsonAtomic ( inside (projectDir, 'mapping/review/dependency-preflight.json' ), persisted); }
229 return report;
230 }
231 function validateImportInputs ( projectDir ) {
232 const manifest = readRequired (projectDir, 'execution/execution-manifest.json' ), setupPlan = readRequired (projectDir, 'setup/setup-plan.json' );
233 const plan = readRequired (projectDir, 'mapping/mapping-plan.json' );
234 if ( read (projectDir, 'orchestration/state.json' , {})?.finalizedAt || read (projectDir, 'orchestration/website-handoff.json' , {})?.finalizedAt) throw new Error ( 'EXECUTION_REVIEW_ONLY' );
235 if (manifest.mappingPlanDigest !== fileDigest (projectDir, 'mapping/mapping-plan.json' )) throw new Error ( 'mapping_plan_digest_mismatch' );
236 let validation = validatedManifests. get (manifest);
237 if ( ! validation) { validation = require ( './execution-manifest' ). validateManifest (manifest); validatedManifests. set (manifest, validation); }
238 if ( ! validation.ok) throw new Error (validation.errors. join ( ';' ));
239 return { manifest, setupPlan, plan };
240 }
241 const validatedManifests = new WeakMap (), planEvidenceFiles = new WeakMap (), manifestSourceFiles = new WeakMap ();
242 function readRequired ( projectDir , file ) {
243 const value = readCached (projectDir, file);
244 if (value === undefined ) throw Object. assign ( new Error ( `ENOENT: no such file or directory, open '${ inside ( projectDir , file ) }'` ), { code: 'ENOENT' });
245 return value;
246 }
247 // Called on every gated send, so the parsed plan and manifest and the paths derived from them are
248 // reused while those files are unchanged; every listed file is still re-stat'ed and re-hashed on change.
249 function fingerprint ( ctx ) {
250 const files = [ 'execution/execution-manifest.json' , 'mapping/mapping-plan.json' , 'setup/setup-plan.json' , 'setup/setup-verification.json' , 'discovery/census.json' , 'mapping/review/mapping-gaps.json' , 'mapping/review/census-reconciliation.json' , 'mapping/review/mapping-summary.md' , 'orchestration/approvals.json' ];
251 const plan = readCached (ctx.projectDir, 'mapping/mapping-plan.json' , {});
252 let evidence = planEvidenceFiles. get (plan);
253 if ( ! evidence) {
254 evidence = [];
255 for ( const a of plan.operationAssessments || []) for ( const e of a.evidence || []) evidence. push (e.artifact);
256 for ( const row of require ( './mapping-field-contract' ). entityRows (plan)) for ( const d of row.fieldMappings || []) if (d.recovery) evidence. push (d.recovery.source.artifact,d.recovery.membership.artifact, ... d.recovery.evidence. map ( e => e.artifact));
257 planEvidenceFiles. set (plan, evidence);
258 }
259 files. push ( ... evidence);
260 let sources = manifestSourceFiles. get (ctx.manifest);
261 if ( ! sources) {
262 sources = [];
263 for ( const task of ctx.manifest.tasks. filter ( task => task.kind === 'import' )) for ( const source of task.sourceBindings || []) sources. push (source.manifestPath, source.extractPath, ... (source.readPlanPath ? [source.readPlanPath] : []));
264 manifestSourceFiles. set (ctx.manifest, sources);
265 }
266 files. push ( ... sources);
267 // Held sets are readiness evidence: an inventory, its builder or the approval request changing
268 // makes an earlier evaluation and its grants stale.
269 for ( const entity of Object. keys ( require ( './read-back-contract' ). DOMAINS )) {
270 const inventory = `execution/read-back-inventory/${ entity }.json` ; files. push (inventory);
271 try { const builder = readCached (ctx.projectDir, inventory).builder?.path; if (builder) files. push (builder); } catch {}
272 }
273 try { const requestId = readCached (ctx.projectDir, 'orchestration/approvals.json' ).mapping?.requestId; if ( / ^ uar_ [0-9a-f-] +$ / . test (requestId || '' )) files. push ( `orchestration/user-actions/${ requestId }.json` ); } catch {}
274 return hash (files. map ( file => { try { return file + ':' + fileDigest (ctx.projectDir, file); } catch { return file + ':missing' ; } }). join ( ' \n ' ));
275 }
276 function statKey ( file ) { try { const stat = fs. statSync (file); return `${ stat . ino }:${ stat . size }:${ stat . mtimeMs }:${ stat . ctimeMs }` ; } catch (e) { if (e.code === 'ENOENT' ) return 'missing' ; throw e; } }
277 function ndjson ( file ) {
278 try { return fs. readFileSync (file, 'utf8' ). split ( / \r ? \n / ). filter (Boolean). map ( line => JSON . parse (line)); }
279 catch (e) { if (e.code === 'ENOENT' ) return []; throw e; }
280 }
281 async function evidenceIndex ( ctx ) {
282 const key = `${ ctx . projectDir }:${ ctx . siteId }:${ ctx . dryRun }` , prior = indexes. get (key);
283 if ( ! subscriptions. has (ctx.projectDir)) subscriptions. set (ctx.projectDir, local. onCrosswalkChange (ctx.projectDir, event => {
284 for ( const [ cachedKey , cached ] of indexes) if (cached.projectDir === ctx.projectDir && cached.dryRun === event.dryRun) {
285 if (cached.crosswalkStamp !== event.before) { indexes. delete (cachedKey); continue ; }
286 for ( const sourceKey of event.remove) { cached.crosswalk. delete (sourceKey); cached.changes. push (sourceKey); }
287 for ( const row of event.updates) { cached.crosswalk. set (row.sourceStableKey, structuredClone (row)); cached.changes. push (row.sourceStableKey); (cached.crosswalkPublished || (cached.crosswalkPublished = [])). push (row.sourceStableKey); }
288 cached.crosswalkStamp = event.after;
289 }
290 }));
291 const crosswalkFile = ctx.dryRun ? local. dryRunCrosswalkPath (ctx.projectDir) : local. crosswalkPath (ctx.projectDir);
292 const crosswalkStamp = statKey (crosswalkFile), passStamp = statKey ( inside (ctx.projectDir, PASS_FILE ));
293 const index = prior || { attempts: new Map (), offset: 0 , changes: [], summaries: new Map (), projectDir: ctx.projectDir, dryRun: ctx.dryRun, recovery: recovery. createRecoveryIndex (ctx.siteId), work: { journalRowsApplied: 0 , dependencyKeysChecked: 0 , waitingKeysProjected: 0 } };
294 let increment;
295 try { increment = await local. readAttemptJournalSince (ctx.projectDir, index.offset); } catch (error) { if (error.code !== 'JOURNAL_MISSING' || index.offset) throw error; increment = { rows: [], nextOffset: 0 , size: 0 }; }
296 if (increment.truncated) { indexes. delete (key); return evidenceIndex (ctx); }
297 for ( const row of increment.rows) { index.work.journalRowsApplied ++ ; local. validateAttemptRow (row); index.attempts. set (row.sourceStableKey, row); recovery. applyRecoveryAttempt (index.recovery, row); index.changes. push (row.sourceStableKey); }
298 index.offset = increment.nextOffset;
299 if (increment.nextOffset !== increment.size) throw new Error ( 'readiness_journal_incomplete_tail' );
300 if ( ! prior || index.crosswalkStamp !== crosswalkStamp) {
301 const rows = ndjson (crosswalkFile);
302 for ( const row of rows) (ctx.dryRun ? local.validateDryRunCrosswalkRow : local.validateCrosswalkRow)(row);
303 index.summaries. clear ();
304 index.crosswalk = new Map (rows. map ( row => [row.sourceStableKey, row])); index.crosswalkStamp = crosswalkStamp;
305 }
306 if ( ! prior || index.passStamp !== passStamp) { index.passes = ndjson ( inside (ctx.projectDir, PASS_FILE )); index.passStamp = passStamp; for ( const row of index.passes) validatePass (row); }
307 indexes. set (key, index); return index;
308 }
309 function validatePass ( row ) {
310 if ( ! row || row.schemaVersion !== 1 || ! row.taskId || ! row.runId || ! row.siteId || ! row.planVersion || ! row.manifestDigest || ! row.setupPlanDigest || ! row.mappingPlanDigest || ! row.sourceSnapshotDigests || ! Array. isArray (row.entityRefs) || ! SELECTIVE . has (row.selection?.mode) || typeof row.dryRun !== 'boolean' || ! Number. isFinite (Date. parse (row.finishedAt))) throw new Error ( 'readiness_pass_malformed' );
311 return row;
312 }
313 // A task's extract must be the version the toolkit recorded, with no fields lost. One the ledger
314 // has never seen is recorded here, before any write; the chain head a read-back inventory pinned
315 // must still be in the ledger, or the extract is re-read from the source.
316 async function recordedExtract ( ctx , source ) {
317 const ledger = require ( './extract-ledger' );
318 let pinned = null ;
319 try {
320 const dir = path. join (ctx.projectDir, 'execution/read-back-inventory' );
321 for ( const name of fs. existsSync (dir) ? fs. readdirSync (dir). filter ( file => file. endsWith ( '.json' )) : []) {
322 const extract = JSON . parse (fs. readFileSync (path. join (dir, name), 'utf8' )).extract;
323 if (extract?.path === source.extractPath && extract.ledgerHead) pinned = extract.ledgerHead;
324 }
325 } catch { throw new Error ( 'dependency_read_back_inventory_unreadable' ); }
326 // Never block: a changed or stripped extract is re-read from the source and reported. Only an
327 // extract that cannot be re-read stops, and only this task.
328 try { await ledger. ensureExtract (ctx.projectDir, { entity: source.sourceEntity, extractPath: source.extractPath, manifestPath: source.manifestPath, keyField: source.keyField, ledgerHead: pinned, recordedBy: 'import-gate' }); return null ; }
329 catch (error) {
330 if (error.code !== 'source_extract_unreadable' ) throw new Error ( `dependency_${ error . code || 'source_extract_unreadable'}` );
331 return { ref: source.ref, state: 'source-extract-unreadable' , reasons: [error.reason || 'extractor_failed' ], recovery: `re-extract:${ source . sourceEntity }` };
332 }
333 }
334 function snapshots ( ctx , task ) {
335 const cacheKey = ctx.projectDir + ':' + task.id, stamp = JSON . stringify ((task.sourceBindings || []). map ( source => [source, statKey ( inside (ctx.projectDir, source.manifestPath)), statKey ( inside (ctx.projectDir, source.extractPath)), source.readPlanPath && statKey ( inside (ctx.projectDir, source.readPlanPath))]));
336 if (snapshotCache. get (cacheKey)?.stamp === stamp) return snapshotCache. get (cacheKey).value;
337 if ( ! Array. isArray (task.sourceBindings) || ! task.sourceBindings. length ) throw new Error ( 'dependency_source_binding_missing' );
338 const digests = {}, keys = new Set (), censusByKey = new Map (), sourceByKey = new Map (); let rowsWritten = 0 , snapshotTotal = 0 ;
339 for ( const source of task.sourceBindings) {
340 if ( ! (task.entityRefs || []). includes (source.ref) || ! source.sourceEntity || ! source.keyField || ! source.keyPrefix) throw new Error ( 'dependency_source_binding_invalid' );
341 const identity = JSON . stringify ([source.ref, source.sourceEntity, source.censusId || `entity:${ source . sourceEntity }` , source.keyField, source.keyPrefix, source.manifestPath, source.extractPath, source.readPlanPath || null ]);
342 if (Object. hasOwn (digests, identity)) throw new Error ( 'dependency_source_binding_duplicate' );
343 const artifacts = { manifest: fileDigest (ctx.projectDir, source.manifestPath), extract: fileDigest (ctx.projectDir, source.extractPath), ... (source.readPlanPath ? { readPlan: fileDigest (ctx.projectDir, source.readPlanPath) } : {}) };
344 const extract = read (ctx.projectDir, source.manifestPath);
345 const records = ndjson ( inside (ctx.projectDir, source.extractPath));
346 let total = extract.snapshotTotal;
347 if (extract.kind === 'eav-read-plan' ) {
348 if ( ! source.readPlanPath) throw new Error ( 'dependency_extract_read_plan_missing' );
349 const client = require ( '../resources/rp-source-wordpress/lib/wix-wp-plugin-v2-client' );
350 if (client. eavExtractionIsReusable ( read (ctx.projectDir, source.readPlanPath), extract, { planFile: source.readPlanPath, ndjsonDigest: fileDigest (ctx.projectDir, source.extractPath), ndjsonRowCount: records. length }). length ) throw new Error ( 'dependency_extract_incomplete' );
351 total = extract.rowsWritten; artifacts.readPlan = fileDigest (ctx.projectDir, source.readPlanPath);
352 }
353 if ( ! Number. isInteger (extract.rowsWritten) || ! Number. isInteger (total) || extract.rowsWritten !== total || extract.rowsWritten < 0 ) throw new Error ( 'dependency_extract_incomplete' );
354 digests[identity] = artifacts;
355 if (records. length !== extract.rowsWritten) throw new Error ( 'dependency_extract_count_mismatch' );
356 for ( const record of records) {
357 const id = valueAt (record, source.keyField);
358 if (id == null || typeof id === 'object' ) throw new Error ( 'dependency_source_key_missing' );
359 const key = source.keyPrefix + String (id);
360 if (keys. has (key)) throw new Error ( 'dependency_source_key_duplicate' );
361 keys. add (key); censusByKey. set (key, source.censusId || `entity:${ source . sourceEntity }` ); sourceByKey. set (key, { ... source, sourceId: String (id) });
362 }
363 rowsWritten += extract.rowsWritten; snapshotTotal += total;
364 }
365 const orderedDigests = Object. fromEntries (Object. entries (digests). sort (([ a ], [ b ]) => a. localeCompare (b)));
366 const value = { digests: orderedDigests, keys, censusByKey, sourceByKey, rowsWritten, snapshotTotal }; snapshotCache. set (cacheKey, { stamp, value }); return value;
367 }
368 function currentPass ( ctx , task , snapshot , pass ) {
369 return pass.taskId === task.id && pass.siteId === ctx.siteId && pass.dryRun === ctx.dryRun &&
370 pass.planVersion === ctx.manifest.planVersion && pass.manifestDigest === fileDigest (ctx.projectDir, 'execution/execution-manifest.json' ) &&
371 pass.setupPlanDigest === fileDigest (ctx.projectDir, 'setup/setup-plan.json' ) && pass.mappingPlanDigest === fileDigest (ctx.projectDir, 'mapping/mapping-plan.json' ) &&
372 equal (pass.sourceSnapshotDigests, snapshot.digests) && pass.selection.mode === 'full' ;
373 }
374 function finishedThisRun ( ctx , index , task ) {
375 return index.passes. some ( pass => pass.taskId === task.id && pass.runId === ctx.runId && pass.siteId === ctx.siteId && pass.dryRun === ctx.dryRun && pass.selection.mode === 'full' );
376 }
377 // While a per-record producer is `in-progress`, a key counts once the producer wrote or re-checked
378 // it in this run, or once it already landed on this site in an earlier run (its latest attempt is a
379 // same-site imported/already-imported row and its same-site crosswalk row is imported). A resumed
380 // producer does not undo an earlier confirmed write; requiring a fresh write or check kept every
381 // dependent of a resumed producer waiting. keyReady still checks site and identity for each key; a
382 // key this run re-attempts is judged by that latest attempt; anything else waits.
383 const LANDED = new Set ([ 'imported' , 'skipped_already_imported' ]);
384 function producedThisRun ( ctx , index , key , dep ) {
385 if (dep.state !== 'in-progress' ) return true ;
386 const attempt = index.attempts. get (key), row = index.crosswalk. get (key);
387 if (attempt?.runId === ctx.runId) return true ;
388 return Boolean (attempt && row && LANDED . has (attempt.status) && attempt.siteId === ctx.siteId && row.siteId === ctx.siteId && row.status === 'imported' );
389 }
390 // The crosswalk row is this source record's own, for its entity: shared by keyReady and the
391 // post-grant fallback in currentRecordTargetId so both apply one target contract.
392 function rowMatchesSource ( row , source ) {
393 const types = source ? [source.ref, source.ref. split ( '/' )[ 1 ], source.ref. replace ( '/' , '-' ), source.ref. replace ( '/' , '.' )] : [];
394 return Boolean (source) && row.targetSystem === 'wix' && row.sourceEntityType === source.sourceEntity && String (row.sourceId) === source.sourceId && types. includes (row.targetEntityType);
395 }
396 function keyReady ( ctx , index , key , owner ) {
397 index.work.dependencyKeysChecked ++ ;
398 const attempt = index.attempts. get (key), row = index.crosswalk. get (key);
399 if ((attempt && attempt.siteId !== ctx.siteId) || (row && row.siteId !== ctx.siteId)) return { ready: false , foreign: true };
400 if ( ! attempt || ! row || row.status !== 'imported' || ! [ 'imported' , 'skipped_already_imported' ]. includes (attempt.status)) return { ready: false };
401 if (owner) {
402 const source = owner.snapshot.sourceByKey. get (key);
403 const types = source ? [source.ref, source.ref. split ( '/' )[ 1 ], source.ref. replace ( '/' , '-' ), source.ref. replace ( '/' , '.' )] : [];
404 if ( ! source || owner.ref && source.ref !== owner.ref || attempt.writeSpecId !== owner.task.id || ! rowMatchesSource (row, source) || ! types. includes (attempt.targetEntityType)) return { ready: false , identity: false };
405 }
406 return { ready: true , row };
407 }
408 // Why a key that is not ready will not land in this run, or null while it still may. While the
409 // producer is in progress only this run's terminal failure is final (an earlier run's may be retried).
410 // Once its pass is current, any key not ready and not held is final: its last status, `not-attempted`,
411 // or `not-in-source` when the producer's extract has no such record.
412 function failedForGood ( ctx , index , key , dep , inSource ) {
413 const attempt = index.attempts. get (key);
414 if (attempt && attempt.siteId !== ctx.siteId) return null ;
415 if (dep.state === 'in-progress' ) return attempt?.runId === ctx.runId && attempt.status === 'failed_terminal' ? attempt.status : null ;
416 if ( ! inSource) return 'not-in-source' ;
417 return attempt?.status || 'not-attempted' ;
418 }
419 // An id the gate never resolved is never sent. For a ref the record can go without
420 // (`dependencyFailureActions: omit`) at a one-array path (a gallery item's id), the items carrying
421 // such ids are dropped, the rest keep their order, and each drop is recorded like a failed
422 // dependency; an id that is exactly one ready record's target is kept as that record's reference.
423 // Any other unresolved id defers the record (returns false).
424 function dropUnchecked ( payload , references , entityRefs , ref , template , unresolved , lookup ) {
425 const parts = template. split ( '[]' );
426 if (parts. length !== 2 || ! parts[ 1 ]. startsWith ( '.' ) || ! unresolved. every ( concrete => failureOmits (entityRefs, ref, concrete))) return false ;
427 const arrayPath = parts[ 0 ], items = valueAt (payload, arrayPath);
428 if ( ! Array. isArray (items)) return false ;
429 // An id that is exactly one ready record's target is that record's reference, not a drop.
430 unresolved = unresolved. filter ( concrete => { const key = lookup (ref, valueAt (payload, concrete)); if (key) references. push ({ ref, sourceStableKey: key, path: concrete, targetId: valueAt (payload, concrete) }); return ! key; });
431 if ( ! unresolved. length ) return true ;
432 const index = concrete => Number (concrete. slice (arrayPath. length + 1 , concrete. indexOf ( ']' , arrayPath. length )));
433 const dropped = new Set (unresolved. map (index)), moved = new Map ();
434 for ( let i = 0 , kept = 0 ; i < items. length ; i ++ ) if ( ! dropped. has (i)) moved. set (i, kept ++ );
435 for ( const concrete of unresolved) references. push ({ ref, sourceStableKey: null , path: concrete, action: 'omit-failed' , dependencyStatus: 'unchecked-reference' , uncheckedTargetId: String ( valueAt (payload, concrete)) });
436 // Granted references in the same array follow their item to its new index.
437 for ( const reference of references) {
438 if (reference.action || ! reference.path. startsWith (arrayPath + '[' )) continue ;
439 const at = index (reference.path);
440 if (moved. has (at)) reference.path = `${ arrayPath }[${ moved . get ( at ) }]` + reference.path. slice (reference.path. indexOf ( ']' , arrayPath. length ) + 1 );
441 }
442 const keys = arrayPath. replace ( / \[ ( \d + ) \] / g , '.$1' ). split ( '.' ), last = keys. pop (), parent = keys. reduce (( value , key ) => value?.[key], payload);
443 parent[last] = items. filter (( _ , i ) => ! dropped. has (i));
444 return true ;
445 }
446 // Target knowledge's `dependencyFailureActions`: `omit` lets a record be written without a
447 // reference whose own dependency record failed. Every entity of the task must declare it at this path.
448 function failureOmits ( entityRefs , ref , concrete ) {
449 return entityRefs. every ( entityRef => domain (entityRef).dependencyFailureActions?.[ref] === 'omit' ) && declaresPath (entityRefs, ref, concrete);
450 }
451 // A producer key the owner intentionally held: its builder hold is in the current, approved
452 // read-back inventory. Such a key is settled for readiness, never missing; a
453 // held set that is not current honors no hold, so its keys stay not ready.
454 async function heldKeys ( ctx , task , snapshot ) {
455 const keys = new Map (), errors = [], stamp = [];
456 const readBack = require ( './read-back-contract' ), plan = read (ctx.projectDir, 'mapping/mapping-plan.json' );
457 for ( const source of task.sourceBindings || []) {
458 const entity = Object. keys (readBack. DOMAINS ). find ( name => `${ readBack . DOMAINS [ name ] }/${ name }` === source.ref);
459 if ( ! entity) continue ;
460 const held = await readBack. approvedHeldRecords (ctx.projectDir, entity, plan);
461 stamp. push ([entity, held.ok, held.inventorySha256 || null ]);
462 if ( ! held.ok) { errors. push (held.reason); continue ; }
463 for ( const [ sourceId , reasons ] of held.held) {
464 const key = source.keyPrefix + sourceId;
465 if (snapshot.sourceByKey. get (key)?.ref === source.ref) keys. set (key, reasons);
466 }
467 }
468 return { keys, errors: [ ...new Set (errors)], stamp: JSON . stringify (stamp) };
469 }
470 function readinessSummary ( ctx , index , task , snapshot , held ) {
471 let summary = index.summaries. get (task.id);
472 const status = key => { const row = keyReady (ctx, index, key, { task, snapshot }); return ! row.ready && ! row.foreign && held.keys. has (key) ? { ... row, held: true } : row; };
473 const add = ( row , sign ) => { summary.ready += sign * Number (row.ready); summary.foreign += sign * Number ( Boolean (row.foreign)); summary.held += sign * Number ( Boolean (row.held)); };
474 if ( ! summary || summary.snapshot !== snapshot || summary.heldStamp !== held.stamp) {
475 summary = { snapshot, heldStamp: held.stamp, ready: 0 , foreign: 0 , held: 0 , statuses: new Map (), offset: index.changes. length };
476 for ( const key of snapshot.keys) { const row = status (key); summary.statuses. set (key, row); add (row, 1 ); }
477 index.summaries. set (task.id, summary); return summary;
478 }
479 for ( const key of new Set (index.changes. slice (summary.offset))) if (snapshot.keys. has (key)) {
480 const after = status (key); add (summary.statuses. get (key), - 1 ); add (after, 1 ); summary.statuses. set (key, after);
481 }
482 summary.offset = index.changes. length ; return summary;
483 }
484 async function evaluateDependency ({ ctx , ref , producer , mode , index , dispositions , proofs : validProofs = new Map () }) {
485 const unknown = ( reason , recovery ) => ({ ref, producer, state: 'unknown' , reasons: [reason], recovery });
486 try {
487 if (producer.kind === 'inherited' ) return { ref, producer, state: 'inherited' , reasons: [] };
488 if (producer.kind === 'no-producer' ) {
489 // Re-validated against current knowledge and source evidence on every evaluation; a catch-up
490 // extract that now contains the dependency contradicts the proof for referencing records.
491 const current = producer.proofs. map ( row => validProofs. get (proofs. proofKey (row.entityRef, row.operation, ref)));
492 if (current. some (( proof , i ) => ! proof || proof.id !== producer.proofs[i].proofId || proof.operationContractDigest !== producer.proofs[i].operationContractDigest)) return unknown ( 'dependency_proof_not_current' , `reverify-dependency-proof:${ ref }` );
493 return { ref, producer, state: current. some ( proof => proof.status === 'contradicted' ) ? 'proof-contradicted' : 'proven' , reasons: [], recovery: `reverify-dependency-proof:${ ref }` };
494 }
495 if ( ! [ 'import-task' , 'secondary-output' , 'child-output' , 'setup-step' , 'disposition' ]. includes (producer.kind)) return unknown ( 'dependency_producer_unsupported' , `regenerate-import:${ ref }` );
496 if (producer.kind === 'disposition' ) {
497 const disposition = dispositions[ref];
498 if ( ! disposition) return unknown ( 'disposition_not_current' , `obtain-disposition-approval:${ ref }` );
499 return { ref, producer, state: disposition.approved ? 'excluded-approved' : 'excluded-unapproved' , reasons: [], recovery: `obtain-disposition-approval:${ ref }` };
500 }
501 if (producer.kind === 'setup-step' ) {
502 const receipt = read (ctx.projectDir, 'setup/setup-verification.json' );
503 const validation = setup. validateSetupVerification ({ verification: receipt, plan: ctx.setupPlan, planDigest: fileDigest (ctx.projectDir, 'setup/setup-plan.json' ), siteId: ctx.siteId, projectDir:ctx.projectDir });
504 if ( ! validation.ok || receipt.dryRun !== ctx.dryRun) return unknown (validation.reasons. join ( ';' ) || 'dry_run_mismatch' , `run-setup:${ producer . stepIds [ 0 ] }` );
505 return { ref, producer, state: 'setup-passed' , reasons: [], receipt };
506 }
507 const task = ctx.manifest.tasks. find ( row => row.id === producer.taskId), snapshot = snapshots (ctx, task);
508 // A per-record edge does not wait for the producer's whole pass until that pass finishes in this
509 // run (whose manifest and setup plan are fixed): each dependent record waits only for the keys it
510 // references, and only keys the producer wrote or re-checked in this run count. A record that
511 // references none sends. A full pass that finished and is no longer current stays `unknown`.
512 const passed = index.passes. some ( pass => currentPass (ctx, task, snapshot, pass));
513 if ( ! passed && (mode !== 'per-record' || finishedThisRun (ctx, index, task))) return unknown ( 'current_full_pass_missing' , `run-import:${ task . id }` );
514 const held = await heldKeys (ctx, task, snapshot), summary = readinessSummary (ctx, index, task, snapshot, held);
515 if (summary.foreign) return unknown ( 'site_evidence_unbound_or_foreign' , `reconcile-site:${ task . id }` );
516 // Every key not imported is an approved hold: settled, so task-complete dependents may run.
517 // A current full pass that left other keys unlanded is settled. Dependents proceed and
518 // records naming those keys become final instead of waiting forever.
519 const state = ! passed ? 'in-progress' : snapshot.rowsWritten === 0 && snapshot.snapshotTotal === 0 ? 'empty' : summary.ready === snapshot.keys.size ? 'complete' : summary.held && summary.ready + summary.held === snapshot.keys.size ? 'complete-with-holds' : 'finished-partial' ;
520 return { ref, producer, state, reasons: held.errors, recovery: `run-import:${ task . id } --failed-only` , snapshot, ... (summary.held ? { heldCount: summary.held, held: held.keys } : {}) };
521 } catch (error) { return unknown ( evidenceReason (error), producer.kind === 'setup-step' ? `run-setup:${ producer . stepIds [ 0 ] }` : `run-import:${ producer . taskId || ''}` ); }
522 }
523 // execution/dependency-readiness.json is shared by task evaluation, the batch projection and
524 // within-write gap records, and observations run concurrently. Every read-modify-write of it runs
525 // through this per-file queue, so overlapping updates never drop one another's tasks or gaps.
526 // (In-process only; one importer process owns a project's execution state.)
527 const diagnosticQueues = new Map ();
528 async function updateDiagnostic ( projectDir , mutate ) {
529 const file = inside (projectDir, 'execution/dependency-readiness.json' ), prior = diagnosticQueues. get (file) || Promise . resolve ();
530 let release; const turn = new Promise ( resolve => { release = resolve; }), tail = prior. then (() => turn);
531 diagnosticQueues. set (file, tail);
532 await prior;
533 try {
534 const diagnostic = read (projectDir, 'execution/dependency-readiness.json' , { schemaVersion: 1 , tasks: {} });
535 diagnostic.tasks = diagnostic.tasks || {};
536 const result = await mutate (diagnostic);
537 if (result !== false ) await require ( './orchestration-state' ). writeJsonAtomic (file, diagnostic);
538 return result;
539 } finally { release (); if (diagnosticQueues. get (file) === tail) diagnosticQueues. delete (file); }
540 }
541 // The holds a deferral waits on. A `dependency-held` row journaled before holdCauses existed names
542 // its one hold at the top level.
543 function holdCausesOf ( deferral ) {
544 if (Array. isArray (deferral?.holdCauses)) return deferral.holdCauses;
545 return deferral?.reason === 'dependency-held' && deferral.ref ? [{ ref: deferral.ref, sourceStableKey: deferral.sourceStableKey, holdReasons: deferral.holdReasons || [] }] : [];
546 }
547 // Dependents waiting on an owner-held record, grouped by the held ref: the cutover cause.
548 function heldDependencies ( waiting ) {
549 const groups = new Map ();
550 for ( const row of waiting || []) for ( const ref of new Set ( holdCausesOf (row.deferral). map ( cause => cause.ref))) {
551 const group = groups. get (ref) || { ref, heldSourceKeys: new Set (), holdReasons: new Set (), waitingCount: 0 };
552 for ( const cause of holdCausesOf (row.deferral). filter ( cause => cause.ref === ref)) { group.heldSourceKeys. add (cause.sourceStableKey); for ( const reason of cause.holdReasons || []) group.holdReasons. add (reason); }
553 group.waitingCount ++ ; groups. set (ref, group);
554 }
555 return [ ... groups. values ()]. sort (( a , b ) => a.ref. localeCompare (b.ref)). map ( group => ({ ... group, heldSourceKeys: [ ... group.heldSourceKeys]. sort (), holdReasons: [ ... group.holdReasons]. sort () }));
556 }
557 function finalDeferrals ( waiting ) {
558 const rows = [];
559 for ( const row of waiting || []) if (row.deferral?.final === true ) for ( const cause of row.deferral.finalCauses || []) rows. push ({ sourceStableKey: row.sourceStableKey, ref: cause.ref, dependencySourceKey: cause.sourceStableKey, parentStatus: cause.parentStatus, linkChoice: cause.linkChoice, ... (cause.linkRequired ? { linkRequired: true } : {}), ... (cause.parentSourceKey ? { parentSourceKey: cause.parentSourceKey } : {}), ... (cause.holdReasons ? { holdReasons: cause.holdReasons } : {}), producerTaskId: cause.producerTaskId ?? null });
560 return rows. sort (( a , b ) => (a.sourceStableKey + a.ref + a.dependencySourceKey). localeCompare (b.sourceStableKey + b.ref + b.dependencySourceKey));
561 }
562 function projectWaiting ( ctx , index ) {
563 if ( ! index) return read (ctx.projectDir, 'execution/dependency-readiness.json' , { tasks: {} }).tasks?.[ctx.taskId]?.waiting || [];
564 index.work.waitingKeysProjected += index.recovery.byTask. get (ctx.taskId)?.size || 0 ;
565 return recovery. waitingForTask (index.recovery, ctx.taskId);
566 }
567 function installReadBack ( receipt , stepId , app ) {
568 const check = (receipt?.requirements || []). find ( row => row?.id === stepId)?.verification;
569 return check?.method === SETUP_CHECK_METHODS . INSTALLED_APPS_READ && check.observed?.appDefId === app.appDefId && check.observed.installed === true ;
570 }
571 async function evaluateTask ({ taskBinding }) {
572 const ctx = capability. requireKind (taskBinding, 'import-task' );
573 const current = validateImportInputs (ctx.projectDir);
574 if ((current.manifest !== ctx.manifest && ! equal (current.manifest, ctx.manifest)) || (current.setupPlan !== ctx.setupPlan && ! equal (current.setupPlan, ctx.setupPlan))) throw new Error ( 'DEPENDENCY_INPUT_CHANGED' );
575 const sourceBlocks = [];
576 for ( const source of ctx.task.sourceBindings || []) { const block = await recordedExtract (ctx, source); if (block) sourceBlocks. push (block); }
577 let adapter, index, inputError;
578 try {
579 adapter = await loadDependencyDispositions (ctx.projectDir, { stage: 'runtime' });
580 const binding = bindDependencyProducers ({ manifest: ctx.manifest, setupPlan: ctx.setupPlan, dispositions: adapter.dispositions, proofs: adapter.proofs, adoptOnlyRefs: adapter.adoptOnlyRefs, adoptOnlyMixed: adapter.adoptOnlyMixed });
581 if ( ! binding.ok) throw new Error (binding.errors. join ( ';' ));
582 index = await evidenceIndex (ctx);
583 } catch (error) { inputError = error; }
584 const dependencies = [];
585 for ( const edge of ctx.task.dependencyEdges) dependencies. push (inputError ? { ... edge, state: 'unknown' , reasons: [ evidenceReason (inputError)], recovery: edge.producer.kind === 'setup-step' ? `run-setup:${ edge . producer . stepIds [ 0 ] }` : edge.producer.kind === 'disposition' ? `obtain-disposition-approval:${ edge . ref }` : `run-import:${ edge . producer . taskId }` } : { ...await evaluateDependency ({ ctx, ... edge, index, dispositions: adapter.dispositions, proofs: adapter.proofs }), mode: edge.mode });
586 // An app the task's entities live in is a prerequisite the entity edges do not name: it must
587 // be a scheduled setup step that passed, and that step's pass already includes the live read-back.
588 for ( const app of requiredAppsFor ([ ... (ctx.task.entityRefs || []), ... (ctx.task.dependencyRefs || [])])) {
589 const stepId = app.stepIds. find ( id => (ctx.setupPlan.steps || []). some ( step => step.id === id));
590 const ref = `app:${ app . app }` ;
591 if ( ! stepId) { dependencies. push ({ ref, producer: { kind: 'setup-step' , stepIds: [app.stepIds[ 0 ]] }, state: 'unknown' , reasons: [ `app_install_step_missing:${ app . app }` ], recovery: `add-setup-step:${ app . stepIds [ 0 ] }` , mode: 'task-complete' }); continue ; }
592 const producer = { kind: 'setup-step' , stepIds: [stepId] };
593 if (inputError) { dependencies. push ({ ref, producer, state: 'unknown' , reasons: [ evidenceReason (inputError)], recovery: `run-setup:${ stepId }` , mode: 'task-complete' }); continue ; }
594 const dep = { ...await evaluateDependency ({ ctx, ref, producer, index, dispositions: adapter.dispositions }), mode: 'task-complete' };
595 // A passed receipt is not enough: the installer once passed this step after installing a
596 // different app, and that receipt stays digest-valid. Only a live installed-apps read of this
597 // app's id counts, so a receipt from before that fix reruns setup.
598 if (dep.state === 'setup-passed' && ! installReadBack (dep.receipt, stepId, app)) { dependencies. push ({ ref, producer, state: 'unknown' , reasons: [ `app_install_not_read_back:${ app . app }` ], recovery: `run-setup:${ stepId }` , mode: 'task-complete' }); continue ; }
599 dependencies. push (dep);
600 }
601 if (inputError && ! dependencies. length ) throw inputError;
602 const blocked = dependencies. filter ( dep => ! SETTLED . has (dep.state) && ! ([ 'in-progress' , 'proof-contradicted' ]. includes (dep.state) && dep.mode === 'per-record' ));
603 // Every other entity builds on the regional setup step (files only; the step already read the site).
604 const regional = require ( './site-regional' ). regionalDependency ({ projectDir: ctx.projectDir, siteId: ctx.siteId, entityRefs: ctx.task.entityRefs || [] });
605 if (regional.gated && ! regional.ok) blocked. push ({ ref: 'site/locale' , state: 'locale-refused' , reasons: [regional.code], recovery: 'site-locale-gate' });
606 blocked. push ( ... sourceBlocks);
607 const waiting = projectWaiting (ctx, index);
608 const evaluation = Object. freeze ({ taskId: ctx.taskId, blocked: blocked. map (({ receipt , snapshot , held , ... row }) => row), ok: blocked. length === 0 , work: Object. freeze ({ ... index?.work }) });
609 const generation = (evaluationGenerations. get (taskBinding) || 0 ) + 1 ; evaluationGenerations. set (taskBinding, generation);
610 evaluations. set (evaluation, { ctx, dependencies, index, adapter, stamp: fingerprint (ctx), taskBinding, generation });
611 const finalRows = finalDeferrals (waiting);
612 const diagnostic = { waiting, heldDependencies: heldDependencies (waiting), ... (finalRows. length ? { finalDeferrals: finalRows } : {}), schemaVersion: 1 , siteId: ctx.siteId, taskId: ctx.taskId, blocked: evaluation.blocked,
613 affectedCount: ctx.task.population ?? null , operatorRole: 'Requires Import Operator' ,
614 action: `Inspect execution/dependency-readiness.json for task ${ ctx . taskId }; resolve its prerequisite and resume only eligible deferred records. Do not resend existing orders.` };
615 // Within-write gaps, reference gaps and omitted references are carried over from the current
616 // file inside the same serialized update.
617 await updateDiagnostic (ctx.projectDir, current => {
618 const gaps = current.tasks[ctx.taskId]?.withinWriteGaps, referenceGaps = current.tasks[ctx.taskId]?.referenceGaps, omitted = current.tasks[ctx.taskId]?.omittedReferences;
619 current.schemaVersion = 1 ; current.tasks[ctx.taskId] = { ... diagnostic, ... (gaps?. length ? { withinWriteGaps: gaps } : {}), ... (referenceGaps?. length ? { referenceGaps } : {}), ... (omitted?. length ? { omittedReferences: omitted } : {}) };
620 });
621 if (blocked. length ) await require ( './execution-log' ). appendExecutionLog (ctx.projectDir, { runId: ctx.runId, phase: 'import' , event: 'dependency-readiness-blocked' , detail: JSON . stringify ({ taskId: ctx.taskId, blocked: evaluation.blocked }), artifacts: [ 'execution/dependency-readiness.json' ] });
622 return evaluation;
623 }
624 function expandPaths ( payload , template ) {
625 const result = [];
626 function visit ( value , parts , prefix ) {
627 if ( ! parts. length ) { result. push (prefix); return ; }
628 const [ part , ... rest ] = parts, array = part. endsWith ( '[]' ), key = array ? part. slice ( 0 , - 2 ) : part;
629 const next = prefix ? prefix + '.' + key : key;
630 if (array) { if (Array. isArray (value?.[key])) value[key]. forEach (( entry , index ) => visit (entry, rest, next + '[' + index + ']' )); }
631 else if (value && Object. hasOwn (value, key)) visit (value[key], rest, next);
632 }
633 visit (payload, template. split ( '.' ), '' ); return result;
634 }
635 // A runtime gap: a record the transform deferred with its own reason (a reference-fidelity or
636 // shipping-cardinality blocker), or one the shared runner wrote with a recorded loss (a missing
637 // parent link, a possible duplicate). It is kept in the runtime diagnostic, never in the reviewed
638 // mapping/review/mapping-gaps.json: that file is readiness and census-reconciliation evidence, so
639 // writing it mid-batch made every grant of the batch stale and the next send was refused.
640 const REFERENCE_GAP_ACTION = 'Review this source-key scope and the recorded reference-fidelity or shipping-cardinality blocker. Obtain the existing required owner decision or verified preserving mapping, regenerate and approve current mapping evidence, then resume only affected deferred records. Do not resend existing orders.' ;
641 // One row per code; each affected record keeps its own detail (target id, parent, prior attempt),
642 // so a cutover item never names only the first record.
643 // operatorRole 'Cutover Operator' marks a gap the Cutover Operator resolves in Business Manager
644 // (no original migration environment needed); the default stays the Import Operator.
645 const GAP_ROLES = new Set ([ 'Requires Import Operator' , 'Cutover Operator' ]), GAP_SEVERITIES = new Set ([ 'blocker' , 'high' , 'medium' , 'low' ]);
646 async function persistReferenceGap ( ctx , sourceStableKey , code , { action = REFERENCE_GAP_ACTION , written = false , detail = {}, operatorRole = 'Requires Import Operator' , severity = 'blocker' } = {}) {
647 if ( ! GAP_ROLES . has (operatorRole) || ! GAP_SEVERITIES . has (severity)) throw new Error ( 'TASK_GAP_INVALID' );
648 await updateDiagnostic (ctx.projectDir, diagnostic => {
649 const task = diagnostic.tasks[ctx.taskId] || (diagnostic.tasks[ctx.taskId] = { schemaVersion: 1 , siteId: ctx.siteId, taskId: ctx.taskId, blocked: [], waiting: [] });
650 const rows = task.referenceGaps || (task.referenceGaps = []);
651 let row = rows. find ( row => row.code === code);
652 if ( ! row) rows. push (row = { code, taskId: ctx.taskId, siteId: ctx.siteId, severity, written, affectedSourceKeys: [], records: [], operatorRole, action });
653 row.records = [ ... (row.records || []). filter ( entry => entry.sourceStableKey !== sourceStableKey), { sourceStableKey, ... detail }]. sort (( a , b ) => a.sourceStableKey. localeCompare (b.sourceStableKey));
654 row.affectedSourceKeys = row.records. map ( entry => entry.sourceStableKey);
655 rows. sort (( a , b ) => a.code. localeCompare (b.code));
656 });
657 }
658 // A gap on a record the shared runner wrote with a recorded loss: `action` is the per-code next
659 // step, `detail` this record's facts (no free text).
660 function assertTaskGap ( code , action , detail ) {
661 if ( typeof code !== 'string' || ! / ^ [a-z0-9-] +$ / . test (code) || typeof action !== 'string' || ! action. trim () || ! detail || typeof detail !== 'object' ) throw new Error ( 'TASK_GAP_INVALID' );
662 }
663 async function recordTaskGap ({ taskBinding , sourceStableKey , code , action , detail = {}, operatorRole , severity }) {
664 assertTaskGap (code, action, detail);
665 return persistReferenceGap (capability. requireKind (taskBinding, 'import-task' ), sourceStableKey, code, { action, written: true , detail, operatorRole, severity });
666 }
667 // The same, for a writer holding only the import client its task was bound to (createWixClient
668 // registers it; nothing else can produce one). Used by toolkit composites such as importOrderHistory.
669 async function recordClientTaskGap ({ wixClient , sourceStableKey , code , action , detail = {}, operatorRole , severity }) {
670 assertTaskGap (code, action, detail);
671 const ctx = capability. clientContext (wixClient);
672 if ( ! ctx || ctx.kind !== 'import-task' ) capability. fail ( 'EXECUTION_CAPABILITY_REQUIRED' );
673 return persistReferenceGap (ctx, sourceStableKey, code, { action, written: true , detail, operatorRole, severity });
674 }
675 // Verified receipts live in protected state (state/write-verification.json and its checkpoint
676 // journal); crosswalk rows usually hold only a digest reference. A revocation can change that state
677 // without touching the crosswalk, so its stamp is part of every receipt cache key.
678 function receiptStateStamp ( projectDir ) {
679 const dir = inside (projectDir, 'state/write-verification-journal' );
680 let names = [];
681 try { names = fs. readdirSync (dir). filter ( name => name. endsWith ( '.json' )). sort (); } catch (e) { if (e.code !== 'ENOENT' ) throw e; }
682 return [ statKey ( inside (projectDir, 'state/write-verification.json' )), ... names. map ( name => name + ':' + statKey (path. join (dir, name)))]. join ( '|' );
683 }
684 // The verified receipts of one producer task's rows for a ref, loaded once and kept until that
685 // task's crosswalk rows or the protected state change (dependent writes alone never reload it).
686 async function taskReceipts ( ctx , index , task , ref , stamp ) {
687 const cache = index.receipts || (index.receipts = new Map ()), snapshot = snapshots (ctx, task), cacheKey = task.id + ':' + ref, prior = cache. get (cacheKey);
688 if (prior && prior.snapshot === snapshot && prior.crosswalk === index.crosswalk && prior.stamp === stamp && ! index.changes. slice (prior.offset). some ( key => snapshot.keys. has (key))) { prior.offset = index.changes. length ; return prior; }
689 const rows = [ ... snapshot.keys]. filter ( key => snapshot.sourceByKey. get (key).ref === ref). map ( key => index.crosswalk. get (key)). filter ( row => row?.verification);
690 const entry = { snapshot, crosswalk: index.crosswalk, stamp, offset: index.changes. length , task, receipts: await require ( './write-verification-state' ). verifiedReceipts (ctx.projectDir, rows), derived: new Map () };
691 cache. set (cacheKey, entry); return entry;
692 }
693 // Every receipt-backed reference the batch may resolve: a verified producer's own rows, and the
694 // parent rows whose receipts name child identities (a product's variants), indexed by child key.
695 async function batchReceipts ( ctx , index , dependencies ) {
696 const result = { stamp: receiptStateStamp (ctx.projectDir), parents: new Map (), children: new Map () };
697 for ( const dep of dependencies) {
698 const task = ctx.manifest.tasks. find ( row => row.id === dep.producer.taskId);
699 if (dep.producer.kind === 'import-task' && task?.verificationLevel === 'verified' && [ 'crm/contact' , 'stores/product' ]. includes (dep.ref)) result.parents. set (dep.ref, await taskReceipts (ctx, index, task, dep.ref, result.stamp));
700 if (dep.producer.kind !== 'child-output' ) continue ;
701 const declared = proofs. operationContract (dep.producer.parentRef, task?.writeOperations?.[dep.producer.parentRef])?.contract.childOutputs?.[dep.ref];
702 if ( ! task || declared?.verification !== 'write-verification-receipt' || ! declared.receipt?.path) { result.children. set (dep.ref, { state: 'child-contract-missing' }); continue ; }
703 const entry = await taskReceipts (ctx, index, task, dep.producer.parentRef, result.stamp);
704 if ( ! entry.derived. has (dep.ref)) {
705 const owners = new Map ();
706 for ( const [ parentKey , receipt ] of entry.receipts) for ( const row of Array. isArray (receipt[declared.receipt.path]) ? receipt[declared.receipt.path] : []) {
707 const key = row?.[declared.receipt.keyField];
708 if ( typeof key === 'string' && key) owners. set (key, [ ... (owners. get (key) || []), { parentKey, receipt, targetId: row[declared.receipt.idField] }]);
709 }
710 entry.derived. set (dep.ref, { snapshot: entry.snapshot, parentTask: task, owners });
711 }
712 result.children. set (dep.ref, entry.derived. get (dep.ref));
713 }
714 return result;
715 }
716 // Exactly one ready, same-site parent whose verified receipt names this child key exactly once.
717 function childIdentity ( ctx , index , children , producer , key , dep = {}) {
718 if ( ! children?.owners) return { state: children?.state || 'child-contract-missing' };
719 const matches = children.owners. get (key) || [];
720 if ( new Set (matches. map ( row => row.parentKey)).size > 1 || matches. length > 1 ) return { state: 'child-identity-ambiguous' };
721 if ( ! matches. length ) return { state: 'child-identity-missing' };
722 const [ match ] = matches, parent = keyReady (ctx, index, match.parentKey, { task: children.parentTask, snapshot: children.snapshot, ref: producer.parentRef });
723 if ( ! parent.ready || ! producedThisRun (ctx, index, match.parentKey, dep)) return { state: 'parent-not-ready' , parentKey: match.parentKey };
724 if ( ! require ( './write-verification' ). validReceipt (match.receipt, { siteId: ctx.siteId, sourceKey: match.parentKey, targetId: parent.row.targetId })) return { state: 'imported-unverified' };
725 if ( typeof match.targetId !== 'string' || ! match.targetId) return { state: 'child-identity-ambiguous' };
726 return { targetId: match.targetId, parentKey: match.parentKey, parentTargetId: parent.row.targetId };
727 }
728 // One written path may hold either of several refs (a coupon scope names a product or a category);
729 // only a ref that declares the path can cover it.
730 // Target knowledge's `sameTaskReferenceFields`: a reference to an entity this same task writes.
731 function sameTaskFields ( entityRefs ) {
732 return entityRefs. flatMap ( entityRef => Object. entries ( domain (entityRef).sameTaskReferenceFields || {})
733 . filter (([ ref ]) => entityRefs. includes (ref))
734 . flatMap (([ ref , templates ]) => templates. map ( template => ({ ref, template }))));
735 }
736 function declaresPath ( entityRefs , ref , concrete ) { return entityRefs. some ( entityRef => ( domain (entityRef).dependencyReferenceFields?.[ref] || []). some ( template => pathMatches (template, concrete))); }
737 function valueAt ( payload , concrete ) { return concrete. replace ( / \[ ( \d + ) \] / g , '.$1' ). split ( '.' ). reduce (( value , key ) => value?.[key], payload); }
738 function isCustomOrderLine ( payload , concrete ) {
739 const match = / ^ lineItems \[ ( \d + ) \]\. catalogReference \. catalogItemId $ / . exec (concrete);
740 if ( ! match) return false ;
741 const line = payload?.lineItems?.[ Number (match[ 1 ])];
742 return Boolean (line?.itemType?.custom) && ! Object. hasOwn (line, 'catalogReference' );
743 }
744 function omitAt ( payload , concrete ) { const keys = concrete. replace ( / \[ ( \d + ) \] / g , '.$1' ). split ( '.' ); const key = keys. pop (); const parent = keys. reduce (( value , part ) => value?.[part], payload); if (parent) delete parent[key]; }
745 async function gateBatch ({ evaluation , records }) {
746 const state = evaluations. get (evaluation);
747 if ( ! state || state.stamp !== fingerprint (state.ctx) || evaluationGenerations. get (state.taskBinding) !== state.generation) throw new Error ( 'DEPENDENCY_EVALUATION_STALE' );
748 if ( ! evaluation.ok) throw Object. assign ( new Error ( 'DEPENDENCY_NOT_READY' ), { diagnostic: evaluation });
749 const { ctx , dependencies , index , adapter } = state, send = [], defer = [], allowed = new Set ();
750 const receipts = await batchReceipts (ctx, index, dependencies);
751 // A parent the same task writes (a nested category) is never an edge; it resolves only from this
752 // task's own landed row, so the journal is re-read once per batch: the previous depth level may
753 // have just written it.
754 const ownFields = sameTaskFields (ctx.task.entityRefs);
755 const ownIndex = ownFields. length ? await evidenceIndex (ctx) : null , checked = checkedRefs (ctx.task);
756 const waivers = require ( './link-waivers' ). loadLinkWaivers (ctx.projectDir, ctx.siteId);
757 // An id the payload carries without the gate (read from a crosswalk) that is exactly one ready
758 // producer record's target counts as that record; it gets the same per-key checks as resolveReference.
759 const byTarget = new Map ();
760 const lookup = ( ref , targetId ) => {
761 const dep = dependencies. find ( row => row.ref === ref), producerTask = dep?.producer.kind === 'import-task' && ctx.manifest.tasks. find ( task => task.id === dep.producer.taskId);
762 if ( ! producerTask || typeof targetId !== 'string' ) return null ;
763 const snapshot = snapshots (ctx, producerTask);
764 if ( ! byTarget. has (ref)) { const found = new Map (); for ( const row of index.crosswalk. values ()) if (snapshot.sourceByKey. get (row.sourceStableKey)?.ref === ref) found. set (row.targetId, [ ... (found. get (row.targetId) || []), row.sourceStableKey]); byTarget. set (ref, found); }
765 const keys = byTarget. get (ref). get (targetId) || [];
766 if (keys. length !== 1 || ! producedThisRun (ctx, index, keys[ 0 ], dep)) return null ;
767 const ready = keyReady (ctx, index, keys[ 0 ], { task: producerTask, snapshot, ref });
768 return ready.ready && ready.row.targetId === targetId ? keys[ 0 ] : null ;
769 };
770 for ( const record of records) {
771 if ( ! record.sourceStableKey) throw new Error ( 'DEPENDENCY_SOURCE_KEY_REQUIRED' );
772 const references = [], omissions = [], holds = [], finals = [], orphans = [], customLineOmissions = []; let blocker;
773 const finalOrWaived = ( ref , key , concrete , dep , parentStatus , extra = {}) => {
774 const waiver = waivers. match ({ taskId: ctx.taskId, sourceStableKey: record.sourceStableKey, ref, dependencySourceKey: key, parentStatus });
775 if (waiver && ! linkRequired (ctx.task.entityRefs, ref) && (waiver.explicitRequest === true || linkChoice (ctx.task.entityRefs, ref) === 'offer-omit' )) {
776 references. push ({ ref, sourceStableKey: key, path: concrete, action: 'omit-failed' , dependencyStatus: 'waived' , parentStatus, waiverId: waiver.id }); return true ;
777 }
778 finals. push ({ ref, sourceStableKey: key, dependencyState: dep.state, parentStatus, linkChoice: linkChoice (ctx.task.entityRefs, ref), ... ( linkRequired (ctx.task.entityRefs, ref) ? { linkRequired: true } : {}), producerTaskId: dep.producer.taskId || null , ... extra });
779 return false ;
780 };
781 const parentFinal = ( key , dep , inSource ) => {
782 if ( ! [ 'import-task' , 'child-output' , 'secondary-output' ]. includes (dep.producer.kind) || ! [ 'finished-partial' , 'complete' , 'complete-with-holds' ]. includes (dep.state)) return null ;
783 const status = failedForGood (ctx, index, key, dep, inSource);
784 if (status === 'deferred' && index.attempts. get (key)?.deferral?.final === true ) return 'parent-final' ;
785 return status && ! PARENT_NOT_FINAL . has (status) ? status : null ;
786 };
787 const resolve = ( ref , key , concrete , mode ) => {
788 if (mode === 'same-task' ) {
789 if ( ! ownFields. some ( field => field.ref === ref && pathMatches (field.template, concrete))) throw new Error ( 'REFERENCE_UNDECLARED' );
790 const target = keyReady (ctx, ownIndex, key, { task: ctx.task, snapshot: snapshots (ctx, ctx.task), ref });
791 if (target.ready) { references. push ({ ref, sourceStableKey: key, path: concrete, targetId: target.row.targetId, producerKind: 'same-task' }); return target.row.targetId; }
792 blocker = { ref, sourceStableKey: key, dependencyState: 'same-task-parent-not-written' }; return undefined ;
793 }
794 const fromSetup = mode === 'setup' ;
795 const dep = dependencies. find ( dep => dep.ref === ref);
796 // A proof says this record writes no such reference; an inherited ref is never written directly.
797 if (dep?.producer.kind === 'no-producer' ) throw new Error ( 'dependency-proof-contradicted' );
798 if (dep && [ 'inherited' , 'pending-setup' ]. includes (dep.producer.kind)) throw new Error ( 'REFERENCE_UNDECLARED' );
799 // A child identity (a variant) is only ever resolved from its parent's verified receipt.
800 if (dep && domain (ref).childOf && ! [ 'child-output' , 'disposition' ]. includes (dep.producer.kind)) throw new Error ( 'REFERENCE_UNDECLARED' );
801 // Code written for a task-complete edge may still call resolveTaskReference after the edge
802 // became per-record: the key is then checked on its own, which is never weaker.
803 const taskOnRecord = mode === 'task' && dep?.mode === 'per-record' && dep.producer.kind === 'import-task' ;
804 if ( ! dep || (fromSetup ? dep.mode !== 'task-complete' || dep.producer.kind !== 'setup-step' : mode === 'task' && ! taskOnRecord ? dep.mode !== 'task-complete' || dep.producer.kind !== 'import-task' || ! [ 'complete' , 'complete-with-holds' , 'finished-partial' , 'empty' ]. includes (dep.state) : dep.mode !== 'per-record' || dep.producer.kind === 'setup-step' )) throw new Error ( 'REFERENCE_UNDECLARED' );
805 if ( ! fromSetup && mode !== 'task' && ! ctx.task.entityRefs. some ( entityRef => ( domain (entityRef).dependencyReferenceFields?.[ref] || []). some ( template => pathMatches (template, concrete)))) throw new Error ( 'REFERENCE_UNDECLARED' );
806 if (fromSetup) {
807 const entries = dep.receipt.requirements. filter ( req => dep.producer.stepIds. includes (req.id)). flatMap ( req => req.referenceBindings || []);
808 const match = entries. find ( binding => binding.ref === ref && binding.sourceStableKey === key && Object. keys (binding.values). some ( template => pathMatches (template, concrete)));
809 const template = match && Object. keys (match.values). find ( template => pathMatches (template, concrete));
810 if ( ! match || ! template) { blocker = { ref, sourceStableKey: key, dependencyState: dep.state }; return undefined ; }
811 const value = match.values[template]; references. push ({ ref, sourceStableKey: key, path: concrete, targetId: value, producerKind: 'setup-step' }); return value;
812 }
813 if (dep.state === 'empty' ) { blocker = { ref, sourceStableKey: key, dependencyState: 'unknown' }; return undefined ; }
814 if (dep.producer.kind === 'secondary-output' ) {
815 // The parent (member) row must be ready for this key, and carry exactly one verified
816 // secondary identity for this ref on this site. Nothing is inferred or recreated.
817 const parentTask = ctx.manifest.tasks. find ( task => task.id === dep.producer.taskId), parentSnapshot = parentTask && snapshots (ctx, parentTask);
818 const parentSource = parentSnapshot?.sourceByKey. get (key);
819 const parent = parentSource?.ref === dep.producer.parentRef && producedThisRun (ctx, index, key, dep) ? keyReady (ctx, index, key, { task: parentTask, snapshot: parentSnapshot, ref: dep.producer.parentRef }) : { ready: false };
820 // The verification kind is whatever the parent's reviewed operation contract declares.
821 const verification = proofs. operationContract (dep.producer.parentRef, parentTask?.writeOperations?.[dep.producer.parentRef])?.contract.secondaryOutputs?.[ref]?.verification;
822 const identity = parent.ready ? local. verifiedSecondaryIdentity (parent.row, ref, ctx.siteId, verification) : null ;
823 // A parent (member) row that can no longer land in this run is a final cause, like any
824 // other producer's: an owner hold, or failed / not attempted / absent after a finished pass.
825 // A landed parent whose identity is unverified is not final; it waits for its read-back.
826 if ( ! identity && ! parent.ready) {
827 const inSource = parentSource?.ref === dep.producer.parentRef, held = inSource && dep.held?. has (key);
828 const status = held ? 'held' : parentFinal (key, dep, inSource);
829 if (status) {
830 if ( finalOrWaived (ref, key, concrete, dep, status, { ... (held ? { holdReasons: dep.held. get (key) } : {}), path: concrete })) return undefined ;
831 // An owner hold is also a hold cause, as for any other producer: the hold decision is surfaced.
832 if (held) holds. push ({ ref, sourceStableKey: key, dependencyState: dep.state, holdReasons: dep.held. get (key) });
833 return undefined ;
834 }
835 }
836 if ( ! identity) { blocker = { ref, sourceStableKey: key, dependencyState: parent.ready ? 'secondary-identity-unverified' : dep.state }; return undefined ; }
837 // Keep the owning parent: a payload that also references the parent ref must name this one.
838 references. push ({ ref, sourceStableKey: key, path: concrete, targetId: identity.targetId, producerKind: 'secondary-output' , parentRef: dep.producer.parentRef, parentTargetId: parent.row.targetId }); return identity.targetId;
839 }
840 const producerTask = ctx.manifest.tasks. find ( task => task.id === dep.producer.taskId);
841 const snapshot = producerTask && snapshots (ctx, producerTask);
842 let source, childState, childParent;
843 if (dep.producer.kind === 'child-output' ) {
844 // A variant: the one ready parent row whose verified receipt names this source key.
845 const child = childIdentity (ctx, index, receipts.children. get (ref), dep.producer, key, dep);
846 if (child.targetId) { references. push ({ ref, sourceStableKey: key, path: concrete, targetId: child.targetId, producerKind: 'child-output' , parentRef: dep.producer.parentRef, parentSourceKey: child.parentKey, parentTargetId: child.parentTargetId }); return child.targetId; }
847 childState = child.state; childParent = child;
848 } else {
849 source = snapshot?.sourceByKey. get (key);
850 const target = source?.ref === ref && producedThisRun (ctx, index, key, dep) ? keyReady (ctx, index, key, { task: producerTask, snapshot, ref }) : { ready: false };
851 if (target.ready) {
852 // The row usually holds a digest reference; the receipt itself comes from protected state.
853 if (receipts.parents. has (ref) && ! require ( './write-verification' ). validReceipt (receipts.parents. get (ref).receipts. get (key), { siteId: ctx.siteId, sourceKey: key, targetId: target.row.targetId })) {
854 blocker = { ref, sourceStableKey: key, dependencyState: 'imported-unverified' }; return undefined ;
855 }
856 references. push ({ ref, sourceStableKey: key, path: concrete, targetId: target.row.targetId }); return target.row.targetId;
857 }
858 // A dependency the record can go without (a product image) never makes it wait. The record is
859 // written without it and the pair is recorded: with the reason the key cannot land in this run,
860 // or `pending` / `held` while it still may. Once the key lands, omittedReady lists the record
861 // and its verified media patch clears the row (recordOmissionsResolved).
862 if (dep.mode === 'per-record' && dep.producer.kind === 'import-task' && failureOmits (ctx.task.entityRefs, ref, concrete)) {
863 const status = source?.ref === ref && dep.held?. has (key) ? 'held' : failedForGood (ctx, index, key, dep, source?.ref === ref) || 'pending' ;
864 const omission = { ref, sourceStableKey: key, path: concrete, action: 'omit-failed' , dependencyStatus: status };
865 // An order's product dependency is omittable only after its builder has deliberately
866 // converted that line to a custom item. Ordinary catalog lines must keep waiting.
867 if (ref === 'stores/product' && ctx.task.entityRefs. includes ( 'ecom/order' )) {
868 customLineOmissions. push ({ omission, blocker: { ref, sourceStableKey: key, dependencyState: dep.state } });
869 return undefined ;
870 }
871 references. push (omission); return undefined ;
872 }
873 // The record names a record the owner held: it waits, with that hold as its cause.
874 // Kept apart from `blocker` so a later unresolved reference never erases the hold cause.
875 if (source?.ref === ref && dep.held?. has (key)) {
876 if ( finalOrWaived (ref, key, concrete, dep, 'held' , { holdReasons: dep.held. get (key), path: concrete })) return undefined ;
877 holds. push ({ ref, sourceStableKey: key, dependencyState: dep.state, holdReasons: dep.held. get (key) }); return undefined ;
878 }
879 }
880 const sourceTask = ctx.manifest.tasks. find ( task => task.id === dep.producer.taskId);
881 const sourceLinks = [ ... (sourceTask?.sourceBindings || []), ... (ctx.task.dependencySourceBindings || [])];
882 const matches = sourceLinks. filter ( source => source.ref === ref && typeof source.keyPrefix === 'string' && key. startsWith (source.keyPrefix));
883 const censusId = (source?.ref === ref && snapshot.censusByKey. get (key)) || (matches. length === 1 ? matches[ 0 ].censusId || `entity:${ matches [ 0 ]. sourceEntity }` : null );
884 const rule = adapter.rules. find ( rule => rule.dependencyRef === ref && ctx.task.entityRefs. includes (rule.dependentRef) && rule.censusId === censusId && adapter.rows. get (censusId)?.state === 'excluded' && adapter.approved. has (censusId) && pathMatches (rule.path, concrete));
885 if (rule?.action === 'omit' ) { omissions. push (concrete); references. push ({ ref, sourceStableKey: key, path: concrete, action: 'omit' }); return undefined ; }
886 if ( ! rule && ! dep.state. startsWith ( 'excluded' ) && adapter.rows. get (censusId)?.state !== 'excluded' ) {
887 if (dep.producer.kind === 'child-output' ) {
888 const settled = [ 'finished-partial' , 'complete' , 'complete-with-holds' ]. includes (dep.state);
889 const parentKey = childParent?.parentKey, parentHeld = parentKey && dep.held?. has (parentKey);
890 if (settled && ! parentKey && childState === 'child-identity-missing' ) { orphans. push ({ ref, key, concrete, dep }); return undefined ; }
891 const status = ! settled || ! parentKey ? null : parentHeld ? 'held' : parentFinal (parentKey, dep, true );
892 if (status) { finalOrWaived (ref, key, concrete, dep, status, { ... (parentKey ? { parentSourceKey: parentKey } : {}), ... (parentHeld ? { holdReasons: dep.held. get (parentKey) } : {}), path: concrete }); return undefined ; }
893 } else {
894 const status = parentFinal (key, dep, source?.ref === ref);
895 if (status) { finalOrWaived (ref, key, concrete, dep, status, { path: concrete }); return undefined ; }
896 }
897 }
898 blocker = { reason: rule?.action === 'defer' ? 'dependency-not-ready' : (dep.state. startsWith ( 'excluded' ) || adapter.rows. get (censusId)?.state === 'excluded' ) ? 'dependency-disposition-rule-missing' : 'dependency-not-ready' , ref, sourceStableKey: key, dependencyState: childState || dep.state };
899 return undefined ;
900 };
901 try {
902 const payload = record.transform ? await record. transform ({ resolveReference : ( ref , key , concrete ) => resolve (ref, key, concrete, 'record' ), resolveTaskReference : ( ref , key , concrete ) => resolve (ref, key, concrete, 'task' ), resolveSetupReference : ( ref , key , concrete ) => resolve (ref, key, concrete, 'setup' ), resolveSameTaskReference : ( ref , key , concrete ) => resolve (ref, key, concrete, 'same-task' ) }) : record.payload;
903 for ( const candidate of customLineOmissions) {
904 if ( isCustomOrderLine (payload, candidate.omission.path)) references. push (candidate.omission);
905 else blocker ||= candidate.blocker;
906 }
907 for ( const orphan of orphans) {
908 const entry = orphan.concrete. slice ( 0 , orphan.concrete. indexOf ( ']' ) + 1 ), parentRef = orphan.dep.producer.parentRef;
909 const parent = entry && finals. find ( row => row.ref === parentRef && row.path?. startsWith (entry))
910 || entry && references. find ( row => row.ref === parentRef && row.action === 'omit-failed' && row.path?. startsWith (entry));
911 finalOrWaived (orphan.ref, orphan.key, orphan.concrete, orphan.dep, parent?.parentStatus || 'child-identity-missing' , { ... (parent ? { parentSourceKey: parent.sourceStableKey } : {}), ... (parent?.holdReasons ? { holdReasons: parent.holdReasons } : {}), path: orphan.concrete });
912 }
913 // A decided builder hold is never sent and is not a reference gap: it is already reviewed in
914 // the read-back inventory. It defers here with its reasons, like any other deferral.
915 const held = require ( './read-back-contract' ). builderHold (payload);
916 if (held) { defer. push ({ ... record, deferral: { reason: 'builder-held' , holds: held } }); continue ; }
917 require ( './mapping-recovery' ). assertPayload (ctx.projectDir, assessments. current ()?.plan || {}, ctx.task.entityRefs, payload, {sourceStableKey:record.sourceStableKey});
918 if (payload?.deferred && typeof payload.reason === 'string' ) { blocker = { reason: payload.reason, dependencyState: 'unknown' }; await persistReferenceGap (ctx, record.sourceStableKey, payload.reason); }
919 for ( const concrete of omissions) if ( valueAt (payload, concrete) !== undefined ) throw new Error ( 'dependency-excluded-reference-present' );
920 // Guarded paths come from the current reviewed operation contract: a payload that writes one
921 // contradicts the proof (including a source catch-up record that now references the ref).
922 for ( const dep of dependencies. filter ( dep => dep.producer.kind === 'no-producer' )) for ( const row of dep.producer.proofs) {
923 const guards = proofs. operationContract (row.entityRef, row.operation)?.contract.dependencies?.[dep.ref]?.guardedPaths;
924 if ( ! guards?. length || proofs. guardHits (payload, guards). length ) throw new Error ( 'dependency-proof-contradicted' );
925 }
926 // A ref the task's operation does not depend on (declared inapplicable) has no reference template here.
927 for ( const targetRef of ctx.task.entityRefs) for ( const [ ref , templates ] of Object. entries ( domain (targetRef).dependencyReferenceFields || {})) if (checked. includes (ref)) for ( const template of templates) {
928 const unresolved = expandPaths (payload, template). filter ( concrete => valueAt (payload, concrete) !== undefined && ! references. some ( reference => reference.path === concrete && ! reference.action && declaresPath (ctx.task.entityRefs, reference.ref, concrete)));
929 if (unresolved. length && ! dropUnchecked (payload, references, ctx.task.entityRefs, ref, template, unresolved, lookup)) throw new Error ( 'dependency-reference-unresolved' );
930 }
931 // A same-task parent id comes only from resolveSameTaskReference, never from local evidence.
932 for ( const field of ownFields) for ( const concrete of expandPaths (payload, field.template)) if ( valueAt (payload, concrete) !== undefined && ! references. some ( reference => reference.producerKind === 'same-task' && reference.path === concrete)) throw new Error ( 'dependency-reference-unresolved' );
933 // A reference resolved but not written (a buyer link dropped after corroboration) sends nothing.
934 const used = references. filter ( reference => reference.action || valueAt (payload, reference.path) !== undefined );
935 for ( const ref of used. filter ( ref => ! ref.action)) if ( valueAt (payload, ref.path) !== ref.targetId) throw new Error ( 'dependency-reference-payload-mismatch' );
936 if ( identityPairMismatch (payload, used, ctx.task.entityRefs)) throw new Error ( 'dependency-identity-pair-mismatch' );
937 const heldFinal = holds. filter ( hold => ! finals. some ( row => row.ref === hold.ref && row.sourceStableKey === hold.sourceStableKey));
938 if (blocker || heldFinal. length ) defer. push ({ ... record, deferral: { reason: blocker ? 'dependency-not-ready' : 'dependency-held' , ... (blocker || heldFinal[ 0 ]), ... (holds. length ? { holdCauses: holds } : {}) } });
939 else if (finals. length ) defer. push ({ ... record, deferral: { reason: 'dependency-parent-final' , final: true , ... finals[ 0 ], finalCauses: finals, ... (holds. length ? { holdCauses: holds } : {}) } });
940 else { allowed. add (record.sourceStableKey); send. push ({ ... record, payload, references: used }); }
941 } catch (error) { defer. push ({ ... record, deferral: { ... (holds. length ? { holdCauses: holds } : {}), reason: [ 'REFERENCE_UNDECLARED' , 'dependency-excluded-reference-present' , 'dependency-reference-payload-mismatch' , 'dependency-reference-unresolved' , 'dependency-proof-contradicted' , 'dependency-identity-pair-mismatch' ]. includes (error.message) ? error.message : 'dependency-transform-failed' } }); }
942 }
943 for ( const record of defer) await local. appendAttempt (ctx.projectDir, { schemaVersion: 1 , attemptId: crypto. randomUUID (), sourceStableKey: record.sourceStableKey, writeSpecId: ctx.taskId, operation: 'create' , targetEntityType: ctx.task.entityRefs[ 0 ], status: 'deferred' , runId: ctx.runId, phase: 'import' , ts: new Date (). toISOString (), deferral: record.deferral });
944 const omitted = send. flatMap ( record => record.references. filter ( reference => reference.action === 'omit-failed' ). map ( reference => ({ ref: reference.ref, dependencySourceKey: reference.sourceStableKey ?? `unchecked:${ reference . uncheckedTargetId }` , dependencyStatus: reference.dependencyStatus, sourceStableKey: record.sourceStableKey })));
945 await updateDiagnostic (ctx.projectDir, async projection => {
946 if ( ! projection.tasks[ctx.taskId]) return false ;
947 projection.tasks[ctx.taskId].waiting = projectWaiting (ctx, await evidenceIndex (ctx));
948 projection.tasks[ctx.taskId].heldDependencies = heldDependencies (projection.tasks[ctx.taskId].waiting);
949 const finalRows = finalDeferrals (projection.tasks[ctx.taskId].waiting);
950 if (finalRows. length ) projection.tasks[ctx.taskId].finalDeferrals = finalRows; else delete projection.tasks[ctx.taskId].finalDeferrals;
951 // The record's own landed state, not a dependency key check (kept out of the scan counters).
952 const current = await evidenceIndex (ctx), landed = key => { const attempt = current.attempts. get (key), row = current.crosswalk. get (key); return attempt?.siteId === ctx.siteId && row?.siteId === ctx.siteId && row.status === 'imported' && [ 'imported' , 'skipped_already_imported' ]. includes (attempt.status); };
953 const rows = mergeOmitted (projection.tasks[ctx.taskId].omittedReferences, send. map ( record => record.sourceStableKey). filter ( key => ! landed (key)), omitted, ctx);
954 if (rows. length ) projection.tasks[ctx.taskId].omittedReferences = rows; else delete projection.tasks[ctx.taskId].omittedReferences;
955 });
956 const payloads = new Map (send. map ( record => [record.sourceStableKey, record.references. map ( reference => Object. freeze ({ ... reference }))]));
957 const grant = Object. freeze ({ taskId: ctx.taskId, runId: ctx.runId, siteId: ctx.siteId });
958 const proofGuards = dependencies. filter ( dep => dep.producer.kind === 'no-producer' ). flatMap ( dep => dep.producer.proofs. flatMap ( row => proofs. operationContract (row.entityRef, row.operation)?.contract.dependencies?.[dep.ref]?.guardedPaths || []));
959 grants. set (grant, { ctx, index, allowed, payloads, proofGuards, stamp: state.stamp, taskBinding: state.taskBinding, generation: state.generation, crosswalkPublishedOffset: (index.crosswalkPublished || []). length });
960 return { send, defer, grant };
961 }
962 // One row per dependent record and dependency key it was sent without. A record admitted again
963 // before it landed replaces its rows (its image may have landed since); a landed record keeps them,
964 // since the live record still lacks what it was written without. The completion report turns rows whose record
965 // landed into cutover items.
966 function mergeOmitted ( existing , admittedKeys , rows , ctx ) {
967 const admitted = new Set (admittedKeys);
968 const byPair = new Map ((existing || []). filter ( row => ! admitted. has (row.sourceStableKey)). map ( row => [row.sourceStableKey + ' \n ' + row.dependencySourceKey, row]));
969 // A writer that reports this omission as its own cutover item (every entity of the task declares it).
970 const ownItem = ref => { const codes = ctx.task.entityRefs. map ( entityRef => domain (entityRef).dependencyFailureCutoverItems?.[ref]); return codes. every ( code => code && code === codes[ 0 ]) ? codes[ 0 ] : null ; };
971 for ( const row of rows) byPair. set (row.sourceStableKey + ' \n ' + row.dependencySourceKey, { ... row, ... ( ownItem (row.ref) ? { cutoverItem: ownItem (row.ref) } : {}), siteId: ctx.siteId, runId: ctx.runId, recordedAt: new Date (). toISOString (),
972 operatorRole: 'Requires Import Operator' , action: row.dependencyStatus === 'waived'
973 ? `${ row . sourceStableKey } was imported without its ${ row . ref } link (${ row . dependencySourceKey }) by an operator waiver, because that record did not land. If the ${ row . ref } record is fixed later, add the link to ${ row . sourceStableKey } with a reviewed update; otherwise this loss is accepted. Do not recreate the record.`
974 : row.dependencyStatus === 'unchecked-reference'
975 ? `The importer built ${ row . sourceStableKey } with a ${ row . ref } id the dependency gate never resolved (${ row . dependencySourceKey . slice ( 'unchecked:' . length ) }), so it was left out. Find the source ${ row . ref } record that id was meant to name, import it through its task if it is missing, then add it to ${ row . sourceStableKey } with a reviewed update. Do not recreate the record.`
976 : `Re-import ${ row . ref } ${ row . dependencySourceKey } (run-import:${ ctx . task . dependencyEdges . find ( edge => edge . ref === row . ref )?. producer . taskId || row . ref } --failed-only) if it has not landed, then run the media patch pass, which adds it to ${ row . sourceStableKey } through a verified update and clears this row. Do not recreate the record.` });
977 return [ ... byPair. values ()]. sort (( a , b ) => (a.sourceStableKey + a.dependencySourceKey). localeCompare (b.sourceStableKey + b.dependencySourceKey));
978 }
979 // A secondary identity belongs to one parent (a contact to the member whose write created or
980 // matched it). When the same payload also carries a reference to that parent ref, both must name
981 // the same parent; two individually verified IDs for different people are refused.
982 // A child identity belongs to one parent too (a variant to its product), but per array entry: the
983 // parent reference in the same entry (the same order line's catalogItemId) must be present and name
984 // that parent.
985 function siblingPath ( template , concrete ) {
986 const parts = template. split ( '.' ), given = concrete. split ( '.' ), out = [];
987 for ( const [ i , part ] of parts. entries ()) {
988 if ( ! part. endsWith ( '[]' )) { out. push (part); continue ; }
989 const match = given[i] && / ^ ( \w + ) \[ ( \d + ) \] $ / . exec (given[i]);
990 if ( ! match || match[ 1 ] !== part. slice ( 0 , - 2 ) || out. some (( seg , j ) => seg !== given[j])) return null ;
991 out. push (given[i]);
992 }
993 return out. join ( '.' );
994 }
995 function identityPairMismatch ( body , references , entityRefs , { partial = false } = {}) {
996 for ( const reference of references. filter ( row => row.producerKind === 'child-output' )) {
997 if (partial && ! Object. hasOwn (body || {}, reference.path. split ( / [. \[ ] / )[ 0 ])) continue ;
998 const paths = entityRefs. flatMap ( entityRef => ( domain (entityRef).dependencyReferenceFields?.[reference.parentRef] || []). map ( template => siblingPath (template, reference.path))). filter (Boolean);
999 if ( ! paths. length || paths. some ( sibling => valueAt (body, sibling) !== reference.parentTargetId)) return true ;
1000 }
1001 for ( const reference of references. filter ( row => row.producerKind === 'secondary-output' )) {
1002 for ( const entityRef of entityRefs) for ( const template of domain (entityRef).dependencyReferenceFields?.[reference.parentRef] || []) {
1003 if (partial && ! Object. hasOwn (body || {}, template. split ( / [. \[ ] / )[ 0 ])) continue ;
1004 if (proofs. valuesAt (body, template). some ( value => value !== undefined && value !== null && value !== '' && value !== reference.parentTargetId)) return true ;
1005 }
1006 }
1007 return false ;
1008 }
1009 // Bulk shapes whose items each belong to one source record. An unknown shape is refused, never
1010 // checked as one body: the per-item key list must line up with the items the request writes.
1011 const BULK_ITEMS = { 'crm/contact' : body => body?.contacts, 'seo/item-seo-tags' : body => body?.entries, 'stores/review' : body => body?.reviews };
1012 const bulkItems = ( entityRefs , body ) => { const shapes = entityRefs. map ( ref => BULK_ITEMS [ref]). filter (Boolean); return shapes. length === 1 ? shapes[ 0 ](body) : undefined ; };
1013 // A record client writes one product through the bulk create (importProducts sends one row per
1014 // record). That body is `{ products: [product] }`: the record's own product is its one item, and
1015 // every check reads that item. Reading the wrapper found no tag at `tags.publicTags.tagIds[0]` and
1016 // refused every tagged product.
1017 const PRODUCT_BULK_CREATE = / ^ \/ stores \/ v3 \/ bulk \/ products(?:-with-inventory) ? \/ create $ / ;
1018 function singleBulkProduct ( request ) {
1019 if ( typeof request?.url !== 'string' || ! Array. isArray (request.body?.products) || request.body.products. length !== 1 ) return undefined ;
1020 try { return PRODUCT_BULK_CREATE . test ( new URL (request.url).pathname) ? request.body.products[ 0 ] : undefined ; } catch { return undefined ; }
1021 }
1022 function verifyGrant ( grant , clientContext , sourceStableKey , request , { auxiliary = false , partial = false } = {}) {
1023 const stored = grants. get (grant);
1024 // A bulk body carries many records; only verifyBulkGrant may admit it, one key per item.
1025 if (stored && request && bulkItems (stored.ctx.task.entityRefs, request.body) !== undefined ) return false ;
1026 const bulkProduct = stored && request && stored.ctx.task.entityRefs. includes ( 'stores/product' ) ? singleBulkProduct (request) : undefined ;
1027 if (stored && request) {
1028 try { require ( './mapping-recovery' ). assertPayload (stored.ctx.projectDir, assessments. current ()?.plan || {}, stored.ctx.task.entityRefs, bulkProduct || request.body, {sourceStableKey}); } catch { return false ; }
1029 }
1030 if (stored && request && ! auxiliary) {
1031 const body = bulkProduct || request.body?.order || request.body?.product || request.body?.member || request.body?.contact || request.body?.specification || request.body?.discountRule || request.body?.category || request.body;
1032 // A partial write (a product PATCH) is checked on the top-level fields it actually sends.
1033 const guards = partial ? stored.proofGuards. filter ( guard => Object. hasOwn (body || {}, proofs. guardPath (guard). split ( / [. \[ ] / )[ 0 ])) : stored.proofGuards;
1034 if (proofs. guardHits (body, guards). length ) return false ;
1035 if ( identityPairMismatch (body, stored.payloads. get (sourceStableKey) || [], stored.ctx.task.entityRefs, { partial })) return false ;
1036 for ( const reference of stored.payloads. get (sourceStableKey) || []) { if (partial && ! Object. hasOwn (body || {}, reference.path. split ( / [. \[ ] / )[ 0 ])) continue ; if (reference.action === 'omit-failed' ) continue ; const value = valueAt (body, reference.path); if (reference.action === 'omit' ? value !== undefined : value !== reference.targetId) return false ; }
1037 // Every reference the outgoing body carries must be one the gate resolved at that exact path:
1038 // a line added or reordered after gating cannot carry an unchecked ID.
1039 const granted = stored.payloads. get (sourceStableKey) || [];
1040 for ( const targetRef of stored.ctx.task.entityRefs) for ( const [ ref , templates ] of Object. entries ( domain (targetRef).dependencyReferenceFields || {})) if ( checkedRefs (stored.ctx.task). includes (ref)) for ( const template of templates) {
1041 if (partial && ! Object. hasOwn (body || {}, template. split ( / [. \[ ] / )[ 0 ])) continue ;
1042 for ( const concrete of expandPaths (body, template)) if ( valueAt (body, concrete) !== undefined && ! granted. some ( row => row.path === concrete && ! row.action && declaresPath (stored.ctx.task.entityRefs, row.ref, concrete))) return false ;
1043 }
1044 for ( const field of sameTaskFields (stored.ctx.task.entityRefs)) {
1045 if (partial && ! Object. hasOwn (body || {}, field.template. split ( / [. \[ ] / )[ 0 ])) continue ;
1046 for ( const concrete of expandPaths (body, field.template)) if ( valueAt (body, concrete) !== undefined && ! granted. some ( row => row.producerKind === 'same-task' && row.path === concrete)) return false ;
1047 }
1048 }
1049 return Boolean (stored && stored.ctx.taskId === clientContext.taskId && stored.ctx.runId === clientContext.runId && stored.ctx.siteId === clientContext.siteId && stored.allowed. has (sourceStableKey) && evaluationGenerations. get (stored.taskBinding) === stored.generation && stored.stamp === fingerprint (stored.ctx));
1050 }
1051 function verifyBulkGrant ( grant , clientContext , sourceStableKeys , request , options = {}) {
1052 if ( ! request || request.sourceStableKey !== undefined || ! Array. isArray (sourceStableKeys) || ! sourceStableKeys. length ) return false ;
1053 const items = bulkItems (grants. get (grant)?.ctx.task.entityRefs || [], request.body);
1054 if ( ! Array. isArray (items) || items. length !== sourceStableKeys. length ) return false ;
1055 // Index loops, not some/every: those skip sparse holes, and a hole would send an unchecked item.
1056 const seen = new Set ();
1057 for ( let i = 0 ; i < sourceStableKeys. length ; i ++ ) {
1058 const key = sourceStableKeys[i];
1059 if ( ! Object. hasOwn (sourceStableKeys, i) || ! Object. hasOwn (items, i) || typeof key !== 'string' || ! key || seen. has (key)) return false ;
1060 seen. add (key);
1061 if ( ! items[i] || typeof items[i] !== 'object' || ! verifyGrant (grant, clientContext, key, { ... request, body: items[i] }, options)) return false ;
1062 }
1063 return true ;
1064 }
1065 function verifyReferenceTargets ( grant , ctx , key , ref , ids ) {
1066 const stored = grants. get (grant);
1067 if ( ! verifyGrant (grant, ctx, key) || ! Array. isArray (ids)) return false ;
1068 const expected = new Set ((stored.payloads. get (key) || []). filter ( row => row.ref === ref && ! row.action). map ( row => row.targetId));
1069 return ids. length === expected.size && new Set (ids).size === ids. length && ids. every ( id => expected. has (id));
1070 }
1071 // One reference id the record was granted for `ref` (any of them, unlike verifyReferenceTargets).
1072 function verifyReferenceMember ( grant , ctx , key , ref , id ) {
1073 const stored = grants. get (grant);
1074 if ( ! verifyGrant (grant, ctx, key) || typeof id !== 'string' || ! id) return false ;
1075 return (stored.payloads. get (key) || []). some ( row => row.ref === ref && ! row.action && row.targetId === id);
1076 }
1077 function currentRecordTargetId ( grant , clientContext , sourceStableKey ) {
1078 const stored = grants. get (grant);
1079 if ( ! verifyGrant (grant, clientContext, sourceStableKey)) return null ;
1080 const current = keyReady (stored.ctx, stored.index, sourceStableKey, { task: stored.ctx.task, snapshot: snapshots (stored.ctx, stored.ctx.task) });
1081 if (current.ready) return current.row.targetId;
1082 // LIVE-FOUND: a record whose existing target this grant's own write reconciled and verified (a
1083 // product an earlier run created) is not ready in the index yet -- the journal is re-read only at
1084 // the next evaluation, so its latest attempt there is still the earlier one -- and its follow-up
1085 // writes (a media patch) were refused. Its crosswalk row is admitted only when the crosswalk
1086 // listener saw it published after this grant (crosswalk updates only: an attempt row never
1087 // counts), for this site and key, and carries a write-verification reference to the same target;
1088 // updateCrosswalkRows checked that receipt before publishing the row. Product tasks only: their
1089 // readers (recordProductMatches for media/tags/membership writes) are the ones this fixes; a review
1090 // or booking task's readers (recordOwnReview, recordSawTarget) keep the index-ready rule.
1091 if ( ! stored.ctx.task.entityRefs. includes ( 'stores/product' )) return null ;
1092 const row = stored.index.crosswalk?. get (sourceStableKey), ref = row?.verification;
1093 const published = Number. isInteger (stored.crosswalkPublishedOffset) && (stored.index.crosswalkPublished || []). slice (stored.crosswalkPublishedOffset). includes (sourceStableKey);
1094 if ( ! published || ! row || row.status !== 'imported' || row.siteId !== stored.ctx.siteId) return null ;
1095 const source = snapshots (stored.ctx, stored.ctx.task).sourceByKey. get (sourceStableKey);
1096 return source?.ref === 'stores/product' && rowMatchesSource (row, source) && ref?.storage === 'write-verification' && ref.kind === 'product'
1097 && ref.sourceKey === sourceStableKey && ref.targetId === row.targetId && ref.siteId === stored.ctx.siteId ? row.targetId : null ;
1098 }
1099 // A within-write dependency the containing write initiated but could not yet verify
1100 // (late or dropped product media) stays visible against the landed parent's target ID, in the
1101 // diagnostic completion and handoff already read. It never resends the parent. Both recording and
1102 // clearing take the writer's registered observation: recording needs an unresolved/unverifiable one
1103 // for this target, site and ref; clearing needs a later `verified` one for the same target, site,
1104 // ref and intended items. Anything else keeps the gap and its operator action.
1105 async function recordWithinWriteGap ({ grant , taskBinding , sourceStableKey , ref , targetId , code , resolved = false , observation = null }) {
1106 // The grant that admitted this record's write identifies it (task, run, site, admitted key). A later
1107 // evaluation of the task -- the next batch, while this product's observation was still running --
1108 // must not make the outcome of an already-sent write unrecordable, so generation is not required.
1109 const stored = grants. get (grant), bound = capability. requireKind (taskBinding, 'import-task' );
1110 if ( ! stored || stored.ctx.taskId !== bound.taskId || stored.ctx.runId !== bound.runId || stored.ctx.siteId !== bound.siteId || ! stored.allowed. has (sourceStableKey)) throw new Error ( 'DEPENDENCY_READINESS_REQUIRED' );
1111 const edge = (stored.ctx.task.dependencyEdges || []). find ( edge => edge.ref === ref);
1112 if ( ! edge || edge.producer.kind !== 'no-producer' || ! edge.producer.proofs. some ( row => row.reason === 'within-write' )) throw new Error ( 'WITHIN_WRITE_EDGE_REQUIRED' );
1113 if ( typeof targetId !== 'string' || ! targetId || typeof code !== 'string' || ! / ^ [a-z0-9-] +$ / . test (code)) throw new Error ( 'WITHIN_WRITE_GAP_INVALID' );
1114 const evidence = require ( '../resources/rp-target-wix/lib/wix-writers' ). withinWriteObservation (observation);
1115 if ( ! evidence || evidence.ref !== ref || evidence.productId !== targetId || evidence.siteId !== stored.ctx.siteId) throw new Error ( 'WITHIN_WRITE_EVIDENCE_INVALID' );
1116 if ( ! resolved && ! [ 'unresolved' , 'unverifiable' ]. includes (evidence.status)) throw new Error ( 'WITHIN_WRITE_EVIDENCE_INVALID' );
1117 let result;
1118 await updateDiagnostic (stored.ctx.projectDir, diagnostic => {
1119 const task = diagnostic.tasks[stored.ctx.taskId] || (diagnostic.tasks[stored.ctx.taskId] = { schemaVersion: 1 , siteId: stored.ctx.siteId, taskId: stored.ctx.taskId, blocked: [], waiting: [] });
1120 const existing = (task.withinWriteGaps || []). find ( row => row.sourceStableKey === sourceStableKey && row.ref === ref);
1121 if (resolved) {
1122 if ( ! existing) { result = task.withinWriteGaps || []; return false ; }
1123 if (evidence.status !== 'verified' || existing.targetId !== targetId || existing.expectedDigest !== evidence.expectedDigest || ! (Date. parse (evidence.observedAt) > Date. parse (existing.recordedAt))) throw new Error ( 'WITHIN_WRITE_EVIDENCE_INVALID' );
1124 }
1125 const gaps = (task.withinWriteGaps || []). filter ( row => row !== existing);
1126 if ( ! resolved) gaps. push ({ ref, sourceStableKey, targetId, code, siteId: stored.ctx.siteId, runId: stored.ctx.runId, expectedDigest: evidence.expectedDigest,
1127 observation: { status: evidence.status, reads: evidence.reads, reasons: evidence.reasons, observedAt: evidence.observedAt }, recordedAt: evidence.observedAt,
1128 operatorRole: 'Requires Import Operator' , action: `Read back target ${ targetId } for ${ ref }; if it is still missing, follow the existing ${ ref } recovery for this record. Do not recreate or resend the landed parent.` });
1129 result = task.withinWriteGaps = gaps. sort (( a , b ) => (a.sourceStableKey + a.ref). localeCompare (b.sourceStableKey + b.ref));
1130 });
1131 return result;
1132 }
1133 // The media patch pass (generic: any ref with `dependencyFailureActions: omit`). Records this task
1134 // already wrote without a dependency that has since landed in this run, so the runner can gate each
1135 // again with its full reference list and send the reviewed update (product media:
1136 // patchStoresProductMediaVerified). Unchecked ids name no record and are never listed.
1137 async function omittedReady ({ taskBinding }) {
1138 const ctx = capability. requireKind (taskBinding, 'import-task' ), index = await evidenceIndex (ctx);
1139 const rows = read (ctx.projectDir, 'execution/dependency-readiness.json' , { tasks: {} }).tasks?.[ctx.taskId]?.omittedReferences || [];
1140 const landed = key => { const attempt = index.attempts. get (key), row = index.crosswalk. get (key); return attempt?.siteId === ctx.siteId && row?.siteId === ctx.siteId && row.status === 'imported' && [ 'imported' , 'skipped_already_imported' ]. includes (attempt.status); };
1141 const result = new Map ();
1142 for ( const row of rows) {
1143 if (row.siteId !== ctx.siteId || ! landed (row.sourceStableKey) || row.dependencySourceKey. startsWith ( 'unchecked:' ) || row.dependencyStatus === 'waived' ) continue ;
1144 const edge = ctx.task.dependencyEdges. find ( edge => edge.ref === row.ref), producer = edge?.producer.kind === 'import-task' && ctx.manifest.tasks. find ( task => task.id === edge.producer.taskId);
1145 if ( ! producer || index.attempts. get (row.dependencySourceKey)?.runId !== ctx.runId || ! keyReady (ctx, index, row.dependencySourceKey, { task: producer, snapshot: snapshots (ctx, producer), ref: row.ref }).ready) continue ;
1146 result. set (row.sourceStableKey, [ ... (result. get (row.sourceStableKey) || []), row.dependencySourceKey]);
1147 }
1148 return [ ... result]. sort (([ a ], [ b ]) => a. localeCompare (b)). map (([ sourceStableKey , dependencySourceKeys ]) => ({ sourceStableKey, dependencySourceKeys: dependencySourceKeys. sort () }));
1149 }
1150 // Clears the omission rows a verified update repaired. `readBack` is the trusted Wix read of the
1151 // record after the update (patchStoresProductMediaVerified's `product`); a row clears only when the
1152 // grant that admitted the update resolved its dependency key and that target id is on the read.
1153 async function recordOmissionsResolved ({ grant , taskBinding , sourceStableKey , readBack , targetId }) {
1154 const stored = grants. get (grant), bound = capability. requireKind (taskBinding, 'import-task' );
1155 if ( ! stored || stored.ctx.taskId !== bound.taskId || stored.ctx.runId !== bound.runId || stored.ctx.siteId !== bound.siteId || ! stored.allowed. has (sourceStableKey)) throw new Error ( 'DEPENDENCY_READINESS_REQUIRED' );
1156 if ( ! require ( '../resources/rp-target-wix/lib/wix-writers' ). validReadBackEvidence (readBack, { siteId: stored.ctx.siteId, targetId })) throw new Error ( 'OMISSION_EVIDENCE_INVALID' );
1157 const values = new Set (), visit = value => { if ( typeof value === 'string' ) values. add (value); else if (value && typeof value === 'object' ) Object. values (value). forEach (visit); };
1158 visit (readBack);
1159 const repaired = new Set ((stored.payloads. get (sourceStableKey) || []). filter ( row => ! row.action && values. has (row.targetId)). map ( row => row.ref + ' \n ' + row.sourceStableKey));
1160 let remaining;
1161 await updateDiagnostic (stored.ctx.projectDir, diagnostic => {
1162 const task = diagnostic.tasks[stored.ctx.taskId];
1163 if ( ! task?.omittedReferences) { remaining = []; return false ; }
1164 task.omittedReferences = task.omittedReferences. filter ( row => row.sourceStableKey !== sourceStableKey || ! repaired. has (row.ref + ' \n ' + row.dependencySourceKey));
1165 remaining = task.omittedReferences. filter ( row => row.sourceStableKey === sourceStableKey);
1166 if ( ! task.omittedReferences. length ) delete task.omittedReferences;
1167 });
1168 return remaining;
1169 }
1170 async function finishTaskPass ({ taskBinding , selection = { mode: 'full' } }) {
1171 const ctx = capability. requireKind (taskBinding, 'import-task' );
1172 if ( ! SELECTIVE . has (selection.mode)) throw new Error ( 'readiness_selection_invalid' );
1173 const snapshot = snapshots (ctx, ctx.task);
1174 const row = { schemaVersion: 1 , taskId: ctx.taskId, entityRefs: ctx.task.entityRefs, runId: ctx.runId, planVersion: ctx.manifest.planVersion,
1175 manifestDigest: fileDigest (ctx.projectDir, 'execution/execution-manifest.json' ), setupPlanDigest: fileDigest (ctx.projectDir, 'setup/setup-plan.json' ),
1176 mappingPlanDigest: fileDigest (ctx.projectDir, 'mapping/mapping-plan.json' ), siteId: ctx.siteId, sourceSnapshotDigests: snapshot.digests,
1177 selection, dryRun: ctx.dryRun, finishedAt: new Date (). toISOString () };
1178 validatePass (row); const passPath = inside (ctx.projectDir, PASS_FILE ); await fsp. mkdir (path. dirname (passPath), { recursive: true }); await fsp. appendFile (passPath, JSON . stringify (row) + ' \n ' ); return row;
1179 }
1180 async function reconcileSiteEvidence ({ taskBinding , readBack }) {
1181 const ctx = capability. requireKind (taskBinding, 'import-task' ), index = await evidenceIndex (ctx), snapshot = snapshots (ctx, ctx.task);
1182 for ( const key of snapshot.keys) {
1183 const row = index.crosswalk. get (key);
1184 if ( ! row || keyReady (ctx, index, key, { task: ctx.task, snapshot }).ready) continue ;
1185 let matched = null ;
1186 try { matched = await readBack ({ targetId: row.targetId, sourceStableKey: key, siteId: ctx.siteId }); } catch {}
1187 if ( ! matched || matched.matchesRecordedWrite !== true || ! require ( '../resources/rp-target-wix/lib/wix-writers' ). validReadBackEvidence (matched.result, { siteId: ctx.siteId, targetId: row.targetId })) continue ;
1188 await local. updateCrosswalkRows (ctx.projectDir, [{ ... row, siteId: ctx.siteId, evidence: 'site-reconciled' }]);
1189 const previous = index.attempts. get (key);
1190 await local. appendAttempt (ctx.projectDir, { ... (previous || { schemaVersion: 1 , sourceStableKey: key, writeSpecId: ctx.taskId, operation: 'create' , targetEntityType: row.targetEntityType, runId: ctx.runId, phase: 'import' }), attemptId: crypto. randomUUID (), siteId: ctx.siteId, status: 'imported' , evidence: 'site-reconciled' , ts: new Date (). toISOString () });
1191 }
1192 }
1193 module . exports = { startImportRun: capability.startImportRun, bindTask: capability.bindTask, bindDependencyProducers, resolveManifestGraph, dependencyModeErrors,
1194 validateImportInputs, validateAssociations, loadDependencyDispositions, preflightMappingDependencies, pluginInventoryErrors, deriveDependencyEdges, evaluateTask, gateBatch,
1195 verifyGrant, verifyBulkGrant, sameTaskReferenceFields: sameTaskFields, recordTaskGap, recordClientTaskGap, verifyReferenceTargets, verifyReferenceMember, holdCausesOf, finalDeferrals, linkChoice, linkRequired, currentRecordTargetId, recordWithinWriteGap, omittedReady, recordOmissionsResolved, finishTaskPass, reconcileSiteEvidence, validatePass, PASS_FILE, pathMatches, valueAt, expandPaths };
1196 for ( const name of [ 'deriveDependencyEdges' , 'dependencyModeErrors' , 'loadDependencyDispositions' , 'preflightMappingDependencies' , 'validateImportInputs' ]) {
1197 const run = module . exports [name];
1198 module . exports [name] = ( projectDir , ... args ) => assessments. withProject (projectDir, () => run (projectDir, ... args));
1199 }
1200 for ( const name of [ 'bindDependencyProducers' , 'resolveManifestGraph' , 'validateAssociations' ]) {
1201 const run = module . exports [name];
1202 module . exports [name] = options => options.plan ? assessments. withPlan (options.plan, options.projectDir, () => run (options)) : run (options);
1203 }
1204 for ( const name of [ 'evaluateTask' , 'finishTaskPass' , 'reconcileSiteEvidence' ]) {
1205 const run = module . exports [name];
1206 module . exports [name] = options => assessments. withProject (capability. requireKind (options.taskBinding, 'import-task' ).projectDir, () => run (options));
1207 }
1208 module . exports . gateBatch = options => {
1209 const state = evaluations. get (options.evaluation);
1210 return state ? assessments. withProject (state.ctx.projectDir, () => gateBatch (options)) : gateBatch (options);
1211 };
1212 for ( const name of [ 'verifyGrant' , 'verifyBulkGrant' ]) {
1213 const run = module . exports [name];
1214 module . exports [name] = ( grant , ... args ) => {
1215 const stored = grants. get (grant);
1216 if ( ! stored) return false ;
1217 try { return assessments. withProject (stored.ctx.projectDir, () => run (grant, ... args)); } catch { return false ; }
1218 };
1219 }