Setting the file. One moment.
Shopify Protected Fields · Rp Source Shopify · wix/skills · Skills Docs
Back to the top of the page function printNodes
— line 314
This file
Number 38.13
Position 13 of 51
Type JavaScript
Size 38 KB
Lines 886 lib/ shopify-protected-fields.js
JavaScript · 886 lines · 38 KB
'node:path'
);
11 const { finished } = require ( 'node:stream/promises' );
12 const {
13 SOURCE_COVERAGE_FILE ,
14 reconcileProject ,
15 upsertAdapterCoverageRows ,
16 writeJsonAtomic ,
17 } = require ( '../../../lib/source-coverage.js' );
18 const { ShopifyCliError } = require ( './shopify-cli.js' );
19
20 const KINDS = Object. freeze ([ 'name' , 'email' , 'phone' , 'address' ]);
21 const ENTITIES = Object. freeze ([ 'customers' , 'orders' ]);
22 const KIND_PROBES_FILE = 'data/shopify-discovery/kind-probes.json' ;
23 const WITHHELD_EXPORT_FILE = 'data/shopify-discovery/withheld-export.json' ;
24 const EXTRACT_MANIFEST_FILE = 'data/source-extract/extract-manifest.json' ;
25 const PROBE_SCHEMA_VERSION = 1 ;
26 const WITHHELD_EXPORT_SCHEMA_VERSION = 1 ;
27 const WITHHELD_EXPORT_CONFIRMATION = 'owner-exported' ;
28 const WITHHELD_EXPORT_CODE = 'shopify-withheld-details-export' ;
29 const PROBE_QUERIES_DIR = path. join (__dirname, '..' , 'queries' );
30
31 const ADDRESS_NAME_FIELDS = Object. freeze ([ 'firstName' , 'lastName' ]);
32 const ADDRESS_PHONE_FIELDS = Object. freeze ([ 'phone' ]);
33 const ADDRESS_FIELDS = Object. freeze ([ 'address1' , 'address2' , 'zip' , 'latitude' , 'longitude' ]);
34 const ADDRESS_UNPROTECTED_FIELDS = Object. freeze ([ 'id' , 'company' , 'city' , 'province' , 'provinceCode' , 'country' , 'countryCodeV2' ]);
35
36 function prefixed ( containers , fields ) {
37 return containers. flatMap (( container ) => fields. map (( field ) => `${ container }.${ field }` ));
38 }
39
40 const CUSTOMER_ADDRESSES = Object. freeze ([ 'defaultAddress' , 'addresses' ]);
41 const ORDER_ADDRESSES = Object. freeze ([
42 'billingAddress' ,
43 'shippingAddress' ,
44 'customer.defaultAddress' ,
45 'customer.addresses' ,
46 ]);
47
48 const PROTECTED_FIELDS = Object. freeze ({
49 customers: Object. freeze ({
50 name: Object. freeze ([ 'firstName' , 'lastName' , ... prefixed ( CUSTOMER_ADDRESSES , ADDRESS_NAME_FIELDS )]),
51 email: Object. freeze ([ 'email' , 'defaultEmailAddress.emailAddress' ]),
52 phone: Object. freeze ([ 'phone' , 'defaultPhoneNumber.phoneNumber' , ... prefixed ( CUSTOMER_ADDRESSES , ADDRESS_PHONE_FIELDS )]),
53 address: Object. freeze ( prefixed ( CUSTOMER_ADDRESSES , ADDRESS_FIELDS )),
54 }),
55 orders: Object. freeze ({
56 name: Object. freeze ([ 'customer.firstName' , 'customer.lastName' , ... prefixed ( ORDER_ADDRESSES , ADDRESS_NAME_FIELDS )]),
57 email: Object. freeze ([ 'email' , 'customer.email' , 'customer.defaultEmailAddress.emailAddress' ]),
58 phone: Object. freeze ([ 'phone' , 'customer.phone' , 'customer.defaultPhoneNumber.phoneNumber' , ... prefixed ( ORDER_ADDRESSES , ADDRESS_PHONE_FIELDS )]),
59 address: Object. freeze ( prefixed ( ORDER_ADDRESSES , ADDRESS_FIELDS )),
60 }),
61 });
62
63 const COMPOSITE_FIELDS = Object. freeze ([
64 'displayName' ,
65 'defaultAddress.name' ,
66 'defaultAddress.formatted' ,
67 'defaultAddress.formattedArea' ,
68 'addresses.name' ,
69 'addresses.formatted' ,
70 'addresses.formattedArea' ,
71 'billingAddress.name' ,
72 'billingAddress.formatted' ,
73 'billingAddress.formattedArea' ,
74 'shippingAddress.name' ,
75 'shippingAddress.formatted' ,
76 'shippingAddress.formattedArea' ,
77 'customer.displayName' ,
78 ]);
79
80 // Filled explicitly from the four committed bulk queries. The contract test compares the
81 // parsed leaf set with this list plus PROTECTED_FIELDS, so adding a query field requires a
82 // classification decision rather than silently becoming unprotected.
83 const UNPROTECTED_FIELDS = Object. freeze ({
84 customers: Object. freeze ([
85 'addresses.city' , 'addresses.company' , 'addresses.country' , 'addresses.countryCodeV2' ,
86 'addresses.id' , 'addresses.province' , 'addresses.provinceCode' , 'createdAt' ,
87 'defaultAddress.city' , 'defaultAddress.company' , 'defaultAddress.country' ,
88 'defaultAddress.countryCodeV2' , 'defaultAddress.id' , 'defaultAddress.province' ,
89 'defaultAddress.provinceCode' , 'defaultEmailAddress.marketingOptInLevel' ,
90 'defaultEmailAddress.marketingState' , 'defaultEmailAddress.marketingUpdatedAt' ,
91 'defaultPhoneNumber.marketingOptInLevel' , 'defaultPhoneNumber.marketingState' ,
92 'defaultPhoneNumber.marketingUpdatedAt' , 'emailMarketingConsent.consentUpdatedAt' ,
93 'emailMarketingConsent.marketingOptInLevel' , 'emailMarketingConsent.marketingState' , 'id' ,
94 'locale' , 'metafields.__typename' , 'metafields.id' , 'metafields.key' ,
95 'metafields.namespace' , 'metafields.type' , 'metafields.value' , 'note' , 'numberOfOrders' ,
96 'smsMarketingConsent.consentUpdatedAt' , 'smsMarketingConsent.marketingOptInLevel' ,
97 'smsMarketingConsent.marketingState' , 'state' , 'tags' , 'taxExempt' , 'updatedAt' ,
98 'verifiedEmail' ,
99 ]),
100 orders: Object. freeze ([
101 'billingAddress.city' , 'billingAddress.company' , 'billingAddress.country' ,
102 'billingAddress.countryCodeV2' , 'billingAddress.province' , 'billingAddress.provinceCode' ,
103 'cancelReason' , 'cancelledAt' , 'closedAt' , 'confirmed' , 'createdAt' , 'currencyCode' ,
104 'customer.id' , 'customerLocale' , 'discountApplications.__typename' ,
105 'discountApplications.allocationMethod' , 'discountApplications.code' ,
106 'discountApplications.description' , 'discountApplications.targetSelection' ,
107 'discountApplications.targetType' , 'discountApplications.title' ,
108 'discountApplications.value.__typename' , 'discountApplications.value.amount' ,
109 'discountApplications.value.currencyCode' , 'discountApplications.value.percentage' ,
110 'discountCodes' , 'displayFinancialStatus' , 'displayFulfillmentStatus' ,
111 'fulfillments.createdAt' , 'fulfillments.deliveredAt' , 'fulfillments.displayStatus' ,
112 'fulfillments.estimatedDeliveryAt' , 'fulfillments.fulfillmentLineItems.__typename' ,
113 'fulfillments.fulfillmentLineItems.id' , 'fulfillments.fulfillmentLineItems.lineItem.id' ,
114 'fulfillments.fulfillmentLineItems.quantity' , 'fulfillments.id' ,
115 'fulfillments.inTransitAt' , 'fulfillments.location.id' , 'fulfillments.name' ,
116 'fulfillments.status' , 'fulfillments.totalQuantity' , 'fulfillments.trackingInfo.company' ,
117 'fulfillments.trackingInfo.number' , 'fulfillments.trackingInfo.url' ,
118 'fulfillments.updatedAt' , 'id' , 'lineItems.__typename' ,
119 'lineItems.discountedUnitPriceSet.presentmentMoney.amount' ,
120 'lineItems.discountedUnitPriceSet.presentmentMoney.currencyCode' ,
121 'lineItems.discountedUnitPriceSet.shopMoney.amount' ,
122 'lineItems.discountedUnitPriceSet.shopMoney.currencyCode' , 'lineItems.id' ,
123 'lineItems.isGiftCard' , 'lineItems.originalUnitPriceSet.presentmentMoney.amount' ,
124 'lineItems.originalUnitPriceSet.presentmentMoney.currencyCode' ,
125 'lineItems.originalUnitPriceSet.shopMoney.amount' ,
126 'lineItems.originalUnitPriceSet.shopMoney.currencyCode' , 'lineItems.product.id' ,
127 'lineItems.quantity' , 'lineItems.requiresShipping' , 'lineItems.sku' ,
128 'lineItems.taxLines.priceSet.shopMoney.amount' ,
129 'lineItems.taxLines.priceSet.shopMoney.currencyCode' , 'lineItems.taxLines.rate' ,
130 'lineItems.taxLines.title' , 'lineItems.taxable' , 'lineItems.title' ,
131 'lineItems.totalDiscountSet.presentmentMoney.amount' ,
132 'lineItems.totalDiscountSet.presentmentMoney.currencyCode' ,
133 'lineItems.totalDiscountSet.shopMoney.amount' ,
134 'lineItems.totalDiscountSet.shopMoney.currencyCode' , 'lineItems.variant.id' ,
135 'lineItems.variantTitle' , 'lineItems.vendor' , 'metafields.__typename' , 'metafields.id' ,
136 'metafields.key' , 'metafields.namespace' , 'metafields.type' , 'metafields.value' , 'name' ,
137 'note' , 'number' , 'paymentGatewayNames' , 'presentmentCurrencyCode' , 'processedAt' ,
138 'refunds.createdAt' , 'refunds.id' , 'refunds.note' , 'refunds.processedAt' ,
139 'refunds.refundLineItems.__typename' , 'refunds.refundLineItems.id' ,
140 'refunds.refundLineItems.lineItem.id' ,
141 'refunds.refundLineItems.priceSet.presentmentMoney.amount' ,
142 'refunds.refundLineItems.priceSet.presentmentMoney.currencyCode' ,
143 'refunds.refundLineItems.priceSet.shopMoney.amount' ,
144 'refunds.refundLineItems.priceSet.shopMoney.currencyCode' ,
145 'refunds.refundLineItems.quantity' , 'refunds.refundLineItems.restockType' ,
146 'refunds.refundLineItems.restocked' ,
147 'refunds.refundLineItems.subtotalSet.presentmentMoney.amount' ,
148 'refunds.refundLineItems.subtotalSet.presentmentMoney.currencyCode' ,
149 'refunds.refundLineItems.subtotalSet.shopMoney.amount' ,
150 'refunds.refundLineItems.subtotalSet.shopMoney.currencyCode' ,
151 'refunds.refundLineItems.totalTaxSet.presentmentMoney.amount' ,
152 'refunds.refundLineItems.totalTaxSet.presentmentMoney.currencyCode' ,
153 'refunds.refundLineItems.totalTaxSet.shopMoney.amount' ,
154 'refunds.refundLineItems.totalTaxSet.shopMoney.currencyCode' ,
155 'refunds.totalRefundedSet.presentmentMoney.amount' ,
156 'refunds.totalRefundedSet.presentmentMoney.currencyCode' ,
157 'refunds.totalRefundedSet.shopMoney.amount' ,
158 'refunds.totalRefundedSet.shopMoney.currencyCode' , 'refunds.transactions.__typename' ,
159 'refunds.transactions.amountSet.presentmentMoney.amount' ,
160 'refunds.transactions.amountSet.presentmentMoney.currencyCode' ,
161 'refunds.transactions.amountSet.shopMoney.amount' ,
162 'refunds.transactions.amountSet.shopMoney.currencyCode' , 'refunds.transactions.gateway' ,
163 'refunds.transactions.id' , 'refunds.transactions.kind' ,
164 'refunds.transactions.processedAt' , 'refunds.transactions.status' , 'refunds.updatedAt' ,
165 'shippingAddress.city' , 'shippingAddress.company' , 'shippingAddress.country' ,
166 'shippingAddress.countryCodeV2' , 'shippingAddress.province' ,
167 'shippingAddress.provinceCode' , 'shippingLines.__typename' , 'shippingLines.code' ,
168 'shippingLines.discountedPriceSet.presentmentMoney.amount' ,
169 'shippingLines.discountedPriceSet.presentmentMoney.currencyCode' ,
170 'shippingLines.discountedPriceSet.shopMoney.amount' ,
171 'shippingLines.discountedPriceSet.shopMoney.currencyCode' , 'shippingLines.id' ,
172 'shippingLines.originalPriceSet.presentmentMoney.amount' ,
173 'shippingLines.originalPriceSet.presentmentMoney.currencyCode' ,
174 'shippingLines.originalPriceSet.shopMoney.amount' ,
175 'shippingLines.originalPriceSet.shopMoney.currencyCode' , 'shippingLines.source' ,
176 'shippingLines.taxLines.priceSet.shopMoney.amount' ,
177 'shippingLines.taxLines.priceSet.shopMoney.currencyCode' , 'shippingLines.taxLines.rate' ,
178 'shippingLines.taxLines.title' , 'shippingLines.title' , 'sourceName' ,
179 'subtotalPriceSet.presentmentMoney.amount' , 'subtotalPriceSet.presentmentMoney.currencyCode' ,
180 'subtotalPriceSet.shopMoney.amount' , 'subtotalPriceSet.shopMoney.currencyCode' , 'tags' ,
181 'taxLines.priceSet.shopMoney.amount' , 'taxLines.priceSet.shopMoney.currencyCode' ,
182 'taxLines.rate' , 'taxLines.title' , 'taxesIncluded' , 'test' ,
183 'totalDiscountsSet.presentmentMoney.amount' ,
184 'totalDiscountsSet.presentmentMoney.currencyCode' , 'totalDiscountsSet.shopMoney.amount' ,
185 'totalDiscountsSet.shopMoney.currencyCode' , 'totalPriceSet.presentmentMoney.amount' ,
186 'totalPriceSet.presentmentMoney.currencyCode' , 'totalPriceSet.shopMoney.amount' ,
187 'totalPriceSet.shopMoney.currencyCode' , 'totalRefundedSet.presentmentMoney.amount' ,
188 'totalRefundedSet.presentmentMoney.currencyCode' , 'totalRefundedSet.shopMoney.amount' ,
189 'totalRefundedSet.shopMoney.currencyCode' ,
190 'totalShippingPriceSet.presentmentMoney.amount' ,
191 'totalShippingPriceSet.presentmentMoney.currencyCode' ,
192 'totalShippingPriceSet.shopMoney.amount' , 'totalShippingPriceSet.shopMoney.currencyCode' ,
193 'totalTaxSet.presentmentMoney.amount' , 'totalTaxSet.presentmentMoney.currencyCode' ,
194 'totalTaxSet.shopMoney.amount' , 'totalTaxSet.shopMoney.currencyCode' ,
195 'transactions.__typename' , 'transactions.amountSet.presentmentMoney.amount' ,
196 'transactions.amountSet.presentmentMoney.currencyCode' , 'transactions.amountSet.shopMoney.amount' ,
197 'transactions.amountSet.shopMoney.currencyCode' , 'transactions.authorizationExpiresAt' ,
198 'transactions.createdAt' , 'transactions.errorCode' , 'transactions.gateway' , 'transactions.id' ,
199 'transactions.kind' , 'transactions.manualPaymentGateway' , 'transactions.parentTransaction.id' ,
200 'transactions.processedAt' , 'transactions.status' , 'transactions.test' , 'updatedAt' ,
201 ]),
202 });
203
204 function singular ( entity ) {
205 if (entity === 'customers' ) return 'customer' ;
206 if (entity === 'orders' ) return 'order' ;
207 throw new Error ( `protected fields do not apply to ${ entity }` );
208 }
209
210 function entityForQuery ( name ) {
211 if (name === 'customers' ) return 'customers' ;
212 if ( String (name). startsWith ( 'orders-' )) return 'orders' ;
213 return null ;
214 }
215
216 function probeKey ( entity , kind ) {
217 return `${ entity }:${ kind }` ;
218 }
219
220 function probeQueryFile ( entity , kind ) {
221 return path. join ( PROBE_QUERIES_DIR , `probe-${ entity }-${ kind }.graphql` );
222 }
223
224 function probeQuery ( entity , kind ) {
225 if ( ! ENTITIES . includes (entity) || ! KINDS . includes (kind)) throw new Error ( `unknown protected-field probe ${ entity }/${ kind }` );
226 return fs. readFileSync ( probeQueryFile (entity, kind), 'utf8' );
227 }
228
229 // ---------------------------------------------------------------------------------------------
230 // Small GraphQL selection parser: projection and the closed-field contract
231 // ---------------------------------------------------------------------------------------------
232
233 function tokenizeGraphQL ( text ) {
234 return ( String (text). replace ( /# [ ^ \n] * / g , ' ' ). match ( / \.\.\. | [_A-Za-z][_0-9A-Za-z] *| - ? \d + (?: \. \d + ) ?| "(?: \\ . | [ ^ " \\ ] ) * " | [$!(){}[ \] :,@=] / g ) || []);
235 }
236
237 function parseGraphQL ( text ) {
238 const tokens = tokenizeGraphQL (text);
239 const rootAt = tokens. indexOf ( '{' );
240 if (rootAt === - 1 ) throw new Error ( 'GraphQL document has no selection set' );
241
242 function balanced ( i , open , close ) {
243 let depth = 0 ;
244 const out = [];
245 for (; i < tokens. length ; i += 1 ) {
246 const token = tokens[i];
247 out. push (token);
248 if (token === open) depth += 1 ;
249 if (token === close) {
250 depth -= 1 ;
251 if (depth === 0 ) return { tokens: out, next: i + 1 };
252 }
253 }
254 throw new Error ( `unterminated GraphQL ${ open }` );
255 }
256
257 function selectionSet ( i ) {
258 if (tokens[i] !== '{' ) throw new Error ( 'selection set must start with {' );
259 const nodes = [];
260 i += 1 ;
261 while (i < tokens. length && tokens[i] !== '}' ) {
262 if (tokens[i] === '...' ) {
263 const head = [ '...' ];
264 i += 1 ;
265 while (i < tokens. length && tokens[i] !== '{' && tokens[i] !== '}' ) {
266 head. push (tokens[i]);
267 i += 1 ;
268 }
269 const child = tokens[i] === '{' ? selectionSet (i) : null ;
270 if (child) i = child.next;
271 nodes. push ({ kind: 'fragment' , head, children: child ? child.nodes : [] });
272 continue ;
273 }
274 const head = [];
275 let name = tokens[i];
276 head. push (tokens[i]);
277 i += 1 ;
278 if (tokens[i] === ':' ) {
279 head. push ( ':' , tokens[i + 1 ]);
280 name = tokens[i + 1 ];
281 i += 2 ;
282 }
283 if (tokens[i] === '(' ) {
284 const args = balanced (i, '(' , ')' );
285 head. push ( ... args.tokens);
286 i = args.next;
287 }
288 while (tokens[i] === '@' ) {
289 head. push ( '@' , tokens[i + 1 ]);
290 i += 2 ;
291 if (tokens[i] === '(' ) {
292 const directive = balanced (i, '(' , ')' );
293 head. push ( ... directive.tokens);
294 i = directive.next;
295 }
296 }
297 let children = null ;
298 if (tokens[i] === '{' ) {
299 const child = selectionSet (i);
300 children = child.nodes;
301 i = child.next;
302 }
303 nodes. push ({ kind: 'field' , name, head, children });
304 }
305 if (tokens[i] !== '}' ) throw new Error ( 'unterminated GraphQL selection set' );
306 return { nodes, next: i + 1 };
307 }
308
309 const parsed = selectionSet (rootAt);
310 return { preamble: tokens. slice ( 0 , rootAt), nodes: parsed.nodes };
311 }
312
313 function printGraphQL ( document ) {
314 function printNodes ( nodes ) {
315 return nodes. map (( node ) => `${ node . head . join ( ' ' ) }${ node . children ? ` { ${ printNodes ( node . children ) } }` : ''}` ). join ( ' ' );
316 }
317 return `${ document . preamble . length ? `${ document . preamble . join ( ' ' ) } ` : ''}{ ${ printNodes ( document . nodes ) } } \n ` ;
318 }
319
320 function logicalPath ( pathParts ) {
321 const stripped = pathParts. filter (( part , index ) => {
322 if (index === 0 && ENTITIES . includes (part)) return false ;
323 return part !== 'edges' && part !== 'node' && ! / ^ \d +$ / . test ( String (part));
324 });
325 return stripped. join ( '.' );
326 }
327
328 function selectedLeafPaths ( queryText , entity ) {
329 const document = parseGraphQL (queryText);
330 const leaves = [];
331 function walk ( nodes , parts ) {
332 for ( const node of nodes) {
333 if (node.kind === 'fragment' ) { walk (node.children, parts); continue ; }
334 const next = [ ... parts, node.name];
335 if (node.children) walk (node.children, next);
336 else leaves. push ( logicalPath (next));
337 }
338 }
339 walk (document.nodes, []);
340 return [ ...new Set (leaves. filter (Boolean))]. sort ();
341 }
342
343 function projectQuery ( queryText , entity , withheldKinds = []) {
344 if ( ! ENTITIES . includes (entity)) return String (queryText);
345 const remove = new Set (withheldKinds. flatMap (( kind ) => PROTECTED_FIELDS [entity][kind] || []));
346 if ( ! remove.size) return String (queryText);
347 const document = parseGraphQL (queryText);
348 function filter ( nodes , parts ) {
349 const out = [];
350 for ( const node of nodes) {
351 if (node.kind === 'fragment' ) {
352 const children = filter (node.children, parts);
353 if (children. length ) out. push ({ ... node, children });
354 continue ;
355 }
356 const next = [ ... parts, node.name];
357 if (node.children) {
358 const children = filter (node.children, next);
359 if (children. length ) out. push ({ ... node, children });
360 } else if ( ! remove. has ( logicalPath (next))) {
361 out. push (node);
362 }
363 }
364 return out;
365 }
366 return printGraphQL ({ ... document, nodes: filter (document.nodes, []) });
367 }
368
369 // ---------------------------------------------------------------------------------------------
370 // Probe records and coverage rows
371 // ---------------------------------------------------------------------------------------------
372
373 async function readJson ( file , fallback = null ) {
374 try {
375 return JSON . parse ( await fsp. readFile (file, 'utf8' ));
376 } catch (error) {
377 if (error.code === 'ENOENT' ) return fallback;
378 throw error;
379 }
380 }
381
382 function emptyProbeState () {
383 return {
384 schemaVersion: PROBE_SCHEMA_VERSION ,
385 probes: {},
386 diagnostics: {},
387 reapproved: {},
388 withheld: { customers: [], orders: [] },
389 projection: { customers: [], orders: [] },
390 };
391 }
392
393 function normalizeProbeState ( value ) {
394 if ( ! value || value.schemaVersion !== PROBE_SCHEMA_VERSION ) return emptyProbeState ();
395 return {
396 ... emptyProbeState (),
397 ... value,
398 probes: value.probes && typeof value.probes === 'object' ? value.probes : {},
399 diagnostics: value.diagnostics && typeof value.diagnostics === 'object' ? value.diagnostics : {},
400 reapproved: value.reapproved && typeof value.reapproved === 'object' ? value.reapproved : {},
401 withheld: {
402 customers: [ ...new Set ((value.withheld && value.withheld.customers) || [])]. filter (( kind ) => KINDS . includes (kind)). sort (),
403 orders: [ ...new Set ((value.withheld && value.withheld.orders) || [])]. filter (( kind ) => KINDS . includes (kind)). sort (),
404 },
405 projection: {
406 customers: [ ...new Set ((value.projection && value.projection.customers) || [])]. filter (( kind ) => KINDS . includes (kind)). sort (),
407 orders: [ ...new Set ((value.projection && value.projection.orders) || [])]. filter (( kind ) => KINDS . includes (kind)). sort (),
408 },
409 };
410 }
411
412 async function readProbeState ( projectDir ) {
413 return normalizeProbeState ( await readJson (path. join (projectDir, KIND_PROBES_FILE ), null ));
414 }
415
416 async function writeProbeState ( projectDir , state ) {
417 state.withheld.customers. sort ();
418 state.withheld.orders. sort ();
419 state.projection = { customers: [ ... state.withheld.customers], orders: [ ... state.withheld.orders] };
420 await writeJsonAtomic (path. join (projectDir, KIND_PROBES_FILE ), state);
421 }
422
423 function errorRecord ( error ) {
424 return {
425 code: error && error.extensions && error.extensions.code ? String (error.extensions.code) : String ((error && error.code) || 'GRAPHQL_ERROR' ),
426 ... (Array. isArray (error && error.path) ? { path: error.path. map (( part ) => typeof part === 'number' ? part : String (part)) } : {}),
427 };
428 }
429
430 function classifyProbe ( entity , kind , errors = []) {
431 const allowed = new Set ( PROTECTED_FIELDS [entity][kind]);
432 const records = errors. map (errorRecord);
433 const matchedPaths = [ ...new Set (records
434 . filter (( error ) => Array. isArray (error.path))
435 . map (( error ) => logicalPath (error.path))
436 . filter (( field ) => allowed. has (field)))]. sort ();
437 return {
438 entity,
439 kind,
440 outcome: matchedPaths. length ? 'withheld' : (records. length ? 'unattributed' : 'readable' ),
441 matchedPaths,
442 errorCodes: [ ...new Set (records. map (( error ) => error.code))],
443 ... (records. length ? { errors: records } : {}),
444 };
445 }
446
447 function isPathlessAccessDenied ( errors ) {
448 return errors. some (( error ) => error.code === 'source_auth_expired'
449 || (error.code === 'ACCESS_DENIED' && ! Array. isArray (error.path)));
450 }
451
452 async function checkSession ( sessionCheck ) {
453 if ( typeof sessionCheck !== 'function' ) return { ok: true };
454 try {
455 const result = await sessionCheck ();
456 return result && result.ok === false ? result : { ok: true };
457 } catch (error) {
458 return { ok: false , error: errorRecord (error) };
459 }
460 }
461
462 async function executeProbe ({ entity , kind , execute , sessionCheck , reapproved = false , markReapproved }) {
463 let errors = [];
464 try {
465 const result = await execute ( probeQuery (entity, kind), { first: 3 });
466 errors = Array. isArray (result && result.errors) ? result.errors : [];
467 } catch (error) {
468 if ( ! (error instanceof ShopifyCliError )) throw error;
469 errors = [error];
470 }
471 const records = errors. map (errorRecord);
472 if ( isPathlessAccessDenied (records)) {
473 const checked = await checkSession (sessionCheck);
474 if ( ! checked.ok && ! reapproved) {
475 await markReapproved ();
476 throw new ShopifyCliError ( 'the Shopify connection failed while attributing protected customer data; re-approve it, then run the same step again' , { code: 'source_auth_expired' , errors: records });
477 }
478 }
479 return classifyProbe (entity, kind, records);
480 }
481
482 function protectedCoverageRow ( entity , kind , probe , { artifact = KIND_PROBES_FILE , pointerPrefix = 'probes' } = {}) {
483 const sourceEntity = `${ singular ( entity ) }-${ kind }` ;
484 const removed = PROTECTED_FIELDS [entity][kind];
485 return {
486 coverageId: `shopify:${ sourceEntity }` ,
487 platform: 'shopify' ,
488 capability: entity === 'customers' ? 'crm.contact' : 'commerce.orders' ,
489 sourceEntity,
490 routes: [entity],
491 reason: 'auth-gated' ,
492 evidence: {
493 artifact,
494 pointer: `${ pointerPrefix }; matched=${ probe . matchedPaths . join ( ',' ) }; projection removes=${ removed . join ( ',' ) }` ,
495 },
496 recoverability: 'requires-development' ,
497 disposition: null ,
498 };
499 }
500
501 function captureFailedRow ( entity , errors , pointer , {
502 artifact = KIND_PROBES_FILE ,
503 recoveryAction = `Fix the Shopify read of ${ entity }, then run Shopify discovery with --reread ${ entity }.` ,
504 } = {}) {
505 return {
506 coverageId: `shopify:${ entity }` ,
507 platform: 'shopify' ,
508 capability: entity === 'customers' ? 'crm.contact' : 'commerce.orders' ,
509 sourceEntity: entity,
510 routes: [entity],
511 reason: 'capture-failed' ,
512 evidence: { artifact, pointer: `${ pointer }; errors=${ JSON . stringify ( errors ) }` },
513 recoverability: 'import-operator' ,
514 recoveryAction,
515 disposition: null ,
516 };
517 }
518
519 function rowsForDiscoveryState ( state ) {
520 const rows = [];
521 for ( const entity of ENTITIES ) {
522 const unattributed = [];
523 for ( const kind of KINDS ) {
524 const probe = state.probes[ probeKey (entity, kind)];
525 if ( ! probe) continue ;
526 if (probe.outcome === 'withheld' ) rows. push ( protectedCoverageRow (entity, kind, probe, { pointerPrefix: `probes.${ probeKey ( entity , kind ) }` }));
527 else if (probe.outcome === 'unattributed' ) unattributed. push ({ kind, errors: probe.errors || [] });
528 }
529 if (unattributed. length ) rows. push ( captureFailedRow (entity, unattributed, `unattributed probes for ${ entity }` ));
530 }
531 return rows;
532 }
533
534 async function runDiscoveryProbes ( ctx ) {
535 const state = await readProbeState (ctx.projectDir);
536 for ( const entity of ENTITIES ) {
537 for ( const kind of KINDS ) {
538 const key = probeKey (entity, kind);
539 if (state.probes[key]) continue ;
540 const probe = await executeProbe ({
541 entity,
542 kind,
543 execute: ctx.execute,
544 sessionCheck: ctx.sessionCheck,
545 reapproved: state.reapproved[key] === true ,
546 markReapproved : async () => {
547 state.reapproved[key] = true ;
548 await writeProbeState (ctx.projectDir, state);
549 },
550 });
551 state.probes[key] = probe;
552 if (probe.outcome === 'withheld' && ! state.withheld[entity]. includes (kind)) state.withheld[entity]. push (kind);
553 await writeProbeState (ctx.projectDir, state);
554 }
555 }
556 await writeProbeState (ctx.projectDir, state);
557 return { file: path. relative ( 'data/shopify-discovery' , KIND_PROBES_FILE ), projection: state.projection, rows: rowsForDiscoveryState (state) };
558 }
559
560 async function resetProbeEntities ( projectDir , entities ) {
561 const relevant = entities. filter (( entity ) => ENTITIES . includes (entity));
562 if ( ! relevant. length ) return false ;
563 const state = await readProbeState (projectDir);
564 for ( const entity of relevant) {
565 for ( const kind of KINDS ) {
566 delete state.probes[ probeKey (entity, kind)];
567 delete state.reapproved[ probeKey (entity, kind)];
568 }
569 state.withheld[entity] = [];
570 }
571 state.diagnostics = {};
572 await writeProbeState (projectDir, state);
573 return true ;
574 }
575
576 // ---------------------------------------------------------------------------------------------
577 // Extract denial handler and denied reset
578 // ---------------------------------------------------------------------------------------------
579
580 async function denialHandler ({ name , entity , entry , projectDir , execute }, { sessionCheck , checkpoint = null } = {}) {
581 const protectedEntity = entityForQuery (name) || entity;
582 if ( ! ENTITIES . includes (protectedEntity)) throw new Error ( `protected denial handler received ${ name }` );
583 const state = await readProbeState (projectDir);
584 const diagnosticKey = `${ name }@${ entry . settlingAt }` ;
585 if ( ! state.diagnostics[diagnosticKey]) {
586 state.diagnostics[diagnosticKey] = {
587 query: name,
588 entity: protectedEntity,
589 settlingAt: entry.settlingAt,
590 withheldBefore: [ ... state.withheld[protectedEntity]],
591 probes: {},
592 reapproved: {},
593 };
594 await writeProbeState (projectDir, state);
595 }
596 const diagnostic = state.diagnostics[diagnosticKey];
597 for ( const kind of KINDS ) {
598 if (diagnostic.probes[kind]) continue ;
599 diagnostic.probes[kind] = await executeProbe ({
600 entity: protectedEntity,
601 kind,
602 execute,
603 sessionCheck,
604 reapproved: diagnostic.reapproved[kind] === true ,
605 markReapproved : async () => {
606 diagnostic.reapproved[kind] = true ;
607 await writeProbeState (projectDir, state);
608 },
609 });
610 await writeProbeState (projectDir, state);
611 // The runner's crash matrix injects failures only after the durable probe write.
612 if ( typeof checkpoint === 'function' ) await checkpoint ( 'after-probe' , { kind, recorded: Object. keys (diagnostic.probes). length });
613 }
614
615 const withheldBefore = Array. isArray (diagnostic.withheldBefore) ? diagnostic.withheldBefore : [];
616 const hasUnattributed = KINDS . some (( kind ) => diagnostic.probes[kind].outcome === 'unattributed' );
617 const newlyWithheld = KINDS . filter (( kind ) => diagnostic.probes[kind].outcome === 'withheld' && ! withheldBefore. includes (kind));
618 if ( ! hasUnattributed && newlyWithheld. length ) {
619 state.withheld[protectedEntity] = [ ...new Set ([ ... state.withheld[protectedEntity], ... newlyWithheld])]. sort ();
620 await writeProbeState (projectDir, state);
621 const rows = newlyWithheld. map (( kind ) => protectedCoverageRow (protectedEntity, kind, diagnostic.probes[kind], { pointerPrefix: `diagnostics.${ diagnosticKey }.${ kind }` }));
622 await upsertAdapterCoverageRows (projectDir, { adapter: 'shopify' , rows, checkpoint });
623 const coverageIds = rows. map (( row ) => row.coverageId);
624 return {
625 handlerOutcome: { attribution: 'withheld' , kinds: newlyWithheld, coverageIds },
626 stop: {
627 code: 'shopify_protected_fields_withheld' ,
628 message: `Shopify withheld ${ newlyWithheld . join ( ', ' ) } from ${ protectedEntity }; ${ coverageIds . join ( ', ' ) } was recorded for a mapping decision.` ,
629 },
630 };
631 }
632
633 const bulkError = entry.error || { message: 'Shopify denied the bulk export' };
634 const row = captureFailedRow (protectedEntity, bulkError, `diagnostics.${ diagnosticKey }; bulk=${ JSON . stringify ( bulkError ) }` , {
635 artifact: EXTRACT_MANIFEST_FILE ,
636 recoveryAction: `Fix the cause of the ${ name } denial, then run shopify-extract.js --retry-failed.` ,
637 });
638 await upsertAdapterCoverageRows (projectDir, { adapter: 'shopify' , rows: [row], checkpoint });
639 return {
640 handlerOutcome: { attribution: 'unattributed' , coverageId: row.coverageId },
641 stop: { code: 'shopify_bulk_failed' , message: `Shopify denied ${ name }, but its four protected-field probes did not attribute the refusal. The projection was not narrowed.` },
642 };
643 }
644
645 async function deniedReset ({ manifest , projectDir }) {
646 const state = await readProbeState (projectDir);
647 const resets = [];
648 for ( const name of manifest.order || []) {
649 const target = manifest.queries && manifest.queries[name];
650 if ( ! target || target.state !== 'denied' || ! target.handlerOutcome || target.handlerOutcome.attribution !== 'withheld' ) continue ;
651 const entity = entityForQuery (name);
652 const kinds = target.handlerOutcome.kinds || [];
653 if ( ! entity || ! kinds. length || ! kinds. every (( kind ) => state.withheld[entity]. includes (kind))) continue ;
654 resets. push ({ name, text: projectQuery (target.text, entity, state.withheld[entity]) });
655 }
656 return resets;
657 }
658
659 // ---------------------------------------------------------------------------------------------
660 // Hollow customer recheck
661 // ---------------------------------------------------------------------------------------------
662
663 function valuesAt ( value , parts ) {
664 if ( ! parts. length ) return [value];
665 if (Array. isArray (value)) return value. flatMap (( item ) => valuesAt (item, parts));
666 if ( ! value || typeof value !== 'object' ) return [];
667 return valuesAt (value[parts[ 0 ]], parts. slice ( 1 ));
668 }
669
670 function hasValue ( record , field ) {
671 return valuesAt (record, field. split ( '.' )). some (( value ) => value !== null && value !== undefined && ( typeof value !== 'string' || value. trim () !== '' ));
672 }
673
674 function isHollowCustomer ( record , withheldKinds ) {
675 const withheldIdentity = [ 'name' , 'email' , 'phone' ]. filter (( kind ) => withheldKinds. includes (kind));
676 if ( ! withheldIdentity. length ) return false ;
677 const readable = [ 'name' , 'email' , 'phone' ]. filter (( kind ) => ! withheldKinds. includes (kind));
678 return ! readable. some (( kind ) => PROTECTED_FIELDS .customers[kind]. some (( field ) => hasValue (record, field)));
679 }
680
681 async function* recheckCustomers ( records , projection ) {
682 const withheldKinds = Array. isArray (projection) ? projection : ((projection && projection.customers) || []);
683 for await ( const record of records) {
684 if ( isHollowCustomer (record, withheldKinds)) {
685 if ( ! record || typeof record.id !== 'string' || ! record.id) throw new Error ( 'a hollow Shopify customer has no id' );
686 yield { kind: 'hollow' , record: { id: record.id } };
687 }
688 else yield { kind: 'keep' , record };
689 }
690 }
691
692 async function coverageRowDispositioned ( projectDir , coverageId ) {
693 const result = await reconcileProject (projectDir);
694 return result.resolved. some (( item ) => item.row.coverageId === coverageId);
695 }
696
697 async function writeLine ( stream , value ) {
698 if ( ! stream. write ( `${ JSON . stringify ( value ) } \n ` )) await new Promise (( resolve ) => stream. once ( 'drain' , resolve));
699 }
700
701 async function copyFileAtomic ( source , destination ) {
702 await fsp. mkdir (path. dirname (destination), { recursive: true });
703 const temp = `${ destination }.${ process . pid }.${ Date . now () }.tmp` ;
704 try {
705 await fsp. copyFile (source, temp);
706 await fsp. rename (temp, destination);
707 } catch (error) {
708 await fsp. rm (temp, { force: true });
709 throw error;
710 }
711 }
712
713 async function customerRecheckHook ({ records , projectDir , stagingDir }) {
714 const state = await readProbeState (projectDir);
715 const stamp = `${ process . pid }.${ Date . now () }` ;
716 const keptFile = path. join (stagingDir, `customers.rechecked.${ stamp }.tmp` );
717 const hollowFile = path. join (stagingDir, `customers-withheld-identity.${ stamp }.tmp` );
718 await fsp. mkdir (stagingDir, { recursive: true });
719 const kept = fs. createWriteStream (keptFile, { encoding: 'utf8' });
720 const hollow = fs. createWriteStream (hollowFile, { encoding: 'utf8' });
721 let keptCount = 0 ;
722 let hollowCount = 0 ;
723 try {
724 for await ( const result of recheckCustomers (records, state.withheld)) {
725 if (result.kind === 'keep' ) { await writeLine (kept, result.record); keptCount += 1 ; }
726 else { await writeLine (hollow, result.record); hollowCount += 1 ; }
727 }
728 kept. end ();
729 hollow. end ();
730 await Promise . all ([ finished (kept), finished (hollow)]);
731 } catch (error) {
732 kept. destroy ();
733 hollow. destroy ();
734 await Promise . all ([fsp. rm (keptFile, { force: true }), fsp. rm (hollowFile, { force: true })]);
735 throw error;
736 }
737
738 const result = {
739 ok: true ,
740 recordsFile: keptFile,
741 recordCount: keptCount,
742 sidecarFiles: hollowCount ? { 'customers-withheld-identity' : hollowFile } : {},
743 manifest: { withheldIdentityCount: hollowCount },
744 };
745 if ( ! hollowCount) {
746 await fsp. rm (hollowFile, { force: true });
747 return result;
748 }
749
750 // The coverage row points at this sidecar while the extract is stopped for mapping, before the
751 // runner's final join. Keep the staged copy too; it becomes the current complete extract's
752 // sidecar after mapping resumes.
753 await copyFileAtomic (hollowFile, path. join (projectDir, 'data/source-extract/customers-withheld-identity.ndjson' ));
754
755 const coverageId = 'shopify:customers-withheld-identity' ;
756 const row = {
757 coverageId,
758 platform: 'shopify' ,
759 capability: 'crm.contact' ,
760 sourceEntity: 'customers-withheld-identity' ,
761 routes: [ 'customers' ],
762 reason: 'auth-gated' ,
763 evidence: { artifact: EXTRACT_MANIFEST_FILE , pointer: `withheldIdentityCount=${ hollowCount }; ids=data/source-extract/customers-withheld-identity.ndjson` },
764 recoverability: 'requires-development' ,
765 disposition: null ,
766 };
767 await upsertAdapterCoverageRows (projectDir, { adapter: 'shopify' , rows: [row] });
768 if ( ! ( await coverageRowDispositioned (projectDir, coverageId))) {
769 result.stop = {
770 code: 'shopify_protected_fields_withheld' ,
771 message: `Shopify extract held back ${ hollowCount } customer(s) with no readable identity; ${ coverageId } was recorded for a mapping decision.` ,
772 details: { coverageId, withheldIdentityCount: hollowCount },
773 };
774 }
775 return result;
776 }
777
778 function createBulkOptions ({ projectDir , sessionCheck , probeState }) {
779 const initial = normalizeProbeState (probeState);
780 return {
781 project : ( name , text ) => {
782 const entity = entityForQuery (name);
783 return entity ? projectQuery (text, entity, initial.withheld[entity]) : text;
784 },
785 handlers: {
786 denied : ( context ) => denialHandler (context, { sessionCheck }),
787 reassemble: { customers: customerRecheckHook },
788 },
789 deniedReset,
790 };
791 }
792
793 // ---------------------------------------------------------------------------------------------
794 // Withheld export attestation and completion-report action
795 // ---------------------------------------------------------------------------------------------
796
797 function isWithheldCoverageId ( coverageId ) {
798 return / ^ shopify:(?:customer | order)-(?:name | email | phone | address) $ / . test ( String (coverageId))
799 || coverageId === 'shopify:customers-withheld-identity' ;
800 }
801
802 function latestReadAt ( manifest ) {
803 const times = [];
804 for ( const entry of Object. values ((manifest && manifest.queries) || {})) {
805 for ( const value of Object. values ((entry && entry.readAt) || {})) {
806 if ( typeof value === 'string' && Number. isFinite (Date. parse (value))) times. push (value);
807 }
808 }
809 times. sort (( a , b ) => Date. parse (a) - Date. parse (b));
810 return times[times. length - 1 ] || null ;
811 }
812
813 function validWithheldConfirmation ( value ) {
814 return Boolean (value
815 && value.schemaVersion === WITHHELD_EXPORT_SCHEMA_VERSION
816 && value.confirmation === WITHHELD_EXPORT_CONFIRMATION
817 && typeof value.confirmedAt === 'string'
818 && Number. isFinite (Date. parse (value.confirmedAt))
819 && typeof value.afterReadAt === 'string'
820 && Number. isFinite (Date. parse (value.afterReadAt))
821 && Array. isArray (value.coverageIds)
822 && value.coverageIds. every (isWithheldCoverageId));
823 }
824
825 async function confirmWithheldExport ( projectDir , confirmation , { now = () => new Date (). toISOString () } = {}) {
826 if (confirmation !== WITHHELD_EXPORT_CONFIRMATION ) throw new Error ( `confirmation must be ${ WITHHELD_EXPORT_CONFIRMATION }` );
827 const coverage = await readJson (path. join (projectDir, SOURCE_COVERAGE_FILE ), null );
828 const coverageIds = coverage && Array. isArray (coverage.rows)
829 ? coverage.rows. map (( row ) => row.coverageId). filter (isWithheldCoverageId). sort ()
830 : [];
831 if ( ! coverageIds. length ) throw new Error ( 'no withheld Shopify detail coverage row exists; nothing was confirmed' );
832 const manifest = await readJson (path. join (projectDir, EXTRACT_MANIFEST_FILE ), null );
833 if ( ! manifest || manifest.status !== 'complete' ) throw new Error ( 'the current Shopify extract manifest is not complete; confirm the export only after the final read' );
834 const afterReadAt = latestReadAt (manifest);
835 if ( ! afterReadAt) throw new Error ( 'the complete Shopify extract manifest has no readAt timestamp' );
836 const file = path. join (projectDir, WITHHELD_EXPORT_FILE );
837 const existing = await readJson (file, undefined ). catch (() => Symbol. for ( 'unreadable' ));
838 if (existing === Symbol. for ( 'unreadable' ) || (existing !== null && ! validWithheldConfirmation (existing))) {
839 await fsp. rename (file, `${ file }.unreadable-${ now () }` );
840 }
841 const record = {
842 schemaVersion: WITHHELD_EXPORT_SCHEMA_VERSION ,
843 confirmedAt: now (),
844 confirmation,
845 coverageIds,
846 afterReadAt,
847 };
848 await writeJsonAtomic (file, record);
849 return record;
850 }
851
852 module . exports = {
853 KINDS,
854 ENTITIES,
855 ADDRESS_FIELDS,
856 ADDRESS_UNPROTECTED_FIELDS,
857 PROTECTED_FIELDS,
858 UNPROTECTED_FIELDS,
859 COMPOSITE_FIELDS,
860 KIND_PROBES_FILE,
861 WITHHELD_EXPORT_FILE,
862 WITHHELD_EXPORT_CONFIRMATION,
863 WITHHELD_EXPORT_CODE,
864 probeQueryFile,
865 probeQuery,
866 logicalPath,
867 selectedLeafPaths,
868 projectQuery,
869 classifyProbe,
870 readProbeState,
871 writeProbeState,
872 protectedCoverageRow,
873 captureFailedRow,
874 runDiscoveryProbes,
875 resetProbeEntities,
876 denialHandler,
877 deniedReset,
878 isHollowCustomer,
879 recheckCustomers,
880 customerRecheckHook,
881 createBulkOptions,
882 isWithheldCoverageId,
883 latestReadAt,
884 validWithheldConfirmation,
885 confirmWithheldExport,
886 };