Setting the file. One moment.
Sampled Ids Batch · Rp Source Wordpress · wix/skills · Skills Docs
Back to the top of the page 39.41
PLUGIN READ PLANS
lib/sampled-ids-batch.js
lib/ sampled-ids-batch.js
JavaScript · 157 lines · 7 KB
14
// concern) and merge, deduplicating by a caller-supplied record key;
15 // 5. stop and report the exact failure on the first page or batch that cannot be read,
16 // rather than returning a partial/undercounted result silently.
17 //
18 // wp-discovery.js and any rp-import-codegen-generated reader for a requestOverride entity with
19 // a $SAMPLED_IDS placeholder both drive this same algorithm, so "does this mechanism scale past
20 // a handful of records" has exactly one implementation to get right.
21
22 const { fetchJson , shouldContinueCollectionPaging } = require ( './wp-http.js' );
23
24 const DEFAULT_PAGE_SIZE = 100 ;
25 const DEFAULT_BATCH_SIZE = 50 ;
26
27 function chunk ( items , size ) {
28 const batches = [];
29 for ( let i = 0 ; i < items. length ; i += size) {
30 batches. push (items. slice (i, i + size));
31 }
32 return batches;
33 }
34
35 // Pages a plain GET collection route to exhaustion, collecting `idField` off every record.
36 // Returns { ok: true, ids } or { ok: false, ids, failure }. On failure, `ids` holds whatever
37 // was collected before the failing page, but callers must treat a failed run as unusable —
38 // never as "the true, if undercounted, id list" — a partial id list understates the dependent
39 // query rather than correctly reporting it as unknown.
40 async function collectAllIds ({
41 baseUrl , headers , route , idField = 'id' , pageSize = DEFAULT_PAGE_SIZE ,
42 query = {}, timeoutMs , progress , fetchJsonFn = fetchJson,
43 }) {
44 const ids = [];
45 let page = 1 ;
46 for (;;) {
47 const response = await fetchJsonFn (baseUrl, route, {
48 headers,
49 method: 'GET' ,
50 query: { ... query, page, per_page: pageSize },
51 timeoutMs,
52 progress,
53 progressContext: { step: 'paginate-dependency' , entity: route, page },
54 });
55 if ( ! response.ok) {
56 return { ok: false , ids, failure: { route, page, status: response.status, statusText: response.statusText } };
57 }
58 // A 2xx whose body is not an array (an envelope, an error object, malformed JSON) is not an
59 // empty page: coercing it to [] would end pagination and report an exact zero instead of
60 // deferring the dependent query as unknown.
61 if ( ! Array. isArray (response.json)) {
62 return { ok: false , ids, failure: { route, page, status: response.status, statusText: response.statusText, reason: 'malformed-response' } };
63 }
64 const items = response.json;
65 for ( const item of items) {
66 const id = item && item[idField];
67 if (id !== undefined && id !== null ) ids. push (id);
68 }
69 if ( ! shouldContinueCollectionPaging ({ responseHeaders: response.headers, page, perPage: pageSize, itemCount: items. length })) {
70 return { ok: true , ids };
71 }
72 page += 1 ;
73 }
74 }
75
76 // Deduplicates `records` by `recordKeyField`, preserving first-seen order. A record missing
77 // the key field is kept as-is (never dropped) since there is nothing to dedupe it against.
78 function dedupeByKey ( records , recordKeyField ) {
79 if ( ! recordKeyField) return records;
80 const seen = new Set ();
81 const deduped = [];
82 for ( const record of records) {
83 const key = record && record[recordKeyField];
84 if (key === undefined || key === null ) {
85 deduped. push (record);
86 continue ;
87 }
88 if (seen. has (key)) continue ;
89 seen. add (key);
90 deduped. push (record);
91 }
92 return deduped;
93 }
94
95 // Splits `ids` into bounded batches and issues one request per batch against `route`, building
96 // each batch's body with `buildBody(batchIds)` and normalizing its RAW, unmodified JSON
97 // response with `normalizeBatch(rawJson, { batchIndex })` — envelope resolution / fragment
98 // reassembly are left to the caller so this module stays agnostic of any one profile's response
99 // shape, but that only works if this module hands over the real payload rather than coercing
100 // it: a batch response wrapped in a responseEnvelope is an OBJECT, not an array, and
101 // pre-coercing it to `[]` here would silently discard it before normalizeBatch ever saw it.
102 // `normalizeBatch` must return `{ ok, records }` — `ok: false` means the response did not
103 // resolve to a usable shape (not merely "resolved to zero records", which is a legitimate `ok:
104 // true` outcome for a batch whose ids simply have no matches) and is treated exactly like an
105 // HTTP failure: stop and defer, discarding merged batches from this run, rather than counting
106 // it as zero. Dedupes across all successful batches. Stops and reports the exact failure on the
107 // first batch that cannot be read or normalized; already-merged batches from that run must be
108 // discarded by the caller, not reported as a partial count.
109 async function queryInBatches ({
110 baseUrl , headers , route , method , buildBody , ids , batchSize = DEFAULT_BATCH_SIZE ,
111 normalizeBatch = ( records ) => (Array. isArray (records) ? { ok: true , records } : { ok: false , records: [] }),
112 recordKeyField = 'id' , timeoutMs , progress , fetchJsonFn = fetchJson,
113 }) {
114 const batches = chunk (ids, batchSize);
115 const merged = [];
116 for ( let batchIndex = 0 ; batchIndex < batches. length ; batchIndex += 1 ) {
117 const batchIds = batches[batchIndex];
118 // `headers` is a real Headers instance (wp-http.js buildHeaders) — spreading it would
119 // silently drop every entry including Authorization, since Headers is not a plain object.
120 const withBody = new Headers (headers);
121 withBody. set ( 'content-type' , 'application/json' );
122 const response = await fetchJsonFn (baseUrl, route, {
123 headers: withBody,
124 method,
125 body: JSON . stringify ( buildBody (batchIds)),
126 timeoutMs,
127 progress,
128 progressContext: { step: 'query-dependent-batch' , entity: route, batch: batchIndex + 1 , total: batches. length },
129 });
130 if ( ! response.ok) {
131 return {
132 ok: false ,
133 records: merged,
134 failure: { route, batchIndex, batchSize: batchIds. length , status: response.status, statusText: response.statusText, reason: 'http-error' },
135 };
136 }
137 const normalized = normalizeBatch (response.json, { batchIndex });
138 if ( ! normalized.ok) {
139 return {
140 ok: false ,
141 records: merged,
142 failure: { route, batchIndex, batchSize: batchIds. length , status: response.status, statusText: response.statusText, reason: 'invalid-shape' },
143 };
144 }
145 merged. push ( ... normalized.records);
146 }
147 return { ok: true , records: dedupeByKey (merged, recordKeyField) };
148 }
149
150 module . exports = {
151 DEFAULT_PAGE_SIZE,
152 DEFAULT_BATCH_SIZE,
153 collectAllIds,
154 queryInBatches,
155 dedupeByKey,
156 chunk,
157 };