Setting the file. One moment.
Sampled Ids Batch · Rp Source Wordpress · wix/skills · Skills Docs
Back to the top of the page lib/sampled-ids-batch.js
lib/ sampled-ids-batch.js
JavaScript · 151 lines · 7 KB
14
// concern) and merge, deduplicating by a caller-supplied record key;
15 // 5. stop and report the exact failure on the first page or batch that cannot be read,
16 // rather than returning a partial/undercounted result silently.
17 //
18 // wp-discovery.js and any rp-import-codegen-generated reader for a requestOverride entity with
19 // a $SAMPLED_IDS placeholder both drive this same algorithm, so "does this mechanism scale past
20 // a handful of records" has exactly one implementation to get right.
21
22 const { fetchJson , shouldContinueCollectionPaging } = require ( './wp-http.js' );
23
24 const DEFAULT_PAGE_SIZE = 100 ;
25 const DEFAULT_BATCH_SIZE = 50 ;
26
27 function chunk ( items , size ) {
28 const batches = [];
29 for ( let i = 0 ; i < items. length ; i += size) {
30 batches. push (items. slice (i, i + size));
31 }
32 return batches;
33 }
34
35 // Pages a plain GET collection route to exhaustion, collecting `idField` off every record.
36 // Returns { ok: true, ids } or { ok: false, ids, failure }. On failure, `ids` holds whatever
37 // was collected before the failing page, but callers must treat a failed run as unusable —
38 // never as "the true, if undercounted, id list" — a partial id list understates the dependent
39 // query rather than correctly reporting it as unknown.
40 async function collectAllIds ({
41 baseUrl , headers , route , idField = 'id' , pageSize = DEFAULT_PAGE_SIZE ,
42 query = {}, timeoutMs , progress , fetchJsonFn = fetchJson,
43 }) {
44 const ids = [];
45 let page = 1 ;
46 for (;;) {
47 const response = await fetchJsonFn (baseUrl, route, {
48 headers,
49 method: 'GET' ,
50 query: { ... query, page, per_page: pageSize },
51 timeoutMs,
52 progress,
53 progressContext: { step: 'paginate-dependency' , entity: route, page },
54 });
55 if ( ! response.ok) {
56 return { ok: false , ids, failure: { route, page, status: response.status, statusText: response.statusText } };
57 }
58 const items = Array. isArray (response.json) ? response.json : [];
59 for ( const item of items) {
60 const id = item && item[idField];
61 if (id !== undefined && id !== null ) ids. push (id);
62 }
63 if ( ! shouldContinueCollectionPaging ({ responseHeaders: response.headers, page, perPage: pageSize, itemCount: items. length })) {
64 return { ok: true , ids };
65 }
66 page += 1 ;
67 }
68 }
69
70 // Deduplicates `records` by `recordKeyField`, preserving first-seen order. A record missing
71 // the key field is kept as-is (never dropped) since there is nothing to dedupe it against.
72 function dedupeByKey ( records , recordKeyField ) {
73 if ( ! recordKeyField) return records;
74 const seen = new Set ();
75 const deduped = [];
76 for ( const record of records) {
77 const key = record && record[recordKeyField];
78 if (key === undefined || key === null ) {
79 deduped. push (record);
80 continue ;
81 }
82 if (seen. has (key)) continue ;
83 seen. add (key);
84 deduped. push (record);
85 }
86 return deduped;
87 }
88
89 // Splits `ids` into bounded batches and issues one request per batch against `route`, building
90 // each batch's body with `buildBody(batchIds)` and normalizing its RAW, unmodified JSON
91 // response with `normalizeBatch(rawJson, { batchIndex })` — envelope resolution / fragment
92 // reassembly are left to the caller so this module stays agnostic of any one profile's response
93 // shape, but that only works if this module hands over the real payload rather than coercing
94 // it: a batch response wrapped in a responseEnvelope is an OBJECT, not an array, and
95 // pre-coercing it to `[]` here would silently discard it before normalizeBatch ever saw it.
96 // `normalizeBatch` must return `{ ok, records }` — `ok: false` means the response did not
97 // resolve to a usable shape (not merely "resolved to zero records", which is a legitimate `ok:
98 // true` outcome for a batch whose ids simply have no matches) and is treated exactly like an
99 // HTTP failure: stop and defer, discarding merged batches from this run, rather than counting
100 // it as zero. Dedupes across all successful batches. Stops and reports the exact failure on the
101 // first batch that cannot be read or normalized; already-merged batches from that run must be
102 // discarded by the caller, not reported as a partial count.
103 async function queryInBatches ({
104 baseUrl , headers , route , method , buildBody , ids , batchSize = DEFAULT_BATCH_SIZE ,
105 normalizeBatch = ( records ) => (Array. isArray (records) ? { ok: true , records } : { ok: false , records: [] }),
106 recordKeyField = 'id' , timeoutMs , progress , fetchJsonFn = fetchJson,
107 }) {
108 const batches = chunk (ids, batchSize);
109 const merged = [];
110 for ( let batchIndex = 0 ; batchIndex < batches. length ; batchIndex += 1 ) {
111 const batchIds = batches[batchIndex];
112 // `headers` is a real Headers instance (wp-http.js buildHeaders) — spreading it would
113 // silently drop every entry including Authorization, since Headers is not a plain object.
114 const withBody = new Headers (headers);
115 withBody. set ( 'content-type' , 'application/json' );
116 const response = await fetchJsonFn (baseUrl, route, {
117 headers: withBody,
118 method,
119 body: JSON . stringify ( buildBody (batchIds)),
120 timeoutMs,
121 progress,
122 progressContext: { step: 'query-dependent-batch' , entity: route, batch: batchIndex + 1 , total: batches. length },
123 });
124 if ( ! response.ok) {
125 return {
126 ok: false ,
127 records: merged,
128 failure: { route, batchIndex, batchSize: batchIds. length , status: response.status, statusText: response.statusText, reason: 'http-error' },
129 };
130 }
131 const normalized = normalizeBatch (response.json, { batchIndex });
132 if ( ! normalized.ok) {
133 return {
134 ok: false ,
135 records: merged,
136 failure: { route, batchIndex, batchSize: batchIds. length , status: response.status, statusText: response.statusText, reason: 'invalid-shape' },
137 };
138 }
139 merged. push ( ... normalized.records);
140 }
141 return { ok: true , records: dedupeByKey (merged, recordKeyField) };
142 }
143
144 module . exports = {
145 DEFAULT_PAGE_SIZE,
146 DEFAULT_BATCH_SIZE,
147 collectAllIds,
148 queryInBatches,
149 dedupeByKey,
150 chunk,
151 };