Setting the file. One moment.
CSV Fileset · Rp Source CSV · wix/skills · Skills Docs
Back to the top of the page
Number 17.3
Position 3 of 11
Type JavaScript
Size 16 KB
Lines 482 lib/ csv-fileset.js
JavaScript · 482 lines · 16 KB
normalizeHeaderName
}
=
require
(
'./csv-parse.js'
);
14
15 const CONCAT_POLICY = {
16 STRICT_SET: 'strict-set' ,
17 UNION: 'union' ,
18 };
19
20 const CONCAT_MIN_JACCARD = 0.9 ;
21 const ROLE_HEADER_MIN_HITS = 2 ;
22
23 // Whole-token filename hints. Matched against tokens, never substrings, so
24 // `product-images.csv` cannot be read as a product file.
25 const GENERIC_ROLE_TOKENS = new Map (Object. entries ({
26 product: 'product' ,
27 products: 'product' ,
28 item: 'product' ,
29 items: 'product' ,
30 sku: 'product' ,
31 skus: 'product' ,
32 catalog: 'product' ,
33 inventory: 'product' ,
34 variant: 'variant' ,
35 variants: 'variant' ,
36 customer: 'customer' ,
37 customers: 'customer' ,
38 user: 'customer' ,
39 users: 'customer' ,
40 member: 'customer' ,
41 members: 'customer' ,
42 contact: 'customer' ,
43 contacts: 'customer' ,
44 order: 'order' ,
45 orders: 'order' ,
46 sale: 'order' ,
47 sales: 'order' ,
48 transaction: 'order' ,
49 transactions: 'order' ,
50 category: 'category' ,
51 categories: 'category' ,
52 collection: 'category' ,
53 collections: 'category' ,
54 taxonomy: 'category' ,
55 image: 'media' ,
56 images: 'media' ,
57 media: 'media' ,
58 photo: 'media' ,
59 photos: 'media' ,
60 asset: 'media' ,
61 assets: 'media' ,
62 post: 'post' ,
63 posts: 'post' ,
64 blog: 'post' ,
65 article: 'post' ,
66 articles: 'post' ,
67 }));
68
69 const GENERIC_FILENAME_NOISE = new Set ([
70 'export' , 'exports' , 'exported' , 'data' , 'full' , 'all' , 'backup' , 'sample' ,
71 'csv' , 'part' , 'parts' , 'final' , 'copy' , 'new' , 'wc' , 'shopify' ,
72 ]);
73
74 const ROLE_HEADER_ANCHORS = {
75 product: [ 'sku' , 'price' , 'product name' , 'title' , 'handle' , 'product code/sku' , 'regular price' ],
76 customer: [ 'email' , 'first name' , 'last name' , 'phone' , 'accepts marketing' ],
77 order: [ 'order number' , 'order id' , 'order date' , 'financial status' , 'fulfillment status' , 'line item name' ],
78 category: [ 'category name' , 'parent category' , 'category id' , 'category slug' ],
79 media: [ 'image url' , 'file name' , 'alt text' , 'image src' ],
80 };
81
82 function normalizeBasename ( filePath ) {
83 const base = path. basename ( String (filePath));
84 const withoutExtension = base. replace ( / \. [a-z0-9] +$ / i , '' );
85 // Strip an ISO date stamp before tokenizing so it is not read as a part index.
86 const withoutDate = withoutExtension. replace ( / \d {4} [-_]\d {2} [-_]\d {2} / g , ' ' );
87 const tokens = withoutDate
88 . toLowerCase ()
89 . split ( / [ ^ a-z0-9] + / )
90 . filter (Boolean)
91 // Split letter/digit boundaries so `part10` becomes ['part', '10'].
92 . flatMap (( token ) => token. split ( /(?<= [a-z] )(?= \d ) | (?<= \d )(?= [a-z] )/ ));
93
94 const numericTokens = tokens. filter (( token ) => / ^ \d +$ / . test (token));
95 const partIndex = tokens. length > 1 && numericTokens. length > 0
96 ? Number. parseInt (numericTokens[numericTokens. length - 1 ], 10 )
97 : null ;
98
99 return {
100 basename: base,
101 stem: withoutExtension,
102 normalizedStem: normalizeHeaderName (withoutExtension),
103 tokens,
104 partIndex,
105 };
106 }
107
108 function assignRoleByFilename ( filePath , { vendorProfile = null } = {}) {
109 const { normalizedStem , tokens } = normalizeBasename (filePath);
110
111 if (vendorProfile && Array. isArray (vendorProfile.fileRoles)) {
112 let best = null ;
113 for ( const entry of vendorProfile.fileRoles) {
114 for ( const hint of entry.filenameHints || []) {
115 const normalizedHint = normalizeHeaderName (hint);
116 if (normalizedHint && normalizedStem. includes (normalizedHint)) {
117 if ( ! best || normalizedHint. length > best.hintLength) {
118 best = { role: entry.role, hintLength: normalizedHint. length , hint };
119 }
120 }
121 }
122 }
123 if (best) {
124 return { role: best.role, roleSource: 'vendor-filename-hint' , evidence: `filename matches the vendor hint "${ best . hint }"` };
125 }
126 }
127
128 // Last matching token wins: in `product-images` the head noun is the qualifier
129 // and the tail noun is the entity.
130 const meaningful = tokens. filter (( token ) => ! GENERIC_FILENAME_NOISE . has (token) && ! / ^ \d +$ / . test (token));
131 for ( let i = meaningful. length - 1 ; i >= 0 ; i -= 1 ) {
132 const role = GENERIC_ROLE_TOKENS . get (meaningful[i]);
133 if (role) {
134 return { role, roleSource: 'filename-generic' , evidence: `filename token "${ meaningful [ i ] }"` };
135 }
136 }
137
138 return null ;
139 }
140
141 function assignRoleByHeader ( headerColumns ) {
142 const header = new Set ((headerColumns || []). map (( column ) => normalizeHeaderName (column)));
143 const scored = Object. entries ( ROLE_HEADER_ANCHORS )
144 . map (([ role , anchors ]) => ({
145 role,
146 hits: anchors. filter (( anchor ) => header. has ( normalizeHeaderName (anchor))),
147 }))
148 . sort (( a , b ) => b.hits. length - a.hits. length || a.role. localeCompare (b.role));
149
150 const winner = scored[ 0 ];
151 const runnerUp = scored[ 1 ];
152 if ( ! winner || winner.hits. length < ROLE_HEADER_MIN_HITS ) {
153 return null ;
154 }
155 if (runnerUp && winner.hits. length - runnerUp.hits. length < 1 ) {
156 return null ;
157 }
158 return {
159 role: winner.role,
160 roleSource: 'header' ,
161 evidence: `header anchors: ${ winner . hits . join ( ', ' ) }` ,
162 };
163 }
164
165 // files: [{ file, header, vendor?, profile?, rowCount? }]
166 function assignRoles ( files , { explicitRoles = {} } = {}) {
167 return files. map (( entry ) => {
168 const explicit = explicitRoles[entry.file] || explicitRoles[path. basename (entry.file)];
169 if (explicit) {
170 return { ... entry, role: explicit, roleSource: 'explicit' , roleEvidence: 'stated by the user' , suggestedRole: null };
171 }
172
173 const byFilename = assignRoleByFilename (entry.file, { vendorProfile: entry.profile || null });
174 if (byFilename) {
175 return { ... entry, role: byFilename.role, roleSource: byFilename.roleSource, roleEvidence: byFilename.evidence, suggestedRole: null };
176 }
177
178 const byHeader = assignRoleByHeader (entry.header);
179 if (byHeader) {
180 return { ... entry, role: byHeader.role, roleSource: byHeader.roleSource, roleEvidence: byHeader.evidence, suggestedRole: null };
181 }
182
183 // Never invent a role from the filename stem — an entity map full of
184 // `weird-export` entities is worse than one honest question.
185 return {
186 ... entry,
187 role: 'unknown' ,
188 roleSource: 'none' ,
189 roleEvidence: 'no vendor hint, filename token, or header anchor matched' ,
190 suggestedRole: null ,
191 };
192 });
193 }
194
195 function headerCompatibility ( headerA , headerB ) {
196 const normalizedA = (headerA || []). map (( column ) => normalizeHeaderName (column));
197 const normalizedB = (headerB || []). map (( column ) => normalizeHeaderName (column));
198 const setA = new Set (normalizedA);
199 const setB = new Set (normalizedB);
200
201 const onlyInA = (headerA || []). filter (( column ) => ! setB. has ( normalizeHeaderName (column)));
202 const onlyInB = (headerB || []). filter (( column ) => ! setA. has ( normalizeHeaderName (column)));
203 const intersection = [ ... setA]. filter (( column ) => setB. has (column)). length ;
204 const union = new Set ([ ... setA, ... setB]).size || 1 ;
205
206 const sameSet = onlyInA. length === 0 && onlyInB. length === 0 ;
207 const sameOrder = sameSet && normalizedA. join ( ' ' ) === normalizedB. join ( ' ' );
208
209 return {
210 // Case, whitespace, punctuation and BOM differences normalize away: names
211 // are the identity, so those files are the same file shape.
212 identical: sameOrder,
213 reordered: sameSet && ! sameOrder,
214 subset: onlyInA. length === 0 && onlyInB. length > 0 ,
215 superset: onlyInA. length > 0 && onlyInB. length === 0 ,
216 jaccard: Number ((intersection / union). toFixed ( 4 )),
217 onlyInA,
218 onlyInB,
219 };
220 }
221
222 function headerRelationOf ( compatibility ) {
223 if (compatibility.identical) {
224 return 'identical' ;
225 }
226 if (compatibility.reordered) {
227 return 'reordered' ;
228 }
229 if (compatibility.subset) {
230 return 'subset' ;
231 }
232 if (compatibility.superset) {
233 return 'superset' ;
234 }
235 return 'divergent' ;
236 }
237
238 // Numeric-suffix-aware: plain lexicographic sorting puts `part10` before
239 // `part2`, which silently reorders a ten-part export.
240 function orderParts ( files ) {
241 return [ ... files]. sort (( a , b ) => {
242 const nameA = normalizeBasename (a.file);
243 const nameB = normalizeBasename (b.file);
244 const stemA = nameA.tokens. filter (( token ) => ! / ^ \d +$ / . test (token)). join ( '-' );
245 const stemB = nameB.tokens. filter (( token ) => ! / ^ \d +$ / . test (token)). join ( '-' );
246 if (stemA !== stemB) {
247 return stemA. localeCompare (stemB);
248 }
249 const indexA = nameA.partIndex === null ? - 1 : nameA.partIndex;
250 const indexB = nameB.partIndex === null ? - 1 : nameB.partIndex;
251 if (indexA !== indexB) {
252 return indexA - indexB;
253 }
254 return String (a.file). localeCompare ( String (b.file));
255 });
256 }
257
258 function columnOrderMap ( header ) {
259 const map = {};
260 (header || []). forEach (( column , index ) => {
261 map[column] = index;
262 });
263 return map;
264 }
265
266 function groupConcatenable ( entries , { policy = CONCAT_POLICY . STRICT_SET , fileOrder = 'natural' } = {}) {
267 const conflicts = [];
268 const byRole = new Map ();
269 for ( const entry of entries) {
270 if ( ! byRole. has (entry.role)) {
271 byRole. set (entry.role, []);
272 }
273 byRole. get (entry.role). push (entry);
274 }
275
276 const streams = [];
277 for ( const role of [ ... byRole. keys ()]. sort ()) {
278 const roleEntries = fileOrder === 'stated' ? byRole. get (role) : orderParts (byRole. get (role));
279
280 const vendors = new Set (roleEntries. map (( entry ) => entry.vendor). filter (( vendor ) => vendor && vendor !== 'custom' ));
281 if (vendors.size > 1 ) {
282 conflicts. push ({
283 kind: 'mixed-vendor-role' ,
284 role,
285 vendors: [ ... vendors]. sort (),
286 files: roleEntries. map (( entry ) => entry.file),
287 halt: true ,
288 message: `role "${ role }" is carried by files from different vendors (${ [ ... vendors ]. sort (). join ( ', ' ) }); resolve which export is authoritative before continuing` ,
289 });
290 }
291
292 let current = null ;
293 for ( const entry of roleEntries) {
294 if ( ! current) {
295 current = {
296 role,
297 primary: entry.file,
298 vendor: entry.vendor || null ,
299 header: entry.header,
300 parts: [{
301 file: entry.file,
302 partOf: null ,
303 headerRelation: 'primary' ,
304 columnOrderMap: columnOrderMap (entry.header),
305 rowCount: entry.rowCount === undefined ? null : entry.rowCount,
306 rowOffset: 0 ,
307 }],
308 driftAcrossParts: [],
309 };
310 streams. push (current);
311 continue ;
312 }
313
314 const compatibility = headerCompatibility (current.header, entry.header);
315 const relation = headerRelationOf (compatibility);
316 const rowOffset = current.parts. reduce (
317 ( sum , part ) => (sum === null || part.rowCount === null ? null : sum + part.rowCount),
318 0 ,
319 );
320
321 if (compatibility.jaccard < CONCAT_MIN_JACCARD ) {
322 // Same role, unrelated shape: two different files that collided on a
323 // role, not two parts of one export.
324 conflicts. push ({
325 kind: 'role-collision' ,
326 role,
327 files: [current.primary, entry.file],
328 jaccard: compatibility.jaccard,
329 halt: true ,
330 message: `"${ entry . file }" and "${ current . primary }" both look like ${ role } files but share only ${ Math . round ( compatibility . jaccard * 100 ) }% of their columns` ,
331 });
332 streams. push ({
333 role,
334 primary: entry.file,
335 vendor: entry.vendor || null ,
336 header: entry.header,
337 parts: [{
338 file: entry.file,
339 partOf: null ,
340 headerRelation: 'primary' ,
341 columnOrderMap: columnOrderMap (entry.header),
342 rowCount: entry.rowCount === undefined ? null : entry.rowCount,
343 rowOffset: 0 ,
344 }],
345 driftAcrossParts: [],
346 });
347 continue ;
348 }
349
350 if (relation === 'identical' || relation === 'reordered' ) {
351 current.parts. push ({
352 file: entry.file,
353 partOf: current.primary,
354 headerRelation: relation,
355 columnOrderMap: columnOrderMap (entry.header),
356 rowCount: entry.rowCount === undefined ? null : entry.rowCount,
357 rowOffset,
358 });
359 continue ;
360 }
361
362 // Subset/superset is legitimate (a re-export after the vendor added a
363 // column) but changes what a blank cell means for half the rows, so it is
364 // never silent: halt with the diff, or concatenate under an explicit
365 // union policy that records the drift.
366 if (policy === CONCAT_POLICY . UNION ) {
367 current.driftAcrossParts. push ({
368 file: entry.file,
369 extraInAdditionalPart: compatibility.onlyInB,
370 missingInAdditionalPart: compatibility.onlyInA,
371 });
372 current.parts. push ({
373 file: entry.file,
374 partOf: current.primary,
375 headerRelation: relation,
376 columnOrderMap: columnOrderMap (entry.header),
377 rowCount: entry.rowCount === undefined ? null : entry.rowCount,
378 rowOffset,
379 });
380 continue ;
381 }
382
383 conflicts. push ({
384 kind: 'header-superset' ,
385 role,
386 files: [current.primary, entry.file],
387 extraInAdditionalPart: compatibility.onlyInB,
388 missingInAdditionalPart: compatibility.onlyInA,
389 halt: true ,
390 message: `"${ entry . file }" has the same role as "${ current . primary }" but a different column set; concatenating would leave those columns blank for half the rows. Re-run with --allow-header-superset to union them, or keep them as separate streams.` ,
391 });
392 streams. push ({
393 role,
394 primary: entry.file,
395 vendor: entry.vendor || null ,
396 header: entry.header,
397 parts: [{
398 file: entry.file,
399 partOf: null ,
400 headerRelation: 'primary' ,
401 columnOrderMap: columnOrderMap (entry.header),
402 rowCount: entry.rowCount === undefined ? null : entry.rowCount,
403 rowOffset: 0 ,
404 }],
405 driftAcrossParts: [],
406 });
407 }
408 }
409
410 return { streams, conflicts };
411 }
412
413 function resolveFileSet ( files , { explicitRoles = {}, policy = CONCAT_POLICY . STRICT_SET , fileOrder = 'natural' } = {}) {
414 const withRoles = assignRoles (files, { explicitRoles });
415 const { streams , conflicts } = groupConcatenable (withRoles, { policy, fileOrder });
416
417 const unknownRoles = withRoles. filter (( entry ) => entry.role === 'unknown' );
418 for ( const entry of unknownRoles) {
419 conflicts. push ({
420 kind: 'unknown-role' ,
421 role: 'unknown' ,
422 files: [entry.file],
423 halt: true ,
424 message: `could not determine which entity "${ entry . file }" carries; ask the user and re-run with --role ${ path . basename ( entry . file ) }:<role>` ,
425 });
426 }
427
428 const roleByFile = new Map (withRoles. map (( entry ) => [entry.file, entry]));
429 const sourceFiles = [];
430 for ( const stream of streams) {
431 for ( const part of stream.parts) {
432 const entry = roleByFile. get (part.file);
433 sourceFiles. push ({
434 file: part.file,
435 role: stream.role,
436 vendor: entry ? entry.vendor || null : null ,
437 roleSource: entry ? entry.roleSource : null ,
438 roleEvidence: entry ? entry.roleEvidence : null ,
439 partOf: part.partOf,
440 headerRelation: part.headerRelation,
441 rowOffset: part.rowOffset,
442 });
443 }
444 }
445
446 return {
447 streams,
448 sourceFiles,
449 conflicts,
450 halt: conflicts. some (( conflict ) => conflict.halt),
451 };
452 }
453
454 // Shape D. The schema is designed now (sourceMeta.joins / sourceMeta.integrity
455 // in source-schema.example.json) but detection lands in v1.1, implemented
456 // against a real normalized bundle rather than a synthetic fixture.
457 function inferJoins () {
458 return {
459 joins: [],
460 integrity: null ,
461 deferred: true ,
462 reason: 'cross-file join inference and referential-integrity checks land in v1.1 (spec 0009 → Scope)' ,
463 };
464 }
465
466 module . exports = {
467 CONCAT_POLICY,
468 CONCAT_MIN_JACCARD,
469 ROLE_HEADER_MIN_HITS,
470 GENERIC_ROLE_TOKENS,
471 ROLE_HEADER_ANCHORS,
472 normalizeBasename,
473 assignRoleByFilename,
474 assignRoleByHeader,
475 assignRoles,
476 headerCompatibility,
477 headerRelationOf,
478 orderParts,
479 groupConcatenable,
480 resolveFileSet,
481 inferJoins,
482 };