Setting the file. One moment.
CSV Discovery · Rp Source CSV · wix/skills · Skills Docs
Back to the top of the page 35.11
Shopify
641
function rawCapturePayload
— line 641
This file
Number 35.1
Position 1 of 12
Type JavaScript
Size 38 KB
Lines 1,027 scripts/ csv-discovery.js
JavaScript · 1,027 lines · 38 KB
=
require
(
'../../../lib/source-coverage.js'
);
7 const {
8 createCurrencyCounter ,
9 mergeCurrencyCounts ,
10 buildCsvSourceRegional ,
11 writeCsvSourceRegional ,
12 } = require ( '../lib/csv-regional.js' );
13
14 // Parse, dialect detection, and type sniffing live in the shared adapter lib so
15 // the bulk-extract reader generated by rp-import-codegen inherits exactly the
16 // same behavior instead of re-deriving it.
17 const {
18 readHeaderRow ,
19 readTailSample ,
20 streamRows ,
21 detectEmptyPolicy ,
22 summarizeColumn ,
23 rowWidthReport ,
24 normalizeHeaderName ,
25 } = require ( '../lib/csv-parse.js' );
26 const {
27 loadVendorProfiles ,
28 detectVendor ,
29 diffProfile ,
30 prefillColumnMap ,
31 } = require ( '../lib/csv-fingerprint.js' );
32 const {
33 classifyLayout ,
34 buildGroups ,
35 columnIndex ,
36 deriveColumnRoles ,
37 deriveColumnValues ,
38 detectDerivedCandidates ,
39 } = require ( '../lib/csv-layout.js' );
40 const {
41 CONCAT_POLICY ,
42 resolveFileSet ,
43 inferJoins ,
44 } = require ( '../lib/csv-fileset.js' );
45
46 const DEFAULT_HEAD_ROWS = 5 ;
47 const DEFAULT_TAIL_ROWS = 3 ;
48 const DEFAULT_SCAN_ROWS = 5000 ;
49
50 let progress;
51
52 function printUsage () {
53 console. log ( `Usage:
54 node csv-discovery.js --file <path> [--file <path> ...] --out-dir <dir> [options]
55
56 Required:
57 --file <path> Input CSV. Repeatable — pass the whole file set in one run
58 so roles and split files are resolved together.
59 --out-dir <dir> Directory to write the raw capture into
60
61 Optional:
62 --vendor <name> User-stated vendor; overrides fingerprint detection.
63 One of: shopify, woocommerce, magento, bigcommerce, custom
64 --delimiter <char> Force the delimiter (default: auto-detect , ; \\ t |)
65 --encoding <name> Force the encoding (default: auto-detect UTF-8 / BOM)
66 --role <path>:<role> Force a file's role. Repeatable.
67 --file-order stated|natural Order of split files (default: natural)
68 --allow-header-superset Union split files whose headers differ instead of halting
69 --head-rows <n> Sample rows from the head. Default: ${ DEFAULT_HEAD_ROWS }
70 --tail-rows <n> Sample rows from the tail. Default: ${ DEFAULT_TAIL_ROWS }
71 --scan-rows <n> Rows retained for inference. Default: ${ DEFAULT_SCAN_ROWS }
72 --currency <code> Store currency the person supplied (ISO 4217, e.g. USD).
73 --time-zone <name> Store time zone the person supplied (IANA name).
74 --language <code> Store primary language the person supplied (e.g. en or en-US).
75 --country <code> Store country the person supplied (ISO 3166-1 alpha-2).
76 Each also reads CSV_STORE_CURRENCY / CSV_STORE_TIME_ZONE /
77 CSV_STORE_LANGUAGE / CSV_STORE_COUNTRY. A value left out is
78 recorded as unknown, never guessed.
79 --progress-log <path> Append progress NDJSON records to this file.
80 --help Show this help text
81
82 Example:
83 node csv-discovery.js \\
84 --file ~/exports/products_export.csv \\
85 --out-dir migrations/acme/data/csv-discovery
86 ` );
87 }
88
89 function parseArgs ( argv ) {
90 const args = {
91 files: [],
92 explicitRoles: {},
93 vendor: null ,
94 delimiter: null ,
95 encoding: null ,
96 currency: null ,
97 timeZone: null ,
98 language: null ,
99 country: null ,
100 fileOrder: 'natural' ,
101 allowHeaderSuperset: false ,
102 headRows: DEFAULT_HEAD_ROWS ,
103 tailRows: DEFAULT_TAIL_ROWS ,
104 scanRows: DEFAULT_SCAN_ROWS ,
105 outDir: null ,
106 help: false ,
107 };
108
109 for ( let i = 0 ; i < argv. length ; i += 1 ) {
110 const arg = argv[i];
111 const next = argv[i + 1 ];
112
113 switch (arg) {
114 case '--help' :
115 case '-h' :
116 args.help = true ;
117 break ;
118 case '--file' :
119 args.files. push (next);
120 i += 1 ;
121 break ;
122 case '--out-dir' :
123 args.outDir = next;
124 i += 1 ;
125 break ;
126 case '--vendor' :
127 args.vendor = next;
128 i += 1 ;
129 break ;
130 case '--delimiter' :
131 args.delimiter = next === ' \\ t' ? ' \t ' : next;
132 i += 1 ;
133 break ;
134 case '--encoding' :
135 args.encoding = next;
136 i += 1 ;
137 break ;
138 case '--currency' :
139 args.currency = next;
140 i += 1 ;
141 break ;
142 case '--time-zone' :
143 args.timeZone = next;
144 i += 1 ;
145 break ;
146 case '--language' :
147 args.language = next;
148 i += 1 ;
149 break ;
150 case '--country' :
151 args.country = next;
152 i += 1 ;
153 break ;
154 case '--role' : {
155 const splitIndex = String (next). lastIndexOf ( ':' );
156 if (splitIndex === - 1 ) {
157 throw new Error ( `Invalid --role value: ${ next } (expected <path>:<role>)` );
158 }
159 args.explicitRoles[ String (next). slice ( 0 , splitIndex)] = String (next). slice (splitIndex + 1 );
160 i += 1 ;
161 break ;
162 }
163 case '--file-order' :
164 args.fileOrder = next;
165 i += 1 ;
166 break ;
167 case '--allow-header-superset' :
168 args.allowHeaderSuperset = true ;
169 break ;
170 case '--head-rows' :
171 args.headRows = Number. parseInt (next, 10 );
172 i += 1 ;
173 break ;
174 case '--tail-rows' :
175 args.tailRows = Number. parseInt (next, 10 );
176 i += 1 ;
177 break ;
178 case '--scan-rows' :
179 args.scanRows = Number. parseInt (next, 10 );
180 i += 1 ;
181 break ;
182 default :
183 if (arg. startsWith ( '--' )) {
184 throw new Error ( `Unknown argument: ${ arg }` );
185 }
186 }
187 }
188
189 if ( ! args.vendor) {
190 args.vendor = process.env. CSV_VENDOR || null ;
191 }
192 if ( ! args.delimiter) {
193 args.delimiter = process.env. CSV_DELIMITER || null ;
194 }
195 if ( ! args.encoding) {
196 args.encoding = process.env. CSV_ENCODING || null ;
197 }
198 args.currency = args.currency || process.env. CSV_STORE_CURRENCY || null ;
199 args.timeZone = args.timeZone || process.env. CSV_STORE_TIME_ZONE || null ;
200 args.language = args.language || process.env. CSV_STORE_LANGUAGE || null ;
201 args.country = args.country || process.env. CSV_STORE_COUNTRY || null ;
202 if ( ! args.outDir) {
203 args.outDir = process.env. CSV_DISCOVERY_OUT_DIR || null ;
204 }
205 if ( ! Number. isFinite (args.headRows) || args.headRows < 1 ) {
206 args.headRows = DEFAULT_HEAD_ROWS ;
207 }
208 if ( ! Number. isFinite (args.tailRows) || args.tailRows < 0 ) {
209 args.tailRows = DEFAULT_TAIL_ROWS ;
210 }
211 if ( ! Number. isFinite (args.scanRows) || args.scanRows < 1 ) {
212 args.scanRows = DEFAULT_SCAN_ROWS ;
213 }
214 if (args.fileOrder !== 'stated' && args.fileOrder !== 'natural' ) {
215 args.fileOrder = 'natural' ;
216 }
217
218 return args;
219 }
220
221 // One streaming pass per file: full row count, bounded row retention for
222 // inference, per-row shape checks and per-row currency counts. The file is never bulk-loaded.
223 async function scanFile ( filePath , { delimiter , encoding , scanRows , header }) {
224 const rows = [];
225 let rowCount = 0 ;
226 let raggedRows = 0 ;
227 let index = 0 ;
228 const currency = createCurrencyCounter (header);
229
230 for await ( const row of streamRows (filePath, { delimiter, encoding, trackQuoted: true })) {
231 index += 1 ;
232 if (index === 1 ) {
233 continue ; // header
234 }
235 rowCount += 1 ;
236 if (rows. length < scanRows) {
237 rows. push (row);
238 }
239 if ( rowWidthReport (header, row.values).ragged) {
240 raggedRows += 1 ;
241 }
242 currency. add (row.values);
243 }
244
245 return { rows, rowCount, raggedRows, truncated: rowCount > rows. length , currencyCounts: currency. counts () };
246 }
247
248 function summarizeColumns ( header , rows ) {
249 return header. map (( name , columnIdx ) => {
250 const samples = rows. map (( row ) => ({
251 value: row.values[columnIdx] === undefined ? '' : row.values[columnIdx],
252 quoted: Boolean (row.quoted && row.quoted[columnIdx]),
253 }));
254 return summarizeColumn (name, samples);
255 });
256 }
257
258 const PRIVATE_HOST = / ^ (https ? :) ? \/\/ (localhost | 127 \. 0 \. 0 \. 1 | 0 \. 0 \. 0 \. 0 | 10 \. | 192 \. 168 \. | \[ ::1 \] )/ i ;
259
260 // Wix Media import fetches from Wix servers, so a local path or private URL in
261 // an image column is not reachable at import time — the same concern the
262 // WordPress adapter records for localhost sources.
263 const MEDIA_LOCATION = / [ \\ /] | \. (jpe ? g | png | gif | webp | avif | svg | bmp | tiff ?| mp4 | mov | pdf) $ / i ;
264
265 function mediaReachability ( columns ) {
266 const notes = [];
267 for ( const column of columns) {
268 const looksLikeMedia = /image | photo | media | picture | asset | file/ i . test (column.name);
269 if ( ! looksLikeMedia || column.examples. length === 0 ) {
270 continue ;
271 }
272 // Only values that look like a location can be unreachable; an "Image
273 // Position" column of 1/2/3 is not a media reference.
274 const locations = column.examples. filter (( example ) => MEDIA_LOCATION . test ( String (example)));
275 if (locations. length === 0 ) {
276 continue ;
277 }
278 const unreachable = locations. filter (( example ) => PRIVATE_HOST . test (example) || ! / ^ https ? : \/\/ / i . test (example));
279 if (unreachable. length > 0 ) {
280 notes. push ({
281 column: column.name,
282 example: unreachable[ 0 ],
283 note: 'not publicly fetchable by Wix; rewrite with CSV_MEDIA_URL_REWRITE_FROM/TO, expose the files publicly, or skip/defer media' ,
284 });
285 }
286 }
287 return notes;
288 }
289
290 function uniqueEntityName ( name , taken ) {
291 if ( ! taken. has (name)) {
292 taken. add (name);
293 return name;
294 }
295 let suffix = 2 ;
296 while (taken. has ( `${ name }-${ suffix }` )) {
297 suffix += 1 ;
298 }
299 const unique = `${ name }-${ suffix }` ;
300 taken. add (unique);
301 return unique;
302 }
303
304 function distinctNonBlank ( rows , index ) {
305 const values = new Set ();
306 for ( const row of rows) {
307 const value = row.values[index];
308 if (value !== undefined && String (value). trim () !== '' ) {
309 values. add (value);
310 }
311 }
312 return values.size;
313 }
314
315 // Turn one resolved stream into entities. A single grouped vendor file yields
316 // several: the parent, its inline children, any declared collection (images),
317 // and one per column-values descriptor.
318 function deriveEntities ( stream , taken ) {
319 const { header , rows , layout , roles , derived , role } = stream;
320 const entities = [];
321 const relations = [];
322
323 const parentName = uniqueEntityName (
324 layout.parentEntity && layout.parentEntity !== 'record' ? layout.parentEntity : role,
325 taken,
326 );
327
328 const childLevels = new Set ((layout.childLevels || []). map (( level ) => normalizeHeaderName (level)));
329 const discriminatorIdx = columnIndex (header, layout.discriminatorColumn);
330 const groups = buildGroups (header, rows, layout);
331
332 let parentOrigin;
333 let parentCount;
334 if (layout.pattern === 'grouped-by-key' ) {
335 parentOrigin = {
336 kind: 'row-group' ,
337 file: stream.primary,
338 groupKey: layout.groupKey,
339 continuation: layout.continuation,
340 };
341 parentCount = groups. length ;
342 } else if (layout.pattern === 'sectioned' ) {
343 parentOrigin = {
344 kind: 'file-rows' ,
345 file: stream.primary,
346 filter: { column: layout.discriminatorColumn, excludeValues: layout.childLevels || [] },
347 };
348 parentCount = rows. filter (( row ) => ! childLevels. has ( normalizeHeaderName (row.values[discriminatorIdx] || '' ))). length ;
349 } else {
350 parentOrigin = { kind: 'file-rows' , file: stream.primary };
351 parentCount = rows. length ;
352 }
353
354 entities. push ({
355 name: parentName,
356 origin: parentOrigin,
357 recordCount: parentCount,
358 inUse: parentCount > 0 ,
359 primaryKey: layout.groupKey || layout.primaryKey || null ,
360 columns: roles.byEntity[layout.parentEntity] || roles.parentColumns,
361 relations: [],
362 });
363
364 if (roles.childColumns. length > 0 && layout.childEntity) {
365 const childName = uniqueEntityName (layout.childEntity, taken);
366 let childCount;
367 let childOrigin;
368 if (layout.pattern === 'sectioned' ) {
369 childCount = rows. filter (( row ) => childLevels. has ( normalizeHeaderName (row.values[discriminatorIdx] || '' ))). length ;
370 childOrigin = {
371 kind: 'file-rows' ,
372 file: stream.primary,
373 filter: { column: layout.discriminatorColumn, includeValues: layout.childLevels || [] },
374 parentRefColumn: layout.parentRefColumn || null ,
375 };
376 } else {
377 // Not `rows.length`: in a grouped file a continuation row can carry only an
378 // extra image, leaving every child column blank. Counting those as variants
379 // inflates the child entity by exactly the number of image-only rows.
380 const childIndexes = roles.childColumns. map (( column ) => columnIndex (header, column)). filter (( index ) => index !== - 1 );
381 childCount = rows. filter (( row ) => childIndexes. some (( index ) => {
382 const value = row.values[index];
383 return value !== undefined && String (value). trim () !== '' ;
384 })). length ;
385 childOrigin = {
386 kind: 'row-group' ,
387 file: stream.primary,
388 groupKey: layout.groupKey,
389 columnGroup: layout.childEntity,
390 };
391 }
392 entities. push ({
393 name: childName,
394 origin: childOrigin,
395 recordCount: childCount,
396 inUse: childCount > 0 ,
397 columns: roles.childColumns,
398 relations: [],
399 });
400 relations. push ({
401 from: parentName,
402 field: childName,
403 targetEntity: childName,
404 cardinality: 'one-to-many' ,
405 evidence: layout.pattern === 'sectioned'
406 ? `layout.discriminatorColumn=${ layout . discriminatorColumn }${ layout . parentRefColumn ? ` + parentRef=${ layout . parentRefColumn }` : ''}`
407 : `layout.groupKey=${ layout . groupKey }` ,
408 });
409 }
410
411 // Overlay-declared collections (Shopify's image columns) — a third bucket the
412 // data alone cannot separate from the variant columns.
413 for ( const [ entity , columns ] of Object. entries (roles.byEntity)) {
414 if (entity === layout.parentEntity || entity === layout.childEntity || columns. length === 0 ) {
415 continue ;
416 }
417 const anchorIdx = columnIndex (header, columns[ 0 ]);
418 const count = anchorIdx === - 1 ? 0 : distinctNonBlank (rows, anchorIdx);
419 const entityName = uniqueEntityName (entity, taken);
420 entities. push ({
421 name: entityName,
422 origin: {
423 kind: 'row-group' ,
424 file: stream.primary,
425 groupKey: layout.groupKey || null ,
426 columnGroup: entity,
427 distinctBy: columns[ 0 ],
428 },
429 recordCount: count,
430 inUse: count > 0 ,
431 columns,
432 relations: [],
433 });
434 relations. push ({
435 from: parentName,
436 field: entityName,
437 targetEntity: entityName,
438 cardinality: 'one-to-many' ,
439 evidence: `layout.columnGroups[${ entity }]` ,
440 });
441 }
442
443 // column-values entities: categories and tags are columns, not files.
444 for ( const descriptor of derived) {
445 if (descriptor.status === 'missing-column' ) {
446 continue ;
447 }
448 const result = deriveColumnValues (header, rows, descriptor, { layout });
449 if (result.missingColumn || result.records. length === 0 ) {
450 continue ;
451 }
452 const entityName = uniqueEntityName (descriptor.entity, taken);
453 entities. push ({
454 name: entityName,
455 origin: { ... result.origin, file: stream.primary },
456 recordCount: result.records. length ,
457 inUse: result.records. length > 0 ,
458 hierarchical: result.hierarchical,
459 columns: [descriptor.fromColumn],
460 sampleRecords: result.records. slice ( 0 , 10 ),
461 relations: [],
462 });
463 relations. push ({
464 from: parentName,
465 field: (descriptor.relation && descriptor.relation.field) || entityName,
466 targetEntity: entityName,
467 cardinality: (descriptor.relation && descriptor.relation.cardinality) || 'many-to-many' ,
468 evidence: `column:${ descriptor . fromColumn }` ,
469 });
470 }
471
472 for ( const relation of relations) {
473 const owner = entities. find (( entity ) => entity.name === relation.from);
474 if (owner) {
475 owner.relations. push ({
476 field: relation.field,
477 targetEntity: relation.targetEntity,
478 cardinality: relation.cardinality,
479 evidence: relation.evidence,
480 });
481 }
482 }
483
484 return entities;
485 }
486
487 function renderEntityFile ( stream , entity , columnsByName ) {
488 const columnRows = (entity.columns || [])
489 . map (( name ) => columnsByName. get (name))
490 . filter (Boolean)
491 . map (( column ) => `| \` ${ column . name } \` | ${ column . type } | ${ column . required ? 'yes' : 'no'} | ${ column . distinctCount }${ column . distinctCapped ? '+' : ''} | ${ column . blankCount } | ${ column . examples . map (( example ) => ` \` ${ String ( example ). slice ( 0 , 60 ) } \` ` ). join ( ', ' ) || '—'} |` )
492 . join ( ' \n ' );
493
494 const relationLines = entity.relations. length > 0
495 ? entity.relations. map (( relation ) => `- \` ${ relation . field } \` → \` ${ relation . targetEntity } \` (${ relation . cardinality }) — evidence: \` ${ relation . evidence } \` ` ). join ( ' \n ' )
496 : '- None' ;
497
498 const derivedBlock = entity.sampleRecords
499 ? ` \n ## Derived records (first ${ entity . sampleRecords . length }) \n\n\`\`\` json \n ${ JSON . stringify ( entity . sampleRecords , null , 2 ) } \n\`\`\`\n `
500 : '' ;
501
502 return `# ${ entity . name } \n\n `
503 + `- Source file: \` ${ path . basename ( stream . primary ) } \`\n `
504 + `- Role: \` ${ stream . role } \`\n `
505 + `- Vendor: \` ${ stream . vendor . vendor } \`\n `
506 + `- Origin: \` ${ entity . origin . kind } \` ${ entity . origin . groupKey ? ` (group key \` ${ entity . origin . groupKey } \` )` : ''}${ entity . origin . column ? ` (column \` ${ entity . origin . column } \` )` : ''} \n `
507 + `- Record count: \` ${ entity . recordCount } \` ${ stream . scan . truncated ? ` (from the first ${ stream . rows . length } scanned rows)` : ''} \n `
508 + `- In use: \` ${ entity . inUse ? 'yes' : 'no'} \`\n `
509 + (entity.hierarchical ? '- Hierarchical: `yes` — a hierarchical source taxonomy mapped to a flat Wix target requires a faithfulness-ledger entry \n ' : '' )
510 + ` \n ## Relations \n ${ relationLines } \n `
511 + ` \n ## Columns \n\n | Column | Type | Required | Distinct | Blank | Examples | \n | --- | --- | --- | ---: | ---: | --- | \n ${ columnRows || '| None | - | - | - | - | - |'} \n `
512 + derivedBlock;
513 }
514
515 function renderIndexFile ( capture ) {
516 const fileRows = capture.sourceFiles
517 . map (( entry ) => `| \` ${ path . basename ( entry . file ) } \` | ${ entry . role } | ${ entry . vendor || 'custom'} | ${ entry . roleSource } | ${ entry . partOf ? ` \` ${ path . basename ( entry . partOf ) } \` ` : '—'} |` )
518 . join ( ' \n ' );
519
520 const streamRowsMd = capture.streams
521 . map (( stream ) => `| \` ${ path . basename ( stream . primary ) } \` | \` ${ stream . layout . pattern } \` | ${ stream . layout . groupKey ? ` \` ${ stream . layout . groupKey } \` ` : ( stream . layout . discriminatorColumn ? ` \` ${ stream . layout . discriminatorColumn } \` ` : '—' ) } | ${ stream . layout . confidence } | ${ stream . layout . source } | ${ stream . layout . halt ? '**halt**' : 'ok'} |` )
522 . join ( ' \n ' );
523
524 const entityRows = capture.streams
525 . flatMap (( stream ) => stream.entities. map (( entity ) => `| ${ entity . name } | \` ${ entity . origin . kind } \` | ${ entity . recordCount } | ${ entity . inUse ? 'yes' : 'no'} | ${ entity . hierarchical ? 'yes' : '—'} | [${ entity . rawFile }](./${ entity . rawFile }) |` ))
526 . join ( ' \n ' );
527
528 const driftLines = capture.streams. map (( stream ) => {
529 const drift = stream.drift;
530 return `### \` ${ path . basename ( stream . primary ) } \` (${ stream . vendor . vendor }) \n\n `
531 + `- Unmapped columns (${ drift . unmappedColumns . length }): ${ drift . unmappedColumns . map (( column ) => ` \` ${ column } \` ` ). join ( ', ' ) || 'none'} \n `
532 + `- Missing expected columns (${ drift . missingExpectedColumns . length }): ${ drift . missingExpectedColumns . map (( column ) => ` \` ${ column } \` ` ). join ( ', ' ) || 'none'} \n ` ;
533 }). join ( ' \n ' );
534
535 const warningLines = capture.warnings. length > 0
536 ? capture.warnings. map (( warning ) => `- **${ warning . kind }**${ warning . file ? ` ( \` ${ warning . file } \` )` : ''}: ${ warning . message }` ). join ( ' \n ' )
537 : '- None' ;
538
539 const haltBanner = capture.halt
540 ? '## ⚠️ Discovery halted to a decision \n\n At least one file could not be resolved deterministically. Do not synthesize `source-schema.json` from this capture until the questions below are answered with the user. \n\n '
541 : '' ;
542
543 return `# CSV Discovery \n\n `
544 + `- Generated at: \` ${ capture . generatedAt } \`\n `
545 + `- Adapter: \` rp-source-csv \`\n `
546 + `- Input files: ${ capture . sourceFiles . length } \n `
547 + `- Logical streams: ${ capture . streams . length } \n `
548 + `- Entities documented: ${ capture . streams . reduce (( sum , stream ) => sum + stream . entities . length , 0 ) } \n\n `
549 + haltBanner
550 + `## Files \n\n | File | Role | Vendor | Role source | Part of | \n | --- | --- | --- | --- | --- | \n ${ fileRows } \n\n `
551 + `## Vendor detection \n\n ${ capture . streams . map (( stream ) => `- \` ${ path . basename ( stream . primary ) } \` → **${ stream . vendor . vendor }** (confidence ${ stream . vendor . confidence }, ${ stream . vendor . source }) — ${ stream . vendor . evidence }${ stream . vendor . nearMiss ? ` \n - Near miss: ${ stream . vendor . nearMiss . vendor } (missing required: ${ stream . vendor . nearMiss . missingRequired . join ( ', ' ) || 'none'})` : ''}${ stream . vendor . conflict ? ` \n - ⚠️ user stated \` ${ stream . vendor . conflict . stated } \` but the header looks like \` ${ stream . vendor . conflict . detected } \` ` : ''}` ). join ( ' \n ' ) } \n\n `
552 + `## Layout \n\n | File | Pattern | Key | Confidence | Source | Status | \n | --- | --- | --- | ---: | --- | --- | \n ${ streamRowsMd } \n\n `
553 + `## Entities \n\n | Entity | Origin | Records | In use | Hierarchical | File | \n | --- | --- | ---: | --- | --- | --- | \n ${ entityRows || '| None | - | - | - | - | - |'} \n\n `
554 + `## Profile drift \n\n ${ driftLines } \n `
555 + `## Warnings \n\n ${ warningLines } \n\n `
556 + `## Cross-file joins \n\n ${ capture . joins . deferred ? `Deferred: ${ capture . joins . reason }` : 'None detected'} \n\n `
557 + `Machine-readable capture: [fileset.json](./fileset.json), [raw-capture.json](./raw-capture.json) \n ` ;
558 }
559
560 function filesetPayload ( capture ) {
561 return {
562 generatedAt: capture.generatedAt,
563 adapter: 'rp-source-csv' ,
564 platform: 'csv' ,
565 csvInputRoot: capture.csvInputRoot,
566 halt: capture.halt,
567 sourceFiles: capture.sourceFiles. map (( entry ) => ({
568 ... entry,
569 file: path. relative (capture.csvInputRoot, entry.file) || path. basename (entry.file),
570 partOf: entry.partOf ? path. relative (capture.csvInputRoot, entry.partOf) || path. basename (entry.partOf) : null ,
571 })),
572 streams: capture.streams. map (( stream ) => ({
573 role: stream.role,
574 primary: path. relative (capture.csvInputRoot, stream.primary) || path. basename (stream.primary),
575 // The header row AS READ. This is the authoritative column list for the whole pipeline —
576 // rp-mapper's deterministic resolver joins it against the vendor overlay, so it belongs in
577 // the hand-off artifact and not only in raw-capture.json (which is evidence, not contract).
578 header: stream.header,
579 parts: stream.parts. map (( part ) => ({
580 file: path. relative (capture.csvInputRoot, part.file) || path. basename (part.file),
581 partOf: part.partOf ? path. relative (capture.csvInputRoot, part.partOf) || path. basename (part.partOf) : null ,
582 headerRelation: part.headerRelation,
583 rowOffset: part.rowOffset,
584 rowCount: part.rowCount,
585 })),
586 vendor: {
587 name: stream.vendor.vendor,
588 confidence: stream.vendor.confidence,
589 source: stream.vendor.source,
590 evidence: stream.vendor.evidence,
591 profileVersion: stream.profile ? stream.profile.profileVersion : null ,
592 nearMiss: stream.vendor.nearMiss,
593 conflict: stream.vendor.conflict,
594 },
595 dialect: stream.dialect,
596 layout: {
597 pattern: stream.layout.pattern,
598 groupKey: stream.layout.groupKey,
599 continuation: stream.layout.continuation,
600 discriminatorColumn: stream.layout.discriminatorColumn,
601 childLevels: stream.layout.childLevels,
602 parentRefColumn: stream.layout.parentRefColumn,
603 confidence: stream.layout.confidence,
604 source: stream.layout.source,
605 halt: stream.layout.halt,
606 evidence: stream.layout.evidence,
607 },
608 drift: stream.drift,
609 mappingHints: stream.mappingHints,
610 quirks: stream.profile ? stream.profile.quirks || [] : [],
611 // Layout-level conflicts (an overlay whose declared grouping the rows
612 // contradict) and column-level ones (an overlay bucketing a column against
613 // the derived role) land in one list, because a reviewer is asking the same
614 // question of both: where did this capture stop believing the overlay?
615 layoutConflicts: [ ... (stream.layout.layoutConflicts || []), ... stream.roles.layoutConflicts],
616 ambiguousColumns: stream.roles.ambiguousColumns,
617 entities: stream.entities. map (( entity ) => ({
618 name: entity.name,
619 // Paths stay relative to csvInputRoot so moving the migration folder
620 // does not invalidate the capture.
621 origin: {
622 ... entity.origin,
623 file: path. relative (capture.csvInputRoot, entity.origin.file) || path. basename (entity.origin.file),
624 },
625 recordCount: entity.recordCount,
626 inUse: entity.inUse,
627 hierarchical: Boolean (entity.hierarchical),
628 rawFile: entity.rawFile,
629 columns: entity.columns,
630 relations: entity.relations,
631 })),
632 })),
633 joins: capture.joins.joins,
634 integrity: capture.joins.integrity,
635 joinsDeferred: capture.joins.deferred,
636 conflicts: capture.conflicts,
637 warnings: capture.warnings,
638 };
639 }
640
641 function rawCapturePayload ( capture ) {
642 return {
643 generatedAt: capture.generatedAt,
644 adapter: 'rp-source-csv' ,
645 streams: capture.streams. map (( stream ) => ({
646 primary: path. relative (capture.csvInputRoot, stream.primary) || path. basename (stream.primary),
647 role: stream.role,
648 header: stream.header,
649 dialect: stream.dialect,
650 scan: {
651 rowCount: stream.scan.rowCount,
652 scannedRows: stream.rows. length ,
653 truncated: stream.scan.truncated,
654 raggedRows: stream.scan.raggedRows,
655 },
656 columns: stream.columns,
657 headSample: stream.headSample,
658 tailSample: stream.tailSample,
659 mediaNotes: stream.mediaNotes,
660 })),
661 };
662 }
663
664 async function capture ( args ) {
665 const profiles = loadVendorProfiles ();
666 const generatedAt = new Date (). toISOString ();
667 const resolvedFiles = args.files. map (( file ) => path. resolve (file));
668 const csvInputRoot = process.env. CSV_INPUT_ROOT
669 ? path. resolve (process.env. CSV_INPUT_ROOT )
670 : (resolvedFiles. length === 1 ? path. dirname (resolvedFiles[ 0 ]) : commonDirectory (resolvedFiles));
671
672 // 1. header + dialect + vendor per file
673 const scanned = [];
674 for ( let i = 0 ; i < resolvedFiles. length ; i += 1 ) {
675 const file = resolvedFiles[i];
676 progress?. progress ( `Reading header of ${ path . basename ( file ) }` , {
677 phase: 'discovery' ,
678 step: 'read-header' ,
679 entity: path. basename (file),
680 count: i + 1 ,
681 total: resolvedFiles. length ,
682 unit: 'files' ,
683 });
684 const head = await readHeaderRow (file, { delimiter: args.delimiter, encoding: args.encoding });
685 const vendor = detectVendor (head.header, profiles, { statedVendor: args.vendor });
686 const scan = await scanFile (file, {
687 delimiter: head.delimiter,
688 encoding: head.encoding,
689 scanRows: args.scanRows,
690 header: head.header,
691 });
692 scanned. push ({ file, head, vendor, scan });
693 }
694
695 // 2. roles, split-file concatenation
696 const fileSet = resolveFileSet (
697 scanned. map (( entry ) => ({
698 file: entry.file,
699 header: entry.head.header,
700 vendor: entry.vendor.vendor,
701 profile: entry.vendor.profile,
702 rowCount: entry.scan.rowCount,
703 })),
704 {
705 explicitRoles: args.explicitRoles,
706 policy: args.allowHeaderSuperset ? CONCAT_POLICY . UNION : CONCAT_POLICY . STRICT_SET ,
707 fileOrder: args.fileOrder,
708 },
709 );
710
711 const byFile = new Map (scanned. map (( entry ) => [entry.file, entry]));
712 const warnings = [];
713 const takenEntityNames = new Set ();
714 const streams = [];
715
716 for ( const stream of fileSet.streams) {
717 const primary = byFile. get (stream.primary);
718 // Split parts share one logical stream, so their rows are inferred together.
719 const rows = stream.parts. flatMap (( part ) => (byFile. get (part.file) || { scan: { rows: [] } }).scan.rows);
720 const header = primary.head.header;
721 const profile = primary.vendor.profile;
722
723 progress?. progress ( `Resolving layout for ${ path . basename ( stream . primary ) }` , {
724 phase: 'discovery' ,
725 step: 'resolve-layout' ,
726 entity: path. basename (stream.primary),
727 });
728
729 const layout = classifyLayout (header, rows, { overlayLayout: profile ? profile.layout : null });
730 const roles = deriveColumnRoles (header, rows, layout, {
731 columnGroups: (profile && profile.layout.columnGroups) || [],
732 });
733 const derived = detectDerivedCandidates (header, rows, { profile });
734 const drift = diffProfile (header, profile);
735 const columns = summarizeColumns (header, rows);
736 const mediaNotes = mediaReachability (columns);
737
738 const tail = await readTailSample (stream.primary, {
739 delimiter: primary.head.delimiter,
740 encoding: primary.head.encoding,
741 header,
742 });
743
744 const resolved = {
745 role: stream.role,
746 primary: stream.primary,
747 parts: stream.parts,
748 header,
749 rows,
750 columns,
751 profile,
752 vendor: primary.vendor,
753 layout,
754 roles,
755 derived,
756 drift,
757 mediaNotes,
758 scan: {
759 rowCount: stream.parts. reduce (( sum , part ) => sum + (part.rowCount || 0 ), 0 ),
760 truncated: stream.parts. some (( part ) => (byFile. get (part.file) || { scan: {} }).scan.truncated),
761 raggedRows: stream.parts. reduce (( sum , part ) => sum + ((byFile. get (part.file) || { scan: {} }).scan.raggedRows || 0 ), 0 ),
762 },
763 dialect: {
764 delimiter: primary.head.delimiter,
765 delimiterAmbiguous: primary.head.delimiterAmbiguous,
766 encoding: primary.head.encoding,
767 bom: primary.head.bom,
768 lineEnding: primary.head.lineEnding,
769 emptyPolicy: detectEmptyPolicy (rows),
770 },
771 headSample: rows. slice ( 0 , args.headRows). map (( row ) => row.values),
772 tailSample: tail.skipped ? [] : tail.rows. slice ( - args.tailRows). map (( row ) => row.values),
773 tailSampleSkipped: tail.skipped ? tail.reason : null ,
774 mappingHints: prefillColumnMap (header, profile),
775 };
776
777 resolved.entities = deriveEntities (resolved, takenEntityNames);
778 for ( const entity of resolved.entities) {
779 entity.rawFile = `${ stream . role }--${ entity . name }.md` ;
780 }
781 streams. push (resolved);
782
783 if (resolved.scan.rowCount === 0 ) {
784 warnings. push ({
785 kind: 'empty-file' ,
786 file: path. basename (stream.primary),
787 message: 'the file has a header but no data rows; its entities are advertised but unused' ,
788 });
789 }
790 if (layout.halt) {
791 warnings. push ({
792 kind: 'layout-ambiguous' ,
793 file: path. basename (stream.primary),
794 message: `layout could not be resolved deterministically (${ layout . evidence . join ( '; ' ) }) — ask the user how rows group before synthesizing the schema` ,
795 });
796 }
797 for ( const conflict of layout.layoutConflicts || []) {
798 warnings. push ({
799 kind: conflict.kind,
800 file: path. basename (stream.primary),
801 message: `the ${ primary . vendor . vendor } overlay's declared layout is not what this file's rows show — ${ conflict . resolution }. Details: ${ layout . evidence [ 0 ] }` ,
802 });
803 }
804 if (primary.vendor.nearMiss) {
805 warnings. push ({
806 kind: 'vendor-near-miss' ,
807 file: path. basename (stream.primary),
808 message: `header looks like a drifted ${ primary . vendor . nearMiss . vendor } export (missing required: ${ primary . vendor . nearMiss . missingRequired . join ( ', ' ) || 'none'}); confirm with the user or set CSV_VENDOR` ,
809 });
810 }
811 if (primary.vendor.conflict) {
812 warnings. push ({
813 kind: 'vendor-conflict' ,
814 file: path. basename (stream.primary),
815 message: `user stated "${ primary . vendor . conflict . stated }" but the header fingerprints as "${ primary . vendor . conflict . detected }"` ,
816 });
817 }
818 if (resolved.dialect.delimiterAmbiguous) {
819 warnings. push ({
820 kind: 'delimiter-ambiguous' ,
821 file: path. basename (stream.primary),
822 message: 'delimiter could not be detected confidently; set CSV_DELIMITER' ,
823 });
824 }
825 if (resolved.scan.raggedRows > 0 ) {
826 warnings. push ({
827 kind: 'ragged-rows' ,
828 file: path. basename (stream.primary),
829 message: `${ resolved . scan . raggedRows } row(s) do not have the header's column count` ,
830 });
831 }
832 if (resolved.tailSampleSkipped) {
833 warnings. push ({
834 kind: 'tail-sample-skipped' ,
835 file: path. basename (stream.primary),
836 message: `tail sample skipped (${ resolved . tailSampleSkipped })` ,
837 });
838 }
839 for ( const note of mediaNotes) {
840 warnings. push ({
841 kind: 'media-reachability' ,
842 file: path. basename (stream.primary),
843 message: `column "${ note . column }" holds values Wix cannot fetch (e.g. ${ note . example }): ${ note . note }` ,
844 });
845 }
846 if (roles.ambiguousColumns. length > 0 ) {
847 warnings. push ({
848 kind: 'ambiguous-columns' ,
849 file: path. basename (stream.primary),
850 message: `could not decide which entity these columns belong to: ${ roles . ambiguousColumns . join ( ', ' ) }` ,
851 });
852 }
853 for ( const candidate of derived) {
854 if (candidate.status === 'proposed' ) {
855 warnings. push ({
856 kind: 'derived-entity-proposed' ,
857 file: path. basename (stream.primary),
858 message: `column "${ candidate . fromColumn }" may be a "${ candidate . entity }" entity (confidence ${ candidate . confidence }); confirm with the user before treating it as one` ,
859 });
860 }
861 if (candidate.status === 'missing-column' ) {
862 warnings. push ({
863 kind: 'derived-column-missing' ,
864 file: path. basename (stream.primary),
865 message: `the ${ primary . vendor . vendor } overlay expects a "${ candidate . fromColumn }" column for the "${ candidate . entity }" entity, but this file has none` ,
866 });
867 }
868 }
869 }
870
871 for ( const conflict of fileSet.conflicts) {
872 warnings. push ({ kind: conflict.kind, file: (conflict.files || []). map (( file ) => path. basename (file)). join ( ', ' ), message: conflict.message });
873 }
874
875 return {
876 generatedAt,
877 csvInputRoot,
878 sourceFiles: fileSet.sourceFiles,
879 streams,
880 conflicts: fileSet.conflicts,
881 joins: inferJoins (),
882 warnings,
883 recordCurrencyCounts: mergeCurrencyCounts (scanned. map (( entry ) => entry.scan.currencyCounts)),
884 supplied: {
885 currency: args.currency,
886 timeZone: args.timeZone,
887 language: args.language,
888 country: args.country,
889 },
890 halt: fileSet.halt || streams. some (( stream ) => stream.layout.halt),
891 };
892 }
893
894 // A file with an unknown role and a stream whose layout halted to a decision are
895 // both inputs this capture could not turn into entities. Always written, `rows: []` when clean.
896 function sourceCoveragePayload ( result ) {
897 return buildCoverageFile ({
898 platform: 'csv' ,
899 adapter: 'csv' ,
900 generatedAt: result.generatedAt,
901 rows: buildCsvCoverageRows ({
902 sourceFiles: result.sourceFiles,
903 streams: result.streams,
904 csvInputRoot: result.csvInputRoot,
905 rawDir: ADAPTER_RAW_DIRS .csv,
906 }),
907 });
908 }
909
910 // The store's regional facts: what the person supplied at intake (an input left out is an explicit
911 // unknown) plus the per-row currency counts as evidence. Always written.
912 function sourceRegionalPayload ( result ) {
913 return buildCsvSourceRegional ({
914 supplied: result.supplied,
915 recordCurrencyCounts: result.recordCurrencyCounts,
916 generatedAt: result.generatedAt,
917 });
918 }
919
920 function commonDirectory ( files ) {
921 if (files. length === 0 ) {
922 return process. cwd ();
923 }
924 const segments = files. map (( file ) => path. dirname (file). split (path.sep));
925 const common = [];
926 for ( let i = 0 ; i < segments[ 0 ]. length ; i += 1 ) {
927 const candidate = segments[ 0 ][i];
928 if (segments. every (( parts ) => parts[i] === candidate)) {
929 common. push (candidate);
930 } else {
931 break ;
932 }
933 }
934 return common. join (path.sep) || path.sep;
935 }
936
937 async function writeOutputs ( outDir , result ) {
938 await fs. mkdir (outDir, { recursive: true });
939 await fs. writeFile (path. join (outDir, 'README.md' ), renderIndexFile (result), 'utf8' );
940 progress?. progress ( 'Wrote CSV discovery index' , { phase: 'discovery' , step: 'write-artifact' , artifact: path. join (outDir, 'README.md' ) });
941
942 await fs. writeFile (path. join (outDir, 'fileset.json' ), `${ JSON . stringify ( filesetPayload ( result ), null , 2 ) } \n ` , 'utf8' );
943 await fs. writeFile (path. join (outDir, 'raw-capture.json' ), `${ JSON . stringify ( rawCapturePayload ( result ), null , 2 ) } \n ` , 'utf8' );
944 await fs. writeFile (path. join (outDir, 'source-coverage.json' ), `${ JSON . stringify ( sourceCoveragePayload ( result ), null , 2 ) } \n ` , 'utf8' );
945 await writeCsvSourceRegional (outDir, sourceRegionalPayload (result));
946 progress?. progress ( 'Wrote CSV discovery machine capture' , { phase: 'discovery' , step: 'write-artifact' , artifact: path. join (outDir, 'fileset.json' ) });
947
948 let entityCount = 0 ;
949 for ( const stream of result.streams) {
950 const columnsByName = new Map (stream.columns. map (( column ) => [column.name, column]));
951 for ( const entity of stream.entities) {
952 await fs. writeFile (path. join (outDir, entity.rawFile), renderEntityFile (stream, entity, columnsByName), 'utf8' );
953 entityCount += 1 ;
954 progress?. progress ( `Wrote CSV discovery artifact for ${ entity . name }` , {
955 phase: 'discovery' ,
956 step: 'write-artifact' ,
957 entity: entity.name,
958 artifact: path. join (outDir, entity.rawFile),
959 });
960 }
961 }
962 return entityCount;
963 }
964
965 async function main () {
966 const parsed = parseProgressArgs (process.argv. slice ( 2 ));
967 progress = createProgressLogger ({
968 script: 'skills/wix-replatform/resources/rp-source-csv/scripts/csv-discovery.js' ,
969 ... parsed.progress,
970 });
971 progress. start ( 'CSV discovery started' , { phase: 'discovery' });
972
973 const args = parseArgs (parsed.args);
974 if (args.help) {
975 printUsage ();
976 progress. complete ( 'CSV discovery help shown' , { phase: 'discovery' , step: 'help' });
977 return ;
978 }
979 if (args.files. length === 0 || ! args.outDir) {
980 printUsage ();
981 progress. error ( 'Missing required CSV discovery arguments' , { phase: 'discovery' });
982 throw new Error ( 'Missing required arguments: at least one --file and --out-dir are required.' );
983 }
984
985 const result = await capture (args);
986 const entityCount = await writeOutputs (path. resolve (args.outDir), result);
987
988 console. log ( `Wrote ${ entityCount } entity file(s), README.md, fileset.json and raw-capture.json to ${ args . outDir }` );
989 if (result.halt) {
990 console. log ( 'Discovery halted to a decision — see the Warnings section of README.md before synthesizing source-schema.json.' );
991 }
992 progress. complete ( 'CSV discovery completed' , {
993 phase: 'discovery' ,
994 artifact: args.outDir,
995 count: entityCount,
996 unit: 'entities' ,
997 });
998 }
999
1000 if (require.main === module ) {
1001 main (). catch (( error ) => {
1002 console. error (error.stack || error.message);
1003 if (progress) {
1004 progress. error (error && error.message ? error.message : 'CSV discovery failed' , { phase: 'discovery' });
1005 }
1006 process.exitCode = 1 ;
1007 });
1008 }
1009
1010 module . exports = {
1011 DEFAULT_HEAD_ROWS,
1012 DEFAULT_TAIL_ROWS,
1013 DEFAULT_SCAN_ROWS,
1014 parseArgs,
1015 scanFile,
1016 summarizeColumns,
1017 mediaReachability,
1018 deriveEntities,
1019 renderIndexFile,
1020 renderEntityFile,
1021 filesetPayload,
1022 rawCapturePayload,
1023 sourceCoveragePayload,
1024 sourceRegionalPayload,
1025 capture,
1026 writeOutputs,
1027 };