Setting the file. One moment.
CSV Discovery · Rp Source CSV · wix/skills · Skills Docs
Back to the top of the page 601
function rawCapturePayload
— line 601
This file
Number 17.1
Position 1 of 11
Type JavaScript
Size 35 KB
Lines 949 scripts/ csv-discovery.js
JavaScript · 949 lines · 35 KB
// same behavior instead of re-deriving it.
10 const {
11 readHeaderRow ,
12 readTailSample ,
13 streamRows ,
14 detectEmptyPolicy ,
15 summarizeColumn ,
16 rowWidthReport ,
17 normalizeHeaderName ,
18 } = require ( '../lib/csv-parse.js' );
19 const {
20 loadVendorProfiles ,
21 detectVendor ,
22 diffProfile ,
23 prefillColumnMap ,
24 } = require ( '../lib/csv-fingerprint.js' );
25 const {
26 classifyLayout ,
27 buildGroups ,
28 columnIndex ,
29 deriveColumnRoles ,
30 deriveColumnValues ,
31 detectDerivedCandidates ,
32 } = require ( '../lib/csv-layout.js' );
33 const {
34 CONCAT_POLICY ,
35 resolveFileSet ,
36 inferJoins ,
37 } = require ( '../lib/csv-fileset.js' );
38
39 const DEFAULT_HEAD_ROWS = 5 ;
40 const DEFAULT_TAIL_ROWS = 3 ;
41 const DEFAULT_SCAN_ROWS = 5000 ;
42
43 let progress;
44
45 function printUsage () {
46 console. log ( `Usage:
47 node csv-discovery.js --file <path> [--file <path> ...] --out-dir <dir> [options]
48
49 Required:
50 --file <path> Input CSV. Repeatable — pass the whole file set in one run
51 so roles and split files are resolved together.
52 --out-dir <dir> Directory to write the raw capture into
53
54 Optional:
55 --vendor <name> User-stated vendor; overrides fingerprint detection.
56 One of: shopify, woocommerce, magento, bigcommerce, custom
57 --delimiter <char> Force the delimiter (default: auto-detect , ; \\ t |)
58 --encoding <name> Force the encoding (default: auto-detect UTF-8 / BOM)
59 --role <path>:<role> Force a file's role. Repeatable.
60 --file-order stated|natural Order of split files (default: natural)
61 --allow-header-superset Union split files whose headers differ instead of halting
62 --head-rows <n> Sample rows from the head. Default: ${ DEFAULT_HEAD_ROWS }
63 --tail-rows <n> Sample rows from the tail. Default: ${ DEFAULT_TAIL_ROWS }
64 --scan-rows <n> Rows retained for inference. Default: ${ DEFAULT_SCAN_ROWS }
65 --progress-log <path> Append progress NDJSON records to this file.
66 --help Show this help text
67
68 Example:
69 node csv-discovery.js \\
70 --file ~/exports/products_export.csv \\
71 --out-dir migrations/acme/data/csv-discovery
72 ` );
73 }
74
75 function parseArgs ( argv ) {
76 const args = {
77 files: [],
78 explicitRoles: {},
79 vendor: null ,
80 delimiter: null ,
81 encoding: null ,
82 fileOrder: 'natural' ,
83 allowHeaderSuperset: false ,
84 headRows: DEFAULT_HEAD_ROWS ,
85 tailRows: DEFAULT_TAIL_ROWS ,
86 scanRows: DEFAULT_SCAN_ROWS ,
87 outDir: null ,
88 help: false ,
89 };
90
91 for ( let i = 0 ; i < argv. length ; i += 1 ) {
92 const arg = argv[i];
93 const next = argv[i + 1 ];
94
95 switch (arg) {
96 case '--help' :
97 case '-h' :
98 args.help = true ;
99 break ;
100 case '--file' :
101 args.files. push (next);
102 i += 1 ;
103 break ;
104 case '--out-dir' :
105 args.outDir = next;
106 i += 1 ;
107 break ;
108 case '--vendor' :
109 args.vendor = next;
110 i += 1 ;
111 break ;
112 case '--delimiter' :
113 args.delimiter = next === ' \\ t' ? ' \t ' : next;
114 i += 1 ;
115 break ;
116 case '--encoding' :
117 args.encoding = next;
118 i += 1 ;
119 break ;
120 case '--role' : {
121 const splitIndex = String (next). lastIndexOf ( ':' );
122 if (splitIndex === - 1 ) {
123 throw new Error ( `Invalid --role value: ${ next } (expected <path>:<role>)` );
124 }
125 args.explicitRoles[ String (next). slice ( 0 , splitIndex)] = String (next). slice (splitIndex + 1 );
126 i += 1 ;
127 break ;
128 }
129 case '--file-order' :
130 args.fileOrder = next;
131 i += 1 ;
132 break ;
133 case '--allow-header-superset' :
134 args.allowHeaderSuperset = true ;
135 break ;
136 case '--head-rows' :
137 args.headRows = Number. parseInt (next, 10 );
138 i += 1 ;
139 break ;
140 case '--tail-rows' :
141 args.tailRows = Number. parseInt (next, 10 );
142 i += 1 ;
143 break ;
144 case '--scan-rows' :
145 args.scanRows = Number. parseInt (next, 10 );
146 i += 1 ;
147 break ;
148 default :
149 if (arg. startsWith ( '--' )) {
150 throw new Error ( `Unknown argument: ${ arg }` );
151 }
152 }
153 }
154
155 if ( ! args.vendor) {
156 args.vendor = process.env. CSV_VENDOR || null ;
157 }
158 if ( ! args.delimiter) {
159 args.delimiter = process.env. CSV_DELIMITER || null ;
160 }
161 if ( ! args.encoding) {
162 args.encoding = process.env. CSV_ENCODING || null ;
163 }
164 if ( ! args.outDir) {
165 args.outDir = process.env. CSV_DISCOVERY_OUT_DIR || null ;
166 }
167 if ( ! Number. isFinite (args.headRows) || args.headRows < 1 ) {
168 args.headRows = DEFAULT_HEAD_ROWS ;
169 }
170 if ( ! Number. isFinite (args.tailRows) || args.tailRows < 0 ) {
171 args.tailRows = DEFAULT_TAIL_ROWS ;
172 }
173 if ( ! Number. isFinite (args.scanRows) || args.scanRows < 1 ) {
174 args.scanRows = DEFAULT_SCAN_ROWS ;
175 }
176 if (args.fileOrder !== 'stated' && args.fileOrder !== 'natural' ) {
177 args.fileOrder = 'natural' ;
178 }
179
180 return args;
181 }
182
183 // One streaming pass per file: full row count, bounded row retention for
184 // inference, and per-row shape checks. The file is never bulk-loaded.
185 async function scanFile ( filePath , { delimiter , encoding , scanRows , header }) {
186 const rows = [];
187 let rowCount = 0 ;
188 let raggedRows = 0 ;
189 let index = 0 ;
190
191 for await ( const row of streamRows (filePath, { delimiter, encoding, trackQuoted: true })) {
192 index += 1 ;
193 if (index === 1 ) {
194 continue ; // header
195 }
196 rowCount += 1 ;
197 if (rows. length < scanRows) {
198 rows. push (row);
199 }
200 if ( rowWidthReport (header, row.values).ragged) {
201 raggedRows += 1 ;
202 }
203 }
204
205 return { rows, rowCount, raggedRows, truncated: rowCount > rows. length };
206 }
207
208 function summarizeColumns ( header , rows ) {
209 return header. map (( name , columnIdx ) => {
210 const samples = rows. map (( row ) => ({
211 value: row.values[columnIdx] === undefined ? '' : row.values[columnIdx],
212 quoted: Boolean (row.quoted && row.quoted[columnIdx]),
213 }));
214 return summarizeColumn (name, samples);
215 });
216 }
217
218 const PRIVATE_HOST = / ^ (https ? :) ? \/\/ (localhost | 127 \. 0 \. 0 \. 1 | 0 \. 0 \. 0 \. 0 | 10 \. | 192 \. 168 \. | \[ ::1 \] )/ i ;
219
220 // Wix Media import fetches from Wix servers, so a local path or private URL in
221 // an image column is not reachable at import time — the same concern the
222 // WordPress adapter records for localhost sources.
223 const MEDIA_LOCATION = / [ \\ /] | \. (jpe ? g | png | gif | webp | avif | svg | bmp | tiff ?| mp4 | mov | pdf) $ / i ;
224
225 function mediaReachability ( columns ) {
226 const notes = [];
227 for ( const column of columns) {
228 const looksLikeMedia = /image | photo | media | picture | asset | file/ i . test (column.name);
229 if ( ! looksLikeMedia || column.examples. length === 0 ) {
230 continue ;
231 }
232 // Only values that look like a location can be unreachable; an "Image
233 // Position" column of 1/2/3 is not a media reference.
234 const locations = column.examples. filter (( example ) => MEDIA_LOCATION . test ( String (example)));
235 if (locations. length === 0 ) {
236 continue ;
237 }
238 const unreachable = locations. filter (( example ) => PRIVATE_HOST . test (example) || ! / ^ https ? : \/\/ / i . test (example));
239 if (unreachable. length > 0 ) {
240 notes. push ({
241 column: column.name,
242 example: unreachable[ 0 ],
243 note: 'not publicly fetchable by Wix; rewrite with CSV_MEDIA_URL_REWRITE_FROM/TO, expose the files publicly, or skip/defer media' ,
244 });
245 }
246 }
247 return notes;
248 }
249
250 function uniqueEntityName ( name , taken ) {
251 if ( ! taken. has (name)) {
252 taken. add (name);
253 return name;
254 }
255 let suffix = 2 ;
256 while (taken. has ( `${ name }-${ suffix }` )) {
257 suffix += 1 ;
258 }
259 const unique = `${ name }-${ suffix }` ;
260 taken. add (unique);
261 return unique;
262 }
263
264 function distinctNonBlank ( rows , index ) {
265 const values = new Set ();
266 for ( const row of rows) {
267 const value = row.values[index];
268 if (value !== undefined && String (value). trim () !== '' ) {
269 values. add (value);
270 }
271 }
272 return values.size;
273 }
274
275 // Turn one resolved stream into entities. A single grouped vendor file yields
276 // several: the parent, its inline children, any declared collection (images),
277 // and one per column-values descriptor.
278 function deriveEntities ( stream , taken ) {
279 const { header , rows , layout , roles , derived , role } = stream;
280 const entities = [];
281 const relations = [];
282
283 const parentName = uniqueEntityName (
284 layout.parentEntity && layout.parentEntity !== 'record' ? layout.parentEntity : role,
285 taken,
286 );
287
288 const childLevels = new Set ((layout.childLevels || []). map (( level ) => normalizeHeaderName (level)));
289 const discriminatorIdx = columnIndex (header, layout.discriminatorColumn);
290 const groups = buildGroups (header, rows, layout);
291
292 let parentOrigin;
293 let parentCount;
294 if (layout.pattern === 'grouped-by-key' ) {
295 parentOrigin = {
296 kind: 'row-group' ,
297 file: stream.primary,
298 groupKey: layout.groupKey,
299 continuation: layout.continuation,
300 };
301 parentCount = groups. length ;
302 } else if (layout.pattern === 'sectioned' ) {
303 parentOrigin = {
304 kind: 'file-rows' ,
305 file: stream.primary,
306 filter: { column: layout.discriminatorColumn, excludeValues: layout.childLevels || [] },
307 };
308 parentCount = rows. filter (( row ) => ! childLevels. has ( normalizeHeaderName (row.values[discriminatorIdx] || '' ))). length ;
309 } else {
310 parentOrigin = { kind: 'file-rows' , file: stream.primary };
311 parentCount = rows. length ;
312 }
313
314 entities. push ({
315 name: parentName,
316 origin: parentOrigin,
317 recordCount: parentCount,
318 inUse: parentCount > 0 ,
319 primaryKey: layout.groupKey || layout.primaryKey || null ,
320 columns: roles.byEntity[layout.parentEntity] || roles.parentColumns,
321 relations: [],
322 });
323
324 if (roles.childColumns. length > 0 && layout.childEntity) {
325 const childName = uniqueEntityName (layout.childEntity, taken);
326 let childCount;
327 let childOrigin;
328 if (layout.pattern === 'sectioned' ) {
329 childCount = rows. filter (( row ) => childLevels. has ( normalizeHeaderName (row.values[discriminatorIdx] || '' ))). length ;
330 childOrigin = {
331 kind: 'file-rows' ,
332 file: stream.primary,
333 filter: { column: layout.discriminatorColumn, includeValues: layout.childLevels || [] },
334 parentRefColumn: layout.parentRefColumn || null ,
335 };
336 } else {
337 // Not `rows.length`: in a grouped file a continuation row can carry only an
338 // extra image, leaving every child column blank. Counting those as variants
339 // inflates the child entity by exactly the number of image-only rows.
340 const childIndexes = roles.childColumns. map (( column ) => columnIndex (header, column)). filter (( index ) => index !== - 1 );
341 childCount = rows. filter (( row ) => childIndexes. some (( index ) => {
342 const value = row.values[index];
343 return value !== undefined && String (value). trim () !== '' ;
344 })). length ;
345 childOrigin = {
346 kind: 'row-group' ,
347 file: stream.primary,
348 groupKey: layout.groupKey,
349 columnGroup: layout.childEntity,
350 };
351 }
352 entities. push ({
353 name: childName,
354 origin: childOrigin,
355 recordCount: childCount,
356 inUse: childCount > 0 ,
357 columns: roles.childColumns,
358 relations: [],
359 });
360 relations. push ({
361 from: parentName,
362 field: childName,
363 targetEntity: childName,
364 cardinality: 'one-to-many' ,
365 evidence: layout.pattern === 'sectioned'
366 ? `layout.discriminatorColumn=${ layout . discriminatorColumn }${ layout . parentRefColumn ? ` + parentRef=${ layout . parentRefColumn }` : ''}`
367 : `layout.groupKey=${ layout . groupKey }` ,
368 });
369 }
370
371 // Overlay-declared collections (Shopify's image columns) — a third bucket the
372 // data alone cannot separate from the variant columns.
373 for ( const [ entity , columns ] of Object. entries (roles.byEntity)) {
374 if (entity === layout.parentEntity || entity === layout.childEntity || columns. length === 0 ) {
375 continue ;
376 }
377 const anchorIdx = columnIndex (header, columns[ 0 ]);
378 const count = anchorIdx === - 1 ? 0 : distinctNonBlank (rows, anchorIdx);
379 const entityName = uniqueEntityName (entity, taken);
380 entities. push ({
381 name: entityName,
382 origin: {
383 kind: 'row-group' ,
384 file: stream.primary,
385 groupKey: layout.groupKey || null ,
386 columnGroup: entity,
387 distinctBy: columns[ 0 ],
388 },
389 recordCount: count,
390 inUse: count > 0 ,
391 columns,
392 relations: [],
393 });
394 relations. push ({
395 from: parentName,
396 field: entityName,
397 targetEntity: entityName,
398 cardinality: 'one-to-many' ,
399 evidence: `layout.columnGroups[${ entity }]` ,
400 });
401 }
402
403 // column-values entities: categories and tags are columns, not files.
404 for ( const descriptor of derived) {
405 if (descriptor.status === 'missing-column' ) {
406 continue ;
407 }
408 const result = deriveColumnValues (header, rows, descriptor, { layout });
409 if (result.missingColumn || result.records. length === 0 ) {
410 continue ;
411 }
412 const entityName = uniqueEntityName (descriptor.entity, taken);
413 entities. push ({
414 name: entityName,
415 origin: { ... result.origin, file: stream.primary },
416 recordCount: result.records. length ,
417 inUse: result.records. length > 0 ,
418 hierarchical: result.hierarchical,
419 columns: [descriptor.fromColumn],
420 sampleRecords: result.records. slice ( 0 , 10 ),
421 relations: [],
422 });
423 relations. push ({
424 from: parentName,
425 field: (descriptor.relation && descriptor.relation.field) || entityName,
426 targetEntity: entityName,
427 cardinality: (descriptor.relation && descriptor.relation.cardinality) || 'many-to-many' ,
428 evidence: `column:${ descriptor . fromColumn }` ,
429 });
430 }
431
432 for ( const relation of relations) {
433 const owner = entities. find (( entity ) => entity.name === relation.from);
434 if (owner) {
435 owner.relations. push ({
436 field: relation.field,
437 targetEntity: relation.targetEntity,
438 cardinality: relation.cardinality,
439 evidence: relation.evidence,
440 });
441 }
442 }
443
444 return entities;
445 }
446
447 function renderEntityFile ( stream , entity , columnsByName ) {
448 const columnRows = (entity.columns || [])
449 . map (( name ) => columnsByName. get (name))
450 . filter (Boolean)
451 . map (( column ) => `| \` ${ column . name } \` | ${ column . type } | ${ column . required ? 'yes' : 'no'} | ${ column . distinctCount }${ column . distinctCapped ? '+' : ''} | ${ column . blankCount } | ${ column . examples . map (( example ) => ` \` ${ String ( example ). slice ( 0 , 60 ) } \` ` ). join ( ', ' ) || '—'} |` )
452 . join ( ' \n ' );
453
454 const relationLines = entity.relations. length > 0
455 ? entity.relations. map (( relation ) => `- \` ${ relation . field } \` → \` ${ relation . targetEntity } \` (${ relation . cardinality }) — evidence: \` ${ relation . evidence } \` ` ). join ( ' \n ' )
456 : '- None' ;
457
458 const derivedBlock = entity.sampleRecords
459 ? ` \n ## Derived records (first ${ entity . sampleRecords . length }) \n\n\`\`\` json \n ${ JSON . stringify ( entity . sampleRecords , null , 2 ) } \n\`\`\`\n `
460 : '' ;
461
462 return `# ${ entity . name } \n\n `
463 + `- Source file: \` ${ path . basename ( stream . primary ) } \`\n `
464 + `- Role: \` ${ stream . role } \`\n `
465 + `- Vendor: \` ${ stream . vendor . vendor } \`\n `
466 + `- Origin: \` ${ entity . origin . kind } \` ${ entity . origin . groupKey ? ` (group key \` ${ entity . origin . groupKey } \` )` : ''}${ entity . origin . column ? ` (column \` ${ entity . origin . column } \` )` : ''} \n `
467 + `- Record count: \` ${ entity . recordCount } \` ${ stream . scan . truncated ? ` (from the first ${ stream . rows . length } scanned rows)` : ''} \n `
468 + `- In use: \` ${ entity . inUse ? 'yes' : 'no'} \`\n `
469 + (entity.hierarchical ? '- Hierarchical: `yes` — a hierarchical source taxonomy mapped to a flat Wix target requires a faithfulness-ledger entry \n ' : '' )
470 + ` \n ## Relations \n ${ relationLines } \n `
471 + ` \n ## Columns \n\n | Column | Type | Required | Distinct | Blank | Examples | \n | --- | --- | --- | ---: | ---: | --- | \n ${ columnRows || '| None | - | - | - | - | - |'} \n `
472 + derivedBlock;
473 }
474
475 function renderIndexFile ( capture ) {
476 const fileRows = capture.sourceFiles
477 . map (( entry ) => `| \` ${ path . basename ( entry . file ) } \` | ${ entry . role } | ${ entry . vendor || 'custom'} | ${ entry . roleSource } | ${ entry . partOf ? ` \` ${ path . basename ( entry . partOf ) } \` ` : '—'} |` )
478 . join ( ' \n ' );
479
480 const streamRowsMd = capture.streams
481 . map (( stream ) => `| \` ${ path . basename ( stream . primary ) } \` | \` ${ stream . layout . pattern } \` | ${ stream . layout . groupKey ? ` \` ${ stream . layout . groupKey } \` ` : ( stream . layout . discriminatorColumn ? ` \` ${ stream . layout . discriminatorColumn } \` ` : '—' ) } | ${ stream . layout . confidence } | ${ stream . layout . source } | ${ stream . layout . halt ? '**halt**' : 'ok'} |` )
482 . join ( ' \n ' );
483
484 const entityRows = capture.streams
485 . flatMap (( stream ) => stream.entities. map (( entity ) => `| ${ entity . name } | \` ${ entity . origin . kind } \` | ${ entity . recordCount } | ${ entity . inUse ? 'yes' : 'no'} | ${ entity . hierarchical ? 'yes' : '—'} | [${ entity . rawFile }](./${ entity . rawFile }) |` ))
486 . join ( ' \n ' );
487
488 const driftLines = capture.streams. map (( stream ) => {
489 const drift = stream.drift;
490 return `### \` ${ path . basename ( stream . primary ) } \` (${ stream . vendor . vendor }) \n\n `
491 + `- Unmapped columns (${ drift . unmappedColumns . length }): ${ drift . unmappedColumns . map (( column ) => ` \` ${ column } \` ` ). join ( ', ' ) || 'none'} \n `
492 + `- Missing expected columns (${ drift . missingExpectedColumns . length }): ${ drift . missingExpectedColumns . map (( column ) => ` \` ${ column } \` ` ). join ( ', ' ) || 'none'} \n ` ;
493 }). join ( ' \n ' );
494
495 const warningLines = capture.warnings. length > 0
496 ? capture.warnings. map (( warning ) => `- **${ warning . kind }**${ warning . file ? ` ( \` ${ warning . file } \` )` : ''}: ${ warning . message }` ). join ( ' \n ' )
497 : '- None' ;
498
499 const haltBanner = capture.halt
500 ? '## ⚠️ Discovery halted to a decision \n\n At least one file could not be resolved deterministically. Do not synthesize `source-schema.json` from this capture until the questions below are answered with the user. \n\n '
501 : '' ;
502
503 return `# CSV Discovery \n\n `
504 + `- Generated at: \` ${ capture . generatedAt } \`\n `
505 + `- Adapter: \` rp-source-csv \`\n `
506 + `- Input files: ${ capture . sourceFiles . length } \n `
507 + `- Logical streams: ${ capture . streams . length } \n `
508 + `- Entities documented: ${ capture . streams . reduce (( sum , stream ) => sum + stream . entities . length , 0 ) } \n\n `
509 + haltBanner
510 + `## Files \n\n | File | Role | Vendor | Role source | Part of | \n | --- | --- | --- | --- | --- | \n ${ fileRows } \n\n `
511 + `## Vendor detection \n\n ${ capture . streams . map (( stream ) => `- \` ${ path . basename ( stream . primary ) } \` → **${ stream . vendor . vendor }** (confidence ${ stream . vendor . confidence }, ${ stream . vendor . source }) — ${ stream . vendor . evidence }${ stream . vendor . nearMiss ? ` \n - Near miss: ${ stream . vendor . nearMiss . vendor } (missing required: ${ stream . vendor . nearMiss . missingRequired . join ( ', ' ) || 'none'})` : ''}${ stream . vendor . conflict ? ` \n - ⚠️ user stated \` ${ stream . vendor . conflict . stated } \` but the header looks like \` ${ stream . vendor . conflict . detected } \` ` : ''}` ). join ( ' \n ' ) } \n\n `
512 + `## Layout \n\n | File | Pattern | Key | Confidence | Source | Status | \n | --- | --- | --- | ---: | --- | --- | \n ${ streamRowsMd } \n\n `
513 + `## Entities \n\n | Entity | Origin | Records | In use | Hierarchical | File | \n | --- | --- | ---: | --- | --- | --- | \n ${ entityRows || '| None | - | - | - | - | - |'} \n\n `
514 + `## Profile drift \n\n ${ driftLines } \n `
515 + `## Warnings \n\n ${ warningLines } \n\n `
516 + `## Cross-file joins \n\n ${ capture . joins . deferred ? `Deferred: ${ capture . joins . reason }` : 'None detected'} \n\n `
517 + `Machine-readable capture: [fileset.json](./fileset.json), [raw-capture.json](./raw-capture.json) \n ` ;
518 }
519
520 function filesetPayload ( capture ) {
521 return {
522 generatedAt: capture.generatedAt,
523 adapter: 'rp-source-csv' ,
524 platform: 'csv' ,
525 csvInputRoot: capture.csvInputRoot,
526 halt: capture.halt,
527 sourceFiles: capture.sourceFiles. map (( entry ) => ({
528 ... entry,
529 file: path. relative (capture.csvInputRoot, entry.file) || path. basename (entry.file),
530 partOf: entry.partOf ? path. relative (capture.csvInputRoot, entry.partOf) || path. basename (entry.partOf) : null ,
531 })),
532 streams: capture.streams. map (( stream ) => ({
533 role: stream.role,
534 primary: path. relative (capture.csvInputRoot, stream.primary) || path. basename (stream.primary),
535 // The header row AS READ. This is the authoritative column list for the whole pipeline —
536 // rp-mapper's deterministic resolver joins it against the vendor overlay, so it belongs in
537 // the hand-off artifact and not only in raw-capture.json (which is evidence, not contract).
538 header: stream.header,
539 parts: stream.parts. map (( part ) => ({
540 file: path. relative (capture.csvInputRoot, part.file) || path. basename (part.file),
541 partOf: part.partOf ? path. relative (capture.csvInputRoot, part.partOf) || path. basename (part.partOf) : null ,
542 headerRelation: part.headerRelation,
543 rowOffset: part.rowOffset,
544 rowCount: part.rowCount,
545 })),
546 vendor: {
547 name: stream.vendor.vendor,
548 confidence: stream.vendor.confidence,
549 source: stream.vendor.source,
550 evidence: stream.vendor.evidence,
551 profileVersion: stream.profile ? stream.profile.profileVersion : null ,
552 nearMiss: stream.vendor.nearMiss,
553 conflict: stream.vendor.conflict,
554 },
555 dialect: stream.dialect,
556 layout: {
557 pattern: stream.layout.pattern,
558 groupKey: stream.layout.groupKey,
559 continuation: stream.layout.continuation,
560 discriminatorColumn: stream.layout.discriminatorColumn,
561 childLevels: stream.layout.childLevels,
562 parentRefColumn: stream.layout.parentRefColumn,
563 confidence: stream.layout.confidence,
564 source: stream.layout.source,
565 halt: stream.layout.halt,
566 evidence: stream.layout.evidence,
567 },
568 drift: stream.drift,
569 mappingHints: stream.mappingHints,
570 quirks: stream.profile ? stream.profile.quirks || [] : [],
571 // Layout-level conflicts (an overlay whose declared grouping the rows
572 // contradict) and column-level ones (an overlay bucketing a column against
573 // the derived role) land in one list, because a reviewer is asking the same
574 // question of both: where did this capture stop believing the overlay?
575 layoutConflicts: [ ... (stream.layout.layoutConflicts || []), ... stream.roles.layoutConflicts],
576 ambiguousColumns: stream.roles.ambiguousColumns,
577 entities: stream.entities. map (( entity ) => ({
578 name: entity.name,
579 // Paths stay relative to csvInputRoot so moving the migration folder
580 // does not invalidate the capture.
581 origin: {
582 ... entity.origin,
583 file: path. relative (capture.csvInputRoot, entity.origin.file) || path. basename (entity.origin.file),
584 },
585 recordCount: entity.recordCount,
586 inUse: entity.inUse,
587 hierarchical: Boolean (entity.hierarchical),
588 rawFile: entity.rawFile,
589 columns: entity.columns,
590 relations: entity.relations,
591 })),
592 })),
593 joins: capture.joins.joins,
594 integrity: capture.joins.integrity,
595 joinsDeferred: capture.joins.deferred,
596 conflicts: capture.conflicts,
597 warnings: capture.warnings,
598 };
599 }
600
601 function rawCapturePayload ( capture ) {
602 return {
603 generatedAt: capture.generatedAt,
604 adapter: 'rp-source-csv' ,
605 streams: capture.streams. map (( stream ) => ({
606 primary: path. relative (capture.csvInputRoot, stream.primary) || path. basename (stream.primary),
607 role: stream.role,
608 header: stream.header,
609 dialect: stream.dialect,
610 scan: {
611 rowCount: stream.scan.rowCount,
612 scannedRows: stream.rows. length ,
613 truncated: stream.scan.truncated,
614 raggedRows: stream.scan.raggedRows,
615 },
616 columns: stream.columns,
617 headSample: stream.headSample,
618 tailSample: stream.tailSample,
619 mediaNotes: stream.mediaNotes,
620 })),
621 };
622 }
623
624 async function capture ( args ) {
625 const profiles = loadVendorProfiles ();
626 const generatedAt = new Date (). toISOString ();
627 const resolvedFiles = args.files. map (( file ) => path. resolve (file));
628 const csvInputRoot = process.env. CSV_INPUT_ROOT
629 ? path. resolve (process.env. CSV_INPUT_ROOT )
630 : (resolvedFiles. length === 1 ? path. dirname (resolvedFiles[ 0 ]) : commonDirectory (resolvedFiles));
631
632 // 1. header + dialect + vendor per file
633 const scanned = [];
634 for ( let i = 0 ; i < resolvedFiles. length ; i += 1 ) {
635 const file = resolvedFiles[i];
636 progress?. progress ( `Reading header of ${ path . basename ( file ) }` , {
637 phase: 'discovery' ,
638 step: 'read-header' ,
639 entity: path. basename (file),
640 count: i + 1 ,
641 total: resolvedFiles. length ,
642 unit: 'files' ,
643 });
644 const head = await readHeaderRow (file, { delimiter: args.delimiter, encoding: args.encoding });
645 const vendor = detectVendor (head.header, profiles, { statedVendor: args.vendor });
646 const scan = await scanFile (file, {
647 delimiter: head.delimiter,
648 encoding: head.encoding,
649 scanRows: args.scanRows,
650 header: head.header,
651 });
652 scanned. push ({ file, head, vendor, scan });
653 }
654
655 // 2. roles, split-file concatenation
656 const fileSet = resolveFileSet (
657 scanned. map (( entry ) => ({
658 file: entry.file,
659 header: entry.head.header,
660 vendor: entry.vendor.vendor,
661 profile: entry.vendor.profile,
662 rowCount: entry.scan.rowCount,
663 })),
664 {
665 explicitRoles: args.explicitRoles,
666 policy: args.allowHeaderSuperset ? CONCAT_POLICY . UNION : CONCAT_POLICY . STRICT_SET ,
667 fileOrder: args.fileOrder,
668 },
669 );
670
671 const byFile = new Map (scanned. map (( entry ) => [entry.file, entry]));
672 const warnings = [];
673 const takenEntityNames = new Set ();
674 const streams = [];
675
676 for ( const stream of fileSet.streams) {
677 const primary = byFile. get (stream.primary);
678 // Split parts share one logical stream, so their rows are inferred together.
679 const rows = stream.parts. flatMap (( part ) => (byFile. get (part.file) || { scan: { rows: [] } }).scan.rows);
680 const header = primary.head.header;
681 const profile = primary.vendor.profile;
682
683 progress?. progress ( `Resolving layout for ${ path . basename ( stream . primary ) }` , {
684 phase: 'discovery' ,
685 step: 'resolve-layout' ,
686 entity: path. basename (stream.primary),
687 });
688
689 const layout = classifyLayout (header, rows, { overlayLayout: profile ? profile.layout : null });
690 const roles = deriveColumnRoles (header, rows, layout, {
691 columnGroups: (profile && profile.layout.columnGroups) || [],
692 });
693 const derived = detectDerivedCandidates (header, rows, { profile });
694 const drift = diffProfile (header, profile);
695 const columns = summarizeColumns (header, rows);
696 const mediaNotes = mediaReachability (columns);
697
698 const tail = await readTailSample (stream.primary, {
699 delimiter: primary.head.delimiter,
700 encoding: primary.head.encoding,
701 header,
702 });
703
704 const resolved = {
705 role: stream.role,
706 primary: stream.primary,
707 parts: stream.parts,
708 header,
709 rows,
710 columns,
711 profile,
712 vendor: primary.vendor,
713 layout,
714 roles,
715 derived,
716 drift,
717 mediaNotes,
718 scan: {
719 rowCount: stream.parts. reduce (( sum , part ) => sum + (part.rowCount || 0 ), 0 ),
720 truncated: stream.parts. some (( part ) => (byFile. get (part.file) || { scan: {} }).scan.truncated),
721 raggedRows: stream.parts. reduce (( sum , part ) => sum + ((byFile. get (part.file) || { scan: {} }).scan.raggedRows || 0 ), 0 ),
722 },
723 dialect: {
724 delimiter: primary.head.delimiter,
725 delimiterAmbiguous: primary.head.delimiterAmbiguous,
726 encoding: primary.head.encoding,
727 bom: primary.head.bom,
728 lineEnding: primary.head.lineEnding,
729 emptyPolicy: detectEmptyPolicy (rows),
730 },
731 headSample: rows. slice ( 0 , args.headRows). map (( row ) => row.values),
732 tailSample: tail.skipped ? [] : tail.rows. slice ( - args.tailRows). map (( row ) => row.values),
733 tailSampleSkipped: tail.skipped ? tail.reason : null ,
734 mappingHints: prefillColumnMap (header, profile),
735 };
736
737 resolved.entities = deriveEntities (resolved, takenEntityNames);
738 for ( const entity of resolved.entities) {
739 entity.rawFile = `${ stream . role }--${ entity . name }.md` ;
740 }
741 streams. push (resolved);
742
743 if (resolved.scan.rowCount === 0 ) {
744 warnings. push ({
745 kind: 'empty-file' ,
746 file: path. basename (stream.primary),
747 message: 'the file has a header but no data rows; its entities are advertised but unused' ,
748 });
749 }
750 if (layout.halt) {
751 warnings. push ({
752 kind: 'layout-ambiguous' ,
753 file: path. basename (stream.primary),
754 message: `layout could not be resolved deterministically (${ layout . evidence . join ( '; ' ) }) — ask the user how rows group before synthesizing the schema` ,
755 });
756 }
757 for ( const conflict of layout.layoutConflicts || []) {
758 warnings. push ({
759 kind: conflict.kind,
760 file: path. basename (stream.primary),
761 message: `the ${ primary . vendor . vendor } overlay's declared layout is not what this file's rows show — ${ conflict . resolution }. Details: ${ layout . evidence [ 0 ] }` ,
762 });
763 }
764 if (primary.vendor.nearMiss) {
765 warnings. push ({
766 kind: 'vendor-near-miss' ,
767 file: path. basename (stream.primary),
768 message: `header looks like a drifted ${ primary . vendor . nearMiss . vendor } export (missing required: ${ primary . vendor . nearMiss . missingRequired . join ( ', ' ) || 'none'}); confirm with the user or set CSV_VENDOR` ,
769 });
770 }
771 if (primary.vendor.conflict) {
772 warnings. push ({
773 kind: 'vendor-conflict' ,
774 file: path. basename (stream.primary),
775 message: `user stated "${ primary . vendor . conflict . stated }" but the header fingerprints as "${ primary . vendor . conflict . detected }"` ,
776 });
777 }
778 if (resolved.dialect.delimiterAmbiguous) {
779 warnings. push ({
780 kind: 'delimiter-ambiguous' ,
781 file: path. basename (stream.primary),
782 message: 'delimiter could not be detected confidently; set CSV_DELIMITER' ,
783 });
784 }
785 if (resolved.scan.raggedRows > 0 ) {
786 warnings. push ({
787 kind: 'ragged-rows' ,
788 file: path. basename (stream.primary),
789 message: `${ resolved . scan . raggedRows } row(s) do not have the header's column count` ,
790 });
791 }
792 if (resolved.tailSampleSkipped) {
793 warnings. push ({
794 kind: 'tail-sample-skipped' ,
795 file: path. basename (stream.primary),
796 message: `tail sample skipped (${ resolved . tailSampleSkipped })` ,
797 });
798 }
799 for ( const note of mediaNotes) {
800 warnings. push ({
801 kind: 'media-reachability' ,
802 file: path. basename (stream.primary),
803 message: `column "${ note . column }" holds values Wix cannot fetch (e.g. ${ note . example }): ${ note . note }` ,
804 });
805 }
806 if (roles.ambiguousColumns. length > 0 ) {
807 warnings. push ({
808 kind: 'ambiguous-columns' ,
809 file: path. basename (stream.primary),
810 message: `could not decide which entity these columns belong to: ${ roles . ambiguousColumns . join ( ', ' ) }` ,
811 });
812 }
813 for ( const candidate of derived) {
814 if (candidate.status === 'proposed' ) {
815 warnings. push ({
816 kind: 'derived-entity-proposed' ,
817 file: path. basename (stream.primary),
818 message: `column "${ candidate . fromColumn }" may be a "${ candidate . entity }" entity (confidence ${ candidate . confidence }); confirm with the user before treating it as one` ,
819 });
820 }
821 if (candidate.status === 'missing-column' ) {
822 warnings. push ({
823 kind: 'derived-column-missing' ,
824 file: path. basename (stream.primary),
825 message: `the ${ primary . vendor . vendor } overlay expects a "${ candidate . fromColumn }" column for the "${ candidate . entity }" entity, but this file has none` ,
826 });
827 }
828 }
829 }
830
831 for ( const conflict of fileSet.conflicts) {
832 warnings. push ({ kind: conflict.kind, file: (conflict.files || []). map (( file ) => path. basename (file)). join ( ', ' ), message: conflict.message });
833 }
834
835 return {
836 generatedAt,
837 csvInputRoot,
838 sourceFiles: fileSet.sourceFiles,
839 streams,
840 conflicts: fileSet.conflicts,
841 joins: inferJoins (),
842 warnings,
843 halt: fileSet.halt || streams. some (( stream ) => stream.layout.halt),
844 };
845 }
846
847 function commonDirectory ( files ) {
848 if (files. length === 0 ) {
849 return process. cwd ();
850 }
851 const segments = files. map (( file ) => path. dirname (file). split (path.sep));
852 const common = [];
853 for ( let i = 0 ; i < segments[ 0 ]. length ; i += 1 ) {
854 const candidate = segments[ 0 ][i];
855 if (segments. every (( parts ) => parts[i] === candidate)) {
856 common. push (candidate);
857 } else {
858 break ;
859 }
860 }
861 return common. join (path.sep) || path.sep;
862 }
863
864 async function writeOutputs ( outDir , result ) {
865 await fs. mkdir (outDir, { recursive: true });
866 await fs. writeFile (path. join (outDir, 'README.md' ), renderIndexFile (result), 'utf8' );
867 progress?. progress ( 'Wrote CSV discovery index' , { phase: 'discovery' , step: 'write-artifact' , artifact: path. join (outDir, 'README.md' ) });
868
869 await fs. writeFile (path. join (outDir, 'fileset.json' ), `${ JSON . stringify ( filesetPayload ( result ), null , 2 ) } \n ` , 'utf8' );
870 await fs. writeFile (path. join (outDir, 'raw-capture.json' ), `${ JSON . stringify ( rawCapturePayload ( result ), null , 2 ) } \n ` , 'utf8' );
871 progress?. progress ( 'Wrote CSV discovery machine capture' , { phase: 'discovery' , step: 'write-artifact' , artifact: path. join (outDir, 'fileset.json' ) });
872
873 let entityCount = 0 ;
874 for ( const stream of result.streams) {
875 const columnsByName = new Map (stream.columns. map (( column ) => [column.name, column]));
876 for ( const entity of stream.entities) {
877 await fs. writeFile (path. join (outDir, entity.rawFile), renderEntityFile (stream, entity, columnsByName), 'utf8' );
878 entityCount += 1 ;
879 progress?. progress ( `Wrote CSV discovery artifact for ${ entity . name }` , {
880 phase: 'discovery' ,
881 step: 'write-artifact' ,
882 entity: entity.name,
883 artifact: path. join (outDir, entity.rawFile),
884 });
885 }
886 }
887 return entityCount;
888 }
889
890 async function main () {
891 const parsed = parseProgressArgs (process.argv. slice ( 2 ));
892 progress = createProgressLogger ({
893 script: 'skills/replatform/resources/rp-source-csv/scripts/csv-discovery.js' ,
894 ... parsed.progress,
895 });
896 progress. start ( 'CSV discovery started' , { phase: 'discovery' });
897
898 const args = parseArgs (parsed.args);
899 if (args.help) {
900 printUsage ();
901 progress. complete ( 'CSV discovery help shown' , { phase: 'discovery' , step: 'help' });
902 return ;
903 }
904 if (args.files. length === 0 || ! args.outDir) {
905 printUsage ();
906 progress. error ( 'Missing required CSV discovery arguments' , { phase: 'discovery' });
907 throw new Error ( 'Missing required arguments: at least one --file and --out-dir are required.' );
908 }
909
910 const result = await capture (args);
911 const entityCount = await writeOutputs (path. resolve (args.outDir), result);
912
913 console. log ( `Wrote ${ entityCount } entity file(s), README.md, fileset.json and raw-capture.json to ${ args . outDir }` );
914 if (result.halt) {
915 console. log ( 'Discovery halted to a decision — see the Warnings section of README.md before synthesizing source-schema.json.' );
916 }
917 progress. complete ( 'CSV discovery completed' , {
918 phase: 'discovery' ,
919 artifact: args.outDir,
920 count: entityCount,
921 unit: 'entities' ,
922 });
923 }
924
925 if (require.main === module ) {
926 main (). catch (( error ) => {
927 console. error (error.stack || error.message);
928 if (progress) {
929 progress. error (error && error.message ? error.message : 'CSV discovery failed' , { phase: 'discovery' });
930 }
931 process.exitCode = 1 ;
932 });
933 }
934
935 module . exports = {
936 DEFAULT_HEAD_ROWS,
937 DEFAULT_TAIL_ROWS,
938 DEFAULT_SCAN_ROWS,
939 parseArgs,
940 scanFile,
941 summarizeColumns,
942 mediaReachability,
943 deriveEntities,
944 renderIndexFile,
945 renderEntityFile,
946 filesetPayload,
947 rawCapturePayload,
948 capture,
949 };