Setting the file. One moment.
CSV Parse · Rp Source CSV · wix/skills · Skills Docs
Back to the top of the page function parseText
— line 269
This file
Number 17.6
Position 6 of 11
Type JavaScript
Size 19 KB
Lines 618 lib/ csv-parse.js
JavaScript · 618 lines · 19 KB
15
// Keep this file dependency-free (Node built-ins only) so the vendored copy
16 // needs no install step.
17
18 const fs = require ( 'node:fs' );
19 const fsp = require ( 'node:fs/promises' );
20 const { StringDecoder } = require ( 'node:string_decoder' );
21
22 const DELIMITER_CANDIDATES = [ ',' , ';' , ' \t ' , '|' ];
23 const DELIMITER_SAMPLE_LINES = 20 ;
24 const DELIMITER_MIN_SCORE = 1.0 ;
25 const DELIMITER_MIN_MARGIN = 1.25 ;
26 const DEFAULT_SAMPLE_BYTES = 256 * 1024 ;
27 const TAIL_SAMPLE_BYTES = 64 * 1024 ;
28
29 // How this writer represents an absent value. Detected per file rather than
30 // assumed: exporters that quote every field (Excel, Magento) can never produce
31 // an unquoted empty, and exporters that quote nothing unnecessary can never
32 // produce a quoted empty. Only a file containing BOTH forms is actually
33 // distinguishing them.
34 const EMPTY_POLICY = {
35 PRESENT_IF_QUOTED: 'present-if-quoted' ,
36 ALWAYS_EMPTY: 'always-empty' ,
37 };
38
39 const ENCODING_ALIASES = {
40 utf8: 'utf8' ,
41 'utf-8' : 'utf8' ,
42 ascii: 'ascii' ,
43 latin1: 'latin1' ,
44 'iso-8859-1' : 'latin1' ,
45 'windows-1252' : 'latin1' ,
46 cp1252: 'latin1' ,
47 utf16le: 'utf16le' ,
48 'utf-16le' : 'utf16le' ,
49 };
50
51 // Parser states. A field is quoted only when the quote opens at field position
52 // 0; a bare `"` anywhere else is a literal character.
53 const S_FIELD_START = 0 ;
54 const S_IN_FIELD = 1 ;
55 const S_IN_QUOTED = 2 ;
56 const S_QUOTE_IN_QUOTED = 3 ;
57
58 function nodeEncodingFor ( encoding ) {
59 const key = String (encoding || 'utf8' ). toLowerCase ();
60 const mapped = ENCODING_ALIASES [key];
61 if ( ! mapped) {
62 throw new Error ( `Unsupported CSV encoding "${ encoding }". Set CSV_ENCODING to one of: ${ Object . keys ( ENCODING_ALIASES ). join ( ', ' ) }.` );
63 }
64 return mapped;
65 }
66
67 function stripBom ( text ) {
68 if ( typeof text === 'string' && text. charCodeAt ( 0 ) === 0xfeff ) {
69 return { text: text. slice ( 1 ), bom: 'utf-8' };
70 }
71 return { text, bom: null };
72 }
73
74 // Byte-order marks decide the encoding before any decoding happens. UTF-16 is
75 // reported as unsupported rather than silently decoded as UTF-8 mojibake;
76 // full transcoding is deliberately out of scope (see SKILL.md → Encoding).
77 function detectEncoding ( buffer ) {
78 if ( ! buffer || buffer. length === 0 ) {
79 return { encoding: 'utf8' , bom: null , supported: true };
80 }
81 if (buffer. length >= 3 && buffer[ 0 ] === 0xef && buffer[ 1 ] === 0xbb && buffer[ 2 ] === 0xbf ) {
82 return { encoding: 'utf8' , bom: 'utf-8' , supported: true };
83 }
84 if (buffer. length >= 2 && buffer[ 0 ] === 0xff && buffer[ 1 ] === 0xfe ) {
85 return { encoding: 'utf16le' , bom: 'utf-16le' , supported: false };
86 }
87 if (buffer. length >= 2 && buffer[ 0 ] === 0xfe && buffer[ 1 ] === 0xff ) {
88 return { encoding: 'utf16be' , bom: 'utf-16be' , supported: false };
89 }
90 return { encoding: 'utf8' , bom: null , supported: true };
91 }
92
93 function detectLineEnding ( sampleText ) {
94 const text = String (sampleText || '' );
95 const crlf = (text. match ( / \r\n / g ) || []). length ;
96 const cr = (text. match ( / \r (?! \n )/ g ) || []). length ;
97 const lf = (text. match ( /(?<! \r ) \n / g ) || []). length ;
98 const ranked = [[ ' \r\n ' , crlf], [ ' \n ' , lf], [ ' \r ' , cr]]. filter (([, count ]) => count > 0 );
99 if (ranked. length === 0 ) {
100 return null ;
101 }
102 if (ranked. length > 1 ) {
103 return 'mixed' ;
104 }
105 return ranked[ 0 ][ 0 ];
106 }
107
108 // The one normalizer. Fingerprint, layout, and fileset all import this rather
109 // than rolling their own, so `Body (HTML)`, `body_html`, and `BODY HTML` are
110 // the same column everywhere in the adapter.
111 function normalizeHeaderName ( name ) {
112 return String (name === undefined || name === null ? '' : name)
113 . replace ( / ^\uFEFF / , '' )
114 . normalize ( 'NFKC' )
115 . toLowerCase ()
116 . replace ( / [ ^ a-z0-9] + / g , '' );
117 }
118
119 // Chunk-fed RFC-4180 state machine.
120 //
121 // Returns rows as `{ values: string[], quoted?: boolean[], line }`. Values are
122 // ALWAYS strings — never null. Whether an empty string means "absent" is a
123 // property of the writer's quoting policy, not of the datum, so that decision
124 // is made once per file by detectEmptyPolicy and applied by coerceEmpty.
125 function createParser ({ delimiter = ',' , trackQuoted = false , skipEmptyLines = true } = {}) {
126 const delim = String (delimiter);
127 let state = S_FIELD_START ;
128 let field = '' ;
129 let fieldWasQuoted = false ;
130 let values = [];
131 let quotedFlags = [];
132 let pendingCR = false ;
133 let lineNo = 1 ;
134 let rowStartLine = 1 ;
135 let blankLines = 0 ;
136
137 function endField () {
138 values. push (field);
139 if (trackQuoted) {
140 quotedFlags. push (fieldWasQuoted);
141 }
142 field = '' ;
143 fieldWasQuoted = false ;
144 state = S_FIELD_START ;
145 }
146
147 function endRow ( rows ) {
148 endField ();
149 const isBlank = values. length === 1 && values[ 0 ] === '' && ! (trackQuoted && quotedFlags[ 0 ]);
150 if (skipEmptyLines && isBlank) {
151 blankLines += 1 ;
152 } else {
153 const row = { values, line: rowStartLine };
154 if (trackQuoted) {
155 row.quoted = quotedFlags;
156 }
157 rows. push (row);
158 }
159 values = [];
160 quotedFlags = [];
161 lineNo += 1 ;
162 rowStartLine = lineNo;
163 }
164
165 function push ( text ) {
166 const rows = [];
167 const input = String (text);
168 for ( let i = 0 ; i < input. length ; i += 1 ) {
169 const ch = input[i];
170
171 // A lone \r can only be resolved by the next character, which may live in
172 // the next chunk — hence the flag rather than a lookahead.
173 if (pendingCR) {
174 pendingCR = false ;
175 if (ch === ' \n ' ) {
176 continue ;
177 }
178 }
179
180 switch (state) {
181 case S_FIELD_START :
182 if (ch === '"' ) {
183 state = S_IN_QUOTED ;
184 fieldWasQuoted = true ;
185 } else if (ch === delim) {
186 endField ();
187 } else if (ch === ' \n ' ) {
188 endRow (rows);
189 } else if (ch === ' \r ' ) {
190 endRow (rows);
191 pendingCR = true ;
192 } else {
193 field += ch;
194 state = S_IN_FIELD ;
195 }
196 break ;
197
198 case S_IN_FIELD :
199 if (ch === delim) {
200 endField ();
201 } else if (ch === ' \n ' ) {
202 endRow (rows);
203 } else if (ch === ' \r ' ) {
204 endRow (rows);
205 pendingCR = true ;
206 } else {
207 // A quote inside an unquoted field is a literal character.
208 field += ch;
209 }
210 break ;
211
212 case S_IN_QUOTED :
213 if (ch === '"' ) {
214 state = S_QUOTE_IN_QUOTED ;
215 } else {
216 // Newlines inside quotes are data and are preserved verbatim,
217 // including the \r of a \r\n pair.
218 if (ch === ' \n ' ) {
219 lineNo += 1 ;
220 }
221 field += ch;
222 }
223 break ;
224
225 case S_QUOTE_IN_QUOTED :
226 if (ch === '"' ) {
227 field += '"' ;
228 state = S_IN_QUOTED ;
229 } else if (ch === delim) {
230 endField ();
231 } else if (ch === ' \n ' ) {
232 endRow (rows);
233 } else if (ch === ' \r ' ) {
234 endRow (rows);
235 pendingCR = true ;
236 } else {
237 field += ch;
238 state = S_IN_FIELD ;
239 }
240 break ;
241
242 default :
243 break ;
244 }
245 }
246 return rows;
247 }
248
249 function end () {
250 const rows = [];
251 const hasPendingContent = values. length > 0
252 || field !== ''
253 || fieldWasQuoted
254 || state === S_IN_QUOTED
255 || state === S_QUOTE_IN_QUOTED ;
256 if (hasPendingContent) {
257 endRow (rows);
258 }
259 return rows;
260 }
261
262 function stats () {
263 return { blankLines, unterminatedQuote: state === S_IN_QUOTED };
264 }
265
266 return { push, end, stats };
267 }
268
269 function parseText ( text , { delimiter = ',' , trackQuoted = false , skipEmptyLines = true } = {}) {
270 const parser = createParser ({ delimiter, trackQuoted, skipEmptyLines });
271 const stripped = stripBom ( String (text === undefined || text === null ? '' : text));
272 const rows = parser. push (stripped.text);
273 return rows. concat (parser. end ());
274 }
275
276 function quoteAwareFieldCounts ( sampleText , delimiter , maxLines ) {
277 const rows = parseText (sampleText, { delimiter });
278 return rows. slice ( 0 , maxLines). map (( row ) => row.values. length );
279 }
280
281 // Score each candidate by how consistently it splits the sample AND by how many
282 // fields it produces. Consistency alone is not enough: a semicolon file whose
283 // text fields each contain one comma splits perfectly consistently on `,` too.
284 function detectDelimiter ( sampleText , { candidates = DELIMITER_CANDIDATES , maxLines = DELIMITER_SAMPLE_LINES } = {}) {
285 const scored = candidates. map (( delimiter ) => {
286 const counts = quoteAwareFieldCounts (sampleText, delimiter, maxLines);
287 if (counts. length === 0 || counts[ 0 ] < 2 ) {
288 return { delimiter, fieldCount: counts[ 0 ] || 0 , consistency: 0 , score: 0 };
289 }
290 const consistency = counts. filter (( count ) => count === counts[ 0 ]). length / counts. length ;
291 return {
292 delimiter,
293 fieldCount: counts[ 0 ],
294 consistency,
295 score: consistency * Math. log2 (counts[ 0 ]),
296 };
297 });
298
299 const ranked = [ ... scored]. sort (( a , b ) => b.score - a.score
300 || candidates. indexOf (a.delimiter) - candidates. indexOf (b.delimiter));
301 const winner = ranked[ 0 ];
302 const runnerUp = ranked[ 1 ];
303 const clearsScore = winner.score >= DELIMITER_MIN_SCORE ;
304 const clearsMargin = ! runnerUp || runnerUp.score === 0 || winner.score / runnerUp.score >= DELIMITER_MIN_MARGIN ;
305 const ambiguous = ! (clearsScore && clearsMargin);
306
307 return {
308 delimiter: ambiguous ? (candidates[ 0 ] || ',' ) : winner.delimiter,
309 confidence: ambiguous ? Math. min ( 0.5 , winner.score / 2 ) : Math. min ( 1 , 0.6 + winner.score / 10 ),
310 ambiguous,
311 candidates: ranked,
312 };
313 }
314
315 async function readSample ( filePath , { bytes = DEFAULT_SAMPLE_BYTES , encoding = null } = {}) {
316 const handle = await fsp. open (filePath, 'r' );
317 try {
318 const stat = await handle. stat ();
319 const length = Math. min (bytes, stat.size);
320 const buffer = Buffer. alloc (length);
321 await handle. read (buffer, 0 , length, 0 );
322 const detected = detectEncoding (buffer);
323 if ( ! encoding && ! detected.supported) {
324 throw new Error ( `${ filePath } looks like ${ detected . encoding } (BOM ${ detected . bom }); only UTF-8 family encodings are supported. Set CSV_ENCODING or convert the file to UTF-8.` );
325 }
326 const decoder = new StringDecoder ( nodeEncodingFor (encoding || detected.encoding));
327 const decoded = stripBom (decoder. write (buffer) + decoder. end ());
328 const truncated = length < stat.size;
329 // A truncated sample almost certainly ends mid-line; dropping the tail keeps
330 // field-count statistics honest.
331 const text = truncated ? decoded.text. slice ( 0 , decoded.text. lastIndexOf ( ' \n ' ) + 1 ) : decoded.text;
332 return {
333 text,
334 bom: detected.bom,
335 encoding: encoding || detected.encoding,
336 truncated,
337 totalBytes: stat.size,
338 };
339 } finally {
340 await handle. close ();
341 }
342 }
343
344 async function readHeaderRow ( filePath , { delimiter = null , encoding = null , sampleBytes = TAIL_SAMPLE_BYTES } = {}) {
345 const sample = await readSample (filePath, { bytes: sampleBytes, encoding });
346 const dialect = delimiter
347 ? { delimiter, confidence: 1 , ambiguous: false , candidates: [] }
348 : detectDelimiter (sample.text);
349 const rows = parseText (sample.text, { delimiter: dialect.delimiter });
350 const header = rows. length > 0 ? rows[ 0 ].values : [];
351 return {
352 header,
353 delimiter: dialect.delimiter,
354 delimiterAmbiguous: dialect.ambiguous,
355 delimiterCandidates: dialect.candidates,
356 encoding: sample.encoding,
357 bom: sample.bom,
358 lineEnding: detectLineEnding (sample.text),
359 rawHeaderLine: sample.text. split ( / \r\n | \n | \r / )[ 0 ] || '' ,
360 };
361 }
362
363 // Read the last window of the file for tail samples. A quoted field containing
364 // a newline can straddle the window boundary and misparse silently, so the
365 // sample is only accepted when the quote count is even and every row has the
366 // header's width.
367 async function readTailSample ( filePath , { bytes = TAIL_SAMPLE_BYTES , delimiter = ',' , encoding = 'utf8' , header = [] } = {}) {
368 const handle = await fsp. open (filePath, 'r' );
369 try {
370 const stat = await handle. stat ();
371 if (stat.size === 0 ) {
372 return { rows: [], skipped: false , reason: null };
373 }
374 const length = Math. min (bytes, stat.size);
375 const position = stat.size - length;
376 const buffer = Buffer. alloc (length);
377 await handle. read (buffer, 0 , length, position);
378 const decoder = new StringDecoder ( nodeEncodingFor (encoding));
379 let text = decoder. write (buffer) + decoder. end ();
380 if (position > 0 ) {
381 const firstBreak = text. indexOf ( ' \n ' );
382 if (firstBreak === - 1 ) {
383 return { rows: [], skipped: true , reason: 'no-line-break-in-window' };
384 }
385 text = text. slice (firstBreak + 1 );
386 } else {
387 text = stripBom (text).text;
388 }
389 const quoteCount = (text. match ( /"/ g ) || []). length ;
390 if (position > 0 && quoteCount % 2 !== 0 ) {
391 return { rows: [], skipped: true , reason: 'odd-quote-count-in-window' };
392 }
393 const rows = parseText (text, { delimiter });
394 const body = position > 0 ? rows : rows. slice ( 1 );
395 if (header. length > 0 && body. some (( row ) => row.values. length !== header. length )) {
396 return { rows: [], skipped: true , reason: 'ragged-rows-in-window' };
397 }
398 return { rows: body, skipped: false , reason: null };
399 } finally {
400 await handle. close ();
401 }
402 }
403
404 async function* streamRows ( filePath , {
405 delimiter = null ,
406 encoding = null ,
407 trackQuoted = false ,
408 skipEmptyLines = true ,
409 maxRows = null ,
410 } = {}) {
411 let resolvedDelimiter = delimiter;
412 let resolvedEncoding = encoding;
413 if ( ! resolvedDelimiter || ! resolvedEncoding) {
414 const sample = await readSample (filePath, { encoding });
415 resolvedEncoding = resolvedEncoding || sample.encoding;
416 resolvedDelimiter = resolvedDelimiter || detectDelimiter (sample.text).delimiter;
417 }
418
419 const decoder = new StringDecoder ( nodeEncodingFor (resolvedEncoding));
420 const parser = createParser ({ delimiter: resolvedDelimiter, trackQuoted, skipEmptyLines });
421 const stream = fs. createReadStream (filePath);
422 let emitted = 0 ;
423 let first = true ;
424
425 for await ( const chunk of stream) {
426 let text = decoder. write (chunk);
427 if (first) {
428 text = stripBom (text).text;
429 first = false ;
430 }
431 for ( const row of parser. push (text)) {
432 yield row;
433 emitted += 1 ;
434 if (maxRows !== null && emitted >= maxRows) {
435 stream. destroy ();
436 return ;
437 }
438 }
439 }
440
441 const tail = decoder. end ();
442 const pending = tail ? parser. push (tail) : [];
443 for ( const row of pending. concat (parser. end ())) {
444 yield row;
445 emitted += 1 ;
446 if (maxRows !== null && emitted >= maxRows) {
447 return ;
448 }
449 }
450 }
451
452 async function* streamRecords ( filePath , options = {}) {
453 let header = options.header || null ;
454 for await ( const row of streamRows (filePath, options)) {
455 if ( ! header) {
456 header = row.values;
457 continue ;
458 }
459 yield { record: toRecord (header, row.values), row };
460 }
461 }
462
463 function toRecord ( header , values ) {
464 const record = {};
465 for ( let i = 0 ; i < header. length ; i += 1 ) {
466 record[header[i]] = i < values. length ? values[i] : '' ;
467 }
468 return record;
469 }
470
471 function rowWidthReport ( header , values ) {
472 return {
473 expected: header. length ,
474 actual: values. length ,
475 padded: values. length < header. length ,
476 truncated: values. length > header. length ,
477 ragged: values. length !== header. length ,
478 };
479 }
480
481 // Requires rows parsed with `trackQuoted: true`.
482 function detectEmptyPolicy ( rows ) {
483 let sawQuotedEmpty = false ;
484 let sawUnquotedEmpty = false ;
485 for ( const row of rows) {
486 const quoted = row.quoted || [];
487 for ( let i = 0 ; i < row.values. length ; i += 1 ) {
488 if (row.values[i] !== '' ) {
489 continue ;
490 }
491 if (quoted[i]) {
492 sawQuotedEmpty = true ;
493 } else {
494 sawUnquotedEmpty = true ;
495 }
496 if (sawQuotedEmpty && sawUnquotedEmpty) {
497 return EMPTY_POLICY . PRESENT_IF_QUOTED ;
498 }
499 }
500 }
501 return EMPTY_POLICY . ALWAYS_EMPTY ;
502 }
503
504 // The one place empty-vs-absent is decided. Generated readers call this so a
505 // required Wix field is never fed an empty string the source did not have.
506 function coerceEmpty ( value , isQuoted , policy ) {
507 if (value !== '' ) {
508 return value;
509 }
510 if (policy === EMPTY_POLICY . PRESENT_IF_QUOTED ) {
511 return isQuoted ? '' : null ;
512 }
513 return '' ;
514 }
515
516 const TYPE_TESTS = [
517 [ 'integer' , ( value ) => / ^ - ? \d +$ / . test (value)],
518 [ 'number' , ( value ) => / ^ - ? ( \d + \. \d *| \. \d +| \d + ) $ / . test (value)],
519 [ 'date-time' , ( value ) => / ^ \d {4} - \d {2} - \d {2} [T ]\d {2} : \d {2} / . test (value)],
520 [ 'date' , ( value ) => / ^ \d {4} - \d {2} - \d {2}$ / . test (value)],
521 [ 'url' , ( value ) => / ^ https ? : \/\/ \S +$ / i . test (value)],
522 [ 'email' , ( value ) => / ^ [ ^ @\s] + @ [ ^ @\s] + \. [ ^ @\s] +$ / . test (value)],
523 ];
524
525 const BOOLEAN_VALUES = new Set ([ 'true' , 'false' , 'yes' , 'no' ]);
526
527 function inferValueType ( values ) {
528 const present = values. filter (( value ) => typeof value === 'string' && value. trim () !== '' );
529 if (present. length === 0 ) {
530 return 'unknown' ;
531 }
532 if (present. every (( value ) => BOOLEAN_VALUES . has (value. trim (). toLowerCase ()))) {
533 return 'boolean' ;
534 }
535 for ( const [ type , test ] of TYPE_TESTS ) {
536 if (present. every (( value ) => test (value. trim ()))) {
537 return type;
538 }
539 }
540 return 'string' ;
541 }
542
543 function summarizeColumn ( name , samples , { maxExamples = 3 , distinctCap = 5000 } = {}) {
544 const distinct = new Set ();
545 let blankCount = 0 ;
546 let quotedEmptyCount = 0 ;
547 let maxLength = 0 ;
548 let multiValueHint = false ;
549 const examples = [];
550 const present = [];
551
552 for ( const sample of samples) {
553 const value = typeof sample === 'string' ? sample : sample.value;
554 const isQuoted = typeof sample === 'string' ? false : Boolean (sample.quoted);
555 if (value === '' || value === null || value === undefined ) {
556 blankCount += 1 ;
557 if (isQuoted) {
558 quotedEmptyCount += 1 ;
559 }
560 continue ;
561 }
562 present. push (value);
563 if (distinct.size < distinctCap) {
564 distinct. add (value);
565 }
566 maxLength = Math. max (maxLength, value. length );
567 if ( ! multiValueHint && / [,;|] / . test (value) && value. length < 200 ) {
568 multiValueHint = true ;
569 }
570 if (examples. length < maxExamples) {
571 examples. push (value);
572 }
573 }
574
575 const total = samples. length ;
576 return {
577 name,
578 type: inferValueType (present),
579 required: total > 0 && blankCount === 0 ,
580 blankCount,
581 quotedEmptyCount,
582 sampled: total,
583 distinctCount: distinct.size,
584 distinctCapped: distinct.size >= distinctCap,
585 unique: present. length > 0 && distinct.size === present. length && distinct.size < distinctCap,
586 maxLength,
587 multiValueHint,
588 examples,
589 };
590 }
591
592 module . exports = {
593 DELIMITER_CANDIDATES,
594 DELIMITER_SAMPLE_LINES,
595 DELIMITER_MIN_SCORE,
596 DELIMITER_MIN_MARGIN,
597 DEFAULT_SAMPLE_BYTES,
598 TAIL_SAMPLE_BYTES,
599 EMPTY_POLICY,
600 createParser,
601 parseText,
602 detectDelimiter,
603 detectEncoding,
604 detectLineEnding,
605 stripBom,
606 normalizeHeaderName,
607 readSample,
608 readHeaderRow,
609 readTailSample,
610 streamRows,
611 streamRecords,
612 toRecord,
613 rowWidthReport,
614 detectEmptyPolicy,
615 coerceEmpty,
616 inferValueType,
617 summarizeColumn,
618 };