Setting the file. One moment.
Read Back Extract · Wix Replatform · wix/skills · Skills Docs
Back to the top of the page lib/read-back-extract.js
lib/ read-back-extract.js
JavaScript · 118 lines · 4 KB
code
9 });
10 }
11 function parseRecord ( text , maxRecordBytes ) {
12 if (Buffer. byteLength (text) > maxRecordBytes) throw fail ( 'extract_record_too_large' );
13 let record;
14 try {
15 record = JSON . parse (text);
16 } catch {
17 throw fail ( 'extract_json_invalid' );
18 }
19 if ( ! record || typeof record !== 'object' || Array. isArray (record)) throw fail ( 'extract_record_invalid' );
20 return record;
21 }
22 async function* readExtract ( file , {
23 format = 'auto' ,
24 maxRecordBytes = MAX_RECORD_BYTES
25 } = {}) {
26 if ( ! [ 'auto' , 'json' , 'ndjson' ]. includes (format) || ! Number. isSafeInteger (maxRecordBytes) || maxRecordBytes < 1 ) throw fail ( 'extract_options_invalid' );
27 const stream = fs. createReadStream (file, {
28 encoding: 'utf8' ,
29 highWaterMark: 64 * 1024
30 });
31 let selected = format === 'auto' ? null : format;
32 let buffer = '' ,
33 depth = 0 ,
34 quoted = false ,
35 escaped = false ,
36 opened = false ,
37 closed = false ,
38 expect = 'first' ,
39 recordBytes = 0 ;
40 try {
41 for await ( let chunk of stream) {
42 if ( ! opened && ! buffer && chunk. charCodeAt ( 0 ) === 0xfeff ) chunk = chunk. slice ( 1 );
43 if ( ! selected) {
44 const first = chunk. search ( / \S / );
45 if (first < 0 ) continue ;
46 selected = chunk[first] === '[' ? 'json' : 'ndjson' ;
47 }
48 if (selected === 'ndjson' ) {
49 // Bound every partial line before reading the next chunk.
50 let start = 0 ;
51 for ( let i = 0 ; i < chunk. length ; i += 1 ) {
52 if (chunk[i] !== ' \n ' ) continue ;
53 buffer += chunk. slice (start, i);
54 if (buffer. trim ()) yield parseRecord (buffer, maxRecordBytes);
55 buffer = '' ;
56 start = i + 1 ;
57 }
58 buffer += chunk. slice (start);
59 if (Buffer. byteLength (buffer) > maxRecordBytes) throw fail ( 'extract_record_too_large' );
60 continue ;
61 }
62 for ( const ch of chunk) {
63 if (closed) {
64 if ( / \S / . test (ch)) throw fail ( 'extract_json_trailing_data' );
65 continue ;
66 }
67 if ( ! opened) {
68 if ( / \s / . test (ch)) continue ;
69 if (ch !== '[' ) throw fail ( 'extract_json_array_required' );
70 opened = true ;
71 continue ;
72 }
73 if ( ! buffer) {
74 if ( / \s / . test (ch)) continue ;
75 if (ch === ']' ) {
76 if (expect === 'value' ) throw fail ( 'extract_json_invalid' );
77 closed = true ;
78 continue ;
79 }
80 if (expect === 'comma' ) {
81 if (ch !== ',' ) throw fail ( 'extract_json_invalid' );
82 expect = 'value' ;
83 continue ;
84 }
85 if (ch !== '{' ) throw fail ( 'extract_record_invalid' );
86 buffer = '{' ;
87 depth = 1 ;
88 quoted = false ;
89 escaped = false ;
90 recordBytes = 1 ;
91 continue ;
92 }
93 buffer += ch;
94 recordBytes += Buffer. byteLength (ch);
95 if (recordBytes > maxRecordBytes) throw fail ( 'extract_record_too_large' );
96 if (quoted) {
97 if (escaped) escaped = false ; else if (ch === ' \\ ' ) escaped = true ; else if (ch === '"' ) quoted = false ;
98 } else if (ch === '"' ) quoted = true ; else if (ch === '{' || ch === '[' ) depth += 1 ; else if (ch === '}' || ch === ']' ) depth -= 1 ;
99 if (depth === 0 ) {
100 yield parseRecord (buffer, maxRecordBytes);
101 buffer = '' ;
102 expect = 'comma' ;
103 }
104 if (depth < 0 ) throw fail ( 'extract_json_invalid' );
105 }
106 }
107 if (selected === 'json' && ( ! opened || ! closed || buffer)) throw fail ( 'extract_json_incomplete' );
108 if (selected === 'ndjson' && buffer. trim ()) yield parseRecord (buffer, maxRecordBytes);
109 // Empty NDJSON is a valid empty extract; empty JSON is never a valid array.
110 if (format === 'json' && ! opened) throw fail ( 'extract_json_incomplete' );
111 } finally {
112 stream. destroy ();
113 }
114 }
115 module . exports = {
116 MAX_RECORD_BYTES,
117 readExtract
118 };