Setting the file. One moment.
CSV Fingerprint · Rp Source CSV · wix/skills · Skills Docs
Back to the top of the page lib/ csv-fingerprint.js
JavaScript · 310 lines · 11 KB
12
const
{
normalizeHeaderName
}
=
require
(
'./csv-parse.js'
);
13
14 const VENDORS_DIR = path. join (__dirname, '..' , 'vendors' );
15
16 // Scoring constants. Deliberately conservative: falling back to `custom` costs
17 // the user a less complete mapping pre-fill, while a wrong vendor match costs
18 // them a wrong layout.
19 const BASE_REQUIRED = 0.5 ;
20 const STRONG_WEIGHT = 0.5 ;
21 const PARTIAL_FACTOR = 0.6 ;
22 const NEGATIVE_PENALTY = 0.35 ;
23 const MATCH_THRESHOLD = 0.7 ;
24 const MARGIN = 0.15 ;
25 const MIN_STRONG_HITS = 2 ;
26 const NEAR_MISS_FLOOR = 0.25 ;
27 const NEAR_MISS_STRONG_RATIO = 0.6 ;
28
29 const CUSTOM_VENDOR = 'custom' ;
30
31 function normalizedSet ( columns ) {
32 const set = new Set ();
33 for ( const column of columns || []) {
34 const normalized = normalizeHeaderName (column);
35 if (normalized) {
36 set. add (normalized);
37 }
38 }
39 return set;
40 }
41
42 function validateProfile ( profile ) {
43 const errors = [];
44 if ( ! profile || typeof profile !== 'object' ) {
45 return { ok: false , errors: [ 'profile is not an object' ] };
46 }
47 if ( ! profile.vendor || typeof profile.vendor !== 'string' ) {
48 errors. push ( 'vendor must be a non-empty string' );
49 }
50 if ( ! profile.profileVersion) {
51 errors. push ( 'profileVersion is required (provenance for drift)' );
52 }
53 if ( ! profile.sourceOfTruth) {
54 errors. push ( 'sourceOfTruth is required' );
55 }
56 const anchors = profile.anchors || {};
57 if ( ! Array. isArray (anchors.required) || anchors.required. length === 0 ) {
58 errors. push ( 'anchors.required must be a non-empty array' );
59 }
60 if ( ! Array. isArray (anchors.strong) || anchors.strong. length < MIN_STRONG_HITS ) {
61 errors. push ( `anchors.strong must hold at least ${ MIN_STRONG_HITS } columns` );
62 }
63 if ( ! profile.layout || typeof profile.layout.pattern !== 'string' ) {
64 errors. push ( 'layout.pattern is required' );
65 }
66 if (profile.columnMap && ! Array. isArray (profile.columnMap)) {
67 errors. push ( 'columnMap must be an array when present' );
68 }
69 for ( const entry of profile.columnMap || []) {
70 if ( ! entry.wixTarget || ! Array. isArray (entry.aliases) || entry.aliases. length === 0 ) {
71 errors. push ( `columnMap entry ${ JSON . stringify ( entry . wixTarget ) } needs a wixTarget and a non-empty aliases array` );
72 }
73 }
74 for ( const derived of profile.derived || []) {
75 if ( ! derived.entity || ! derived.fromColumn) {
76 errors. push ( 'each derived entry needs entity + fromColumn' );
77 }
78 }
79 return { ok: errors. length === 0 , errors };
80 }
81
82 function loadVendorProfiles ( dir = VENDORS_DIR ) {
83 const files = fs. readdirSync (dir)
84 . filter (( file ) => file. endsWith ( '.json' ))
85 . sort ();
86 return files. map (( file ) => {
87 const profile = JSON . parse (fs. readFileSync (path. join (dir, file), 'utf8' ));
88 const validation = validateProfile (profile);
89 if ( ! validation.ok) {
90 throw new Error ( `Invalid vendor profile ${ file }: ${ validation . errors . join ( '; ' ) }` );
91 }
92 return profile;
93 });
94 }
95
96 function aliasIndex ( profile ) {
97 // normalized column name -> wixTarget, built from columnMap only.
98 const index = new Map ();
99 for ( const entry of profile.columnMap || []) {
100 for ( const alias of entry.aliases) {
101 const normalized = normalizeHeaderName (alias);
102 if (normalized && ! index. has (normalized)) {
103 index. set (normalized, { wixTarget: entry.wixTarget, alias });
104 }
105 }
106 }
107 return index;
108 }
109
110 // Anchors match by normalized EXACT equality and never through columnMap
111 // aliases. Aliases exist to be permissive for mapping; permissiveness is poison
112 // for identification — Shopify's variant.sku aliases include a bare "SKU",
113 // which every WooCommerce and Magento export also has.
114 function scoreProfile ( headerColumns , profile ) {
115 const header = normalizedSet (headerColumns);
116 const anchors = profile.anchors || {};
117 const required = anchors.required || [];
118 const strong = anchors.strong || [];
119 const negative = anchors.negative || [];
120
121 const matchedRequired = required. filter (( column ) => header. has ( normalizeHeaderName (column)));
122 const missingRequired = required. filter (( column ) => ! header. has ( normalizeHeaderName (column)));
123 const matchedStrong = strong. filter (( column ) => header. has ( normalizeHeaderName (column)));
124 const missingStrong = strong. filter (( column ) => ! header. has ( normalizeHeaderName (column)));
125 const negativeHits = negative. filter (( column ) => header. has ( normalizeHeaderName (column)));
126
127 const requiredRatio = required. length === 0 ? 0 : matchedRequired. length / required. length ;
128 const strongRatio = strong. length === 0 ? 0 : matchedStrong. length / strong. length ;
129
130 const core = BASE_REQUIRED + STRONG_WEIGHT * strongRatio;
131 let score = requiredRatio === 1 ? core : PARTIAL_FACTOR * requiredRatio * core;
132 if (negativeHits. length > 0 ) {
133 score *= NEGATIVE_PENALTY ;
134 }
135
136 // Evidence only — deliberately NOT part of the score. Coupling identification
137 // to columnMap completeness would mean adding a mapping alias could change
138 // which vendor a file is detected as.
139 const aliases = aliasIndex (profile);
140 const anchorNames = normalizedSet ([ ... required, ... strong]);
141 const knownColumns = (headerColumns || []). filter (( column ) => {
142 const normalized = normalizeHeaderName (column);
143 return anchorNames. has (normalized) || aliases. has (normalized);
144 });
145 const vocabularyCoverage = headerColumns && headerColumns. length > 0
146 ? knownColumns. length / headerColumns. length
147 : 0 ;
148
149 return {
150 vendor: profile.vendor,
151 score: Number (score. toFixed ( 4 )),
152 requiredRatio,
153 strongRatio,
154 matchedRequired,
155 missingRequired,
156 matchedStrong,
157 missingStrong,
158 negativeHits,
159 vocabularyCoverage: Number (vocabularyCoverage. toFixed ( 4 )),
160 };
161 }
162
163 function evidenceLine ( winner , runnerUp ) {
164 const parts = [
165 `required ${ winner . matchedRequired . length }/${ winner . matchedRequired . length + winner . missingRequired . length }` ,
166 `strong ${ winner . matchedStrong . length }/${ winner . matchedStrong . length + winner . missingStrong . length }` ,
167 `score ${ winner . score }` ,
168 ];
169 if (winner.negativeHits. length > 0 ) {
170 parts. push ( `negative anchors present: ${ winner . negativeHits . join ( ', ' ) }` );
171 }
172 parts. push (runnerUp ? `runner-up ${ runnerUp . vendor } ${ runnerUp . score }` : 'no runner-up' );
173 return parts. join ( '; ' );
174 }
175
176 function detectVendor ( headerColumns , profiles , { statedVendor = null } = {}) {
177 const scores = profiles
178 . map (( profile ) => scoreProfile (headerColumns, profile))
179 . sort (( a , b ) => b.score - a.score || a.vendor. localeCompare (b.vendor));
180
181 const winner = scores[ 0 ] || null ;
182 const runnerUp = scores[ 1 ] || null ;
183 const profileByVendor = new Map (profiles. map (( profile ) => [profile.vendor, profile]));
184
185 let reason = 'match' ;
186 let matched = Boolean (winner);
187 if ( ! winner || winner.score < MATCH_THRESHOLD ) {
188 matched = false ;
189 reason = 'below-threshold' ;
190 } else if (winner.matchedStrong. length < MIN_STRONG_HITS ) {
191 matched = false ;
192 reason = 'insufficient-strong-anchors' ;
193 } else if (runnerUp && winner.score - runnerUp.score < MARGIN ) {
194 matched = false ;
195 reason = 'ambiguous' ;
196 }
197
198 // A drifted vendor export should become a user question, not silence.
199 let nearMiss = null ;
200 if ( ! matched) {
201 const candidate = scores. find (( entry ) => entry.score >= NEAR_MISS_FLOOR
202 && entry.strongRatio >= NEAR_MISS_STRONG_RATIO
203 && entry.requiredRatio > 0
204 && entry.negativeHits. length === 0 );
205 if (candidate) {
206 nearMiss = {
207 vendor: candidate.vendor,
208 score: candidate.score,
209 missingRequired: candidate.missingRequired,
210 missingStrong: candidate.missingStrong,
211 };
212 }
213 }
214
215 const detectedVendor = matched ? winner.vendor : CUSTOM_VENDOR ;
216
217 // A user-stated vendor wins, but detection still runs so a disagreement is
218 // visible rather than silently overridden.
219 if (statedVendor) {
220 const stated = String (statedVendor). toLowerCase ();
221 if (stated !== CUSTOM_VENDOR && ! profileByVendor. has (stated)) {
222 throw new Error ( `Unknown vendor "${ statedVendor }". Known vendors: ${ [ ... profileByVendor . keys ()]. join ( ', ' ) }, ${ CUSTOM_VENDOR }.` );
223 }
224 const statedScore = scores. find (( entry ) => entry.vendor === stated) || null ;
225 return {
226 vendor: stated,
227 profile: profileByVendor. get (stated) || null ,
228 confidence: statedScore ? statedScore.score : 1 ,
229 source: 'user' ,
230 reason: 'user-stated' ,
231 evidence: `vendor stated by the user${ statedScore ? `; fingerprint would score it ${ statedScore . score }` : ''}` ,
232 scores,
233 runnerUp,
234 nearMiss,
235 conflict: detectedVendor !== stated ? { stated, detected: detectedVendor } : null ,
236 };
237 }
238
239 return {
240 vendor: detectedVendor,
241 profile: matched ? profileByVendor. get (winner.vendor) : null ,
242 confidence: winner ? winner.score : 0 ,
243 source: matched ? 'fingerprint' : 'fallback' ,
244 reason,
245 evidence: winner ? evidenceLine (winner, runnerUp) : 'no vendor profiles available' ,
246 scores,
247 runnerUp,
248 nearMiss,
249 conflict: null ,
250 };
251 }
252
253 // The two drift lists are scoped differently on purpose: `unmappedColumns` is
254 // about mapping coverage (anchors + columnMap), `missingExpectedColumns` is
255 // about format drift (anchors only — including every columnMap alias would
256 // make the list enormous and useless).
257 function diffProfile ( headerColumns , profile ) {
258 if ( ! profile) {
259 return { unmappedColumns: [ ... (headerColumns || [])], missingExpectedColumns: [] };
260 }
261 const aliases = aliasIndex (profile);
262 const anchors = profile.anchors || {};
263 const anchorNames = normalizedSet ([ ... (anchors.required || []), ... (anchors.strong || [])]);
264 const header = normalizedSet (headerColumns);
265
266 const unmappedColumns = (headerColumns || []). filter (( column ) => {
267 const normalized = normalizeHeaderName (column);
268 return normalized && ! anchorNames. has (normalized) && ! aliases. has (normalized);
269 });
270 const missingExpectedColumns = [ ... (anchors.required || []), ... (anchors.strong || [])]
271 . filter (( column ) => ! header. has ( normalizeHeaderName (column)));
272
273 return { unmappedColumns, missingExpectedColumns };
274 }
275
276 // Advisory pre-fill for rp-mapper. Every column this does not cover still flows
277 // through discovery → mapper → user unchanged.
278 function prefillColumnMap ( headerColumns , profile ) {
279 if ( ! profile) {
280 return [];
281 }
282 const aliases = aliasIndex (profile);
283 const hints = [];
284 for ( const column of headerColumns || []) {
285 const hit = aliases. get ( normalizeHeaderName (column));
286 if (hit) {
287 hints. push ({ column, wixTarget: hit.wixTarget, matchedAlias: hit.alias });
288 }
289 }
290 return hints;
291 }
292
293 module . exports = {
294 VENDORS_DIR,
295 BASE_REQUIRED,
296 STRONG_WEIGHT,
297 PARTIAL_FACTOR,
298 NEGATIVE_PENALTY,
299 MATCH_THRESHOLD,
300 MARGIN,
301 MIN_STRONG_HITS,
302 NEAR_MISS_FLOOR,
303 CUSTOM_VENDOR,
304 loadVendorProfiles,
305 validateProfile,
306 scoreProfile,
307 detectVendor,
308 diffProfile,
309 prefillColumnMap,
310 };