Setting the file. One moment. Bundled file LICENSE
scripts/normalize.mjs
JavaScript·283 lines·11 KB
if
(values.
every
(
v
=>
/
^
-
?
\d
+$
/
.
test
(v)))
return
{ type:
'integer'
};
13 if (values.every(v => /^-?\d+(\.\d+)?$/.test(v))) return { type: 'number' };
14 if (values.every(v => v === 'true' || v === 'false')) return { type: 'boolean' };
15 return { type: 'string' };
16}
17
18function statusSignature(rows) {
19 const ct = new Set(rows.map(r => (r.contentType || '').split(';')[0].trim().toLowerCase()).filter(Boolean));
20 const status = new Set(rows.map(r => (r.status != null ? Math.floor(r.status / 100) + 'xx' : 'none')));
21 return [...ct].sort().join(',') + '|' + [...status].sort().join(',');
22}
23
24// ---------------------------------------------------------------------------
25// Noise classification — tag endpoints that are infrastructure, not user-facing
26// ---------------------------------------------------------------------------
27const NOISE_PATH_PATTERNS = [
28 // Tracking / analytics / telemetry
29 /\/track(ing)?[\/\b]/i, /\/pixel/i, /\/beacon/i, /\/log[\/\b]/i,
30 /\/impression/i, /\/pageview/i, /\/click[\/\b]/i,
31 /\/session[-_]?start/i, /\/batch\/(impression|list)/i,
32 /\/dag\/v\d+\//i,
33 /\/trackgoal/i, /\/profileview/i, /\/sessionstart/i,
34 /\/dinerTrust/i, /\/trackDiner/i,
35 /\/profile-view$/i, /\/track\/search$/i,
36 /\/mix$/i,
37 // Cookie / consent / privacy
38 /\/cookie[-_]?consent/i, /\/consent\//i, /\/onetrust/i,
39 // Experimentation
40 /\/bucket[-_]?experiment/i, /\/experiment[\/\b]/i, /\/feature[-_]?flag/i,
41 // Bot defense / fingerprinting
42 /\/akam\//i, /\/akamai\//i, /\/human$/i,
43 // Session plumbing (not user-facing API)
44 /\/session$/i, /\/authenticate\/start$/i,
45];
46
47const NOISE_BODY_SIGNALS = [
48 /^sensor_data$/, // Akamai bot fingerprint
49 /^body$/, // Obfuscated payloads (Akamai, etc.)
50];
51
52function classifyEndpoint(endpoint) {
53 const p = endpoint.path;
54 const m = endpoint.method;
55
56 // HTML page renders are not API endpoints
57 const htmlRows = endpoint.sampleRows.filter(r =>
58 (r.contentType || '').includes('text/html'));
59 if (htmlRows.length === endpoint.sampleRows.length && m === 'GET') return 'page';
60
61 // Path-based noise detection
62 if (NOISE_PATH_PATTERNS.some(re => re.test(p))) return 'noise';
63
64 // Obfuscated paths (random-looking segments with mixed case, no real structure)
65 const segs = p.split('/').filter(Boolean);
66 const obfuscated = segs.filter(s =>
67 /[A-Za-z0-9_-]{8,}/.test(s) &&
68 !/^(v\d+|api|dapi|graphql|rest|fe|gql)$/i.test(s) &&
69 /[A-Z]/.test(s) && /[a-z]/.test(s));
70 if (obfuscated.length >= 2) return 'noise';
71
72 // Body-based: if every sample's request body only has noise-signal keys
73 if (endpoint.sampleRows.length > 0) {
74 const allNoise = endpoint.sampleRows.every(r => {
75 if (!r.reqBody || typeof r.reqBody !== 'object') return false;
76 const keys = Object.keys(r.reqBody);
77 return keys.length > 0 && keys.every(k => NOISE_BODY_SIGNALS.some(re => re.test(k)));
78 });
79 if (allNoise) return 'noise';
80 }
81
82 return 'api';
83}
84
85// ---------------------------------------------------------------------------
86// GraphQL / multiplexed endpoint decomposition
87// ---------------------------------------------------------------------------
88function detectDiscriminator(rows) {
89 // Check if these rows share a URL path but have a body field that acts as
90 // a discriminator (operationName for GraphQL, method for JSON-RPC, etc.)
91 const candidates = ['operationName', 'method', 'action', 'type', 'command'];
92 for (const field of candidates) {
93 const values = new Set();
94 let matchCount = 0;
95 for (const r of rows) {
96 if (r.reqBody && typeof r.reqBody === 'object' && typeof r.reqBody[field] === 'string') {
97 values.add(r.reqBody[field]);
98 matchCount++;
99 }
100 }
101 if (matchCount >= rows.length * 0.8 && values.size >= 2) {
102 return { field, values: [...values] };
103 }
104 }
105
106 // Also check query params (OpenTable uses ?opname= for GraphQL)
107 for (const field of ['opname', 'operationName', 'op', 'action']) {
108 const values = new Set();
109 let matchCount = 0;
110 for (const r of rows) {
111 if (r.query && typeof r.query[field] === 'string') {
112 values.add(r.query[field]);
113 matchCount++;
114 }
115 }
116 if (matchCount >= rows.length * 0.8 && values.size >= 2) {
117 return { field, values: [...values], source: 'query' };
118 }
119 }
120
121 return null;
122}
123
124function decomposeMultiplexed(endpoint) {
125 const disc = detectDiscriminator(endpoint.sampleRows);
126 if (!disc) return [endpoint];
127
128 const byOp = new Map();
129 for (const row of endpoint.sampleRows) {
130 let opName;
131 if (disc.source === 'query') {
132 opName = row.query?.[disc.field] || '__unknown__';
133 } else {
134 opName = (row.reqBody && typeof row.reqBody === 'object')
135 ? row.reqBody[disc.field] || '__unknown__'
136 : '__unknown__';
137 }
138 if (!byOp.has(opName)) byOp.set(opName, []);
139 byOp.get(opName).push(row);
140 }
141
142 const sub = [];
143 for (const [opName, rows] of byOp) {
144 // Build a virtual endpoint per operation
145 const virtualPath = `${endpoint.path} [${opName}]`;
146 sub.push({
147 ...endpoint,
148 endpointKey: `${endpoint.method} ${endpoint.origin}${virtualPath}`,
149 path: virtualPath,
150 operationName: opName,
151 discriminatorField: disc.field,
152 parentPath: endpoint.path,
153 sampleRows: rows,
154 sampleCount: rows.length,
155 });
156 }
157 return sub;
158}
159
160export function normalize(outDir) {
161 const filtered = readJsonl(intermediatePath(outDir, 'filtered.jsonl'));
162
163 // Pass 1: bucket by (origin, method, single-pass template).
164 const buckets = new Map();
165 for (const row of filtered) {
166 const t = templatize(row.path);
167 const key = `${row.method} ${row.origin}${t.template}`;
168 let b = buckets.get(key);
169 if (!b) { b = { origin: row.origin, method: row.method, template: t.template, params: t.params, rows: [], rawPaths: new Set() }; buckets.set(key, b); }
170 b.rows.push(row);
171 b.rawPaths.add(row.path);
172 }
173
174 // Pass 2: re-templatize each bucket using its raw-path set so slugs can be
175 // detected.
176 const refined = new Map();
177 for (const [, b] of buckets) {
178 const rawPaths = [...b.rawPaths];
179 const t = rawPaths.length > 1 ? templatizeWithSlugs(rawPaths) : { template: b.template, params: b.params };
180 const key = `${b.method} ${b.origin}${t.template}`;
181 let r = refined.get(key);
182 if (!r) {
183 r = { origin: b.origin, method: b.method, template: t.template, params: t.params, rows: [], rawPaths: new Set(), originalKeys: [] };
184 refined.set(key, r);
185 }
186 r.rows.push(...b.rows);
187 for (const p of b.rawPaths) r.rawPaths.add(p);
188 r.originalKeys.push({ template: b.template, sig: statusSignature(b.rows) });
189 }
190
191 // Build endpoint records, classify, and decompose.
192 const preEndpoints = [];
193 for (const [, e] of refined) {
194 const flags = [];
195 const sigs = new Set(e.originalKeys.map(k => k.sig));
196 if (sigs.size > 1) flags.push('divergent-response-shape');
197 if (e.rows.length === 1) flags.push('single-sample');
198 const statuses = new Set(e.rows.map(r => r.status).filter(s => s != null));
199 if (statuses.size === 1) flags.push('single-status');
200 const cts = new Set(e.rows.map(r => (r.contentType || '').split(';')[0].trim()).filter(Boolean));
201 if (cts.size > 1) flags.push('mixed-content-types');
202 const withBody = e.rows.filter(r => r.reqBody != null).length;
203 if (withBody > 0 && withBody < e.rows.length) flags.push('request-body-only-on-some-samples');
204
205 const qSamples = new Map();
206 for (const r of e.rows) {
207 for (const k of Object.keys(r.query || {})) {
208 if (!qSamples.has(k)) qSamples.set(k, []);
209 qSamples.get(k).push(r.query[k]);
210 }
211 }
212 const queryParams = [];
213 for (const [name, values] of qSamples.entries()) {
214 const present = e.rows.filter(r => name in (r.query || {})).length;
215 queryParams.push({
216 name,
217 in: 'query',
218 required: present === e.rows.length,
219 schema: inferQueryType(values),
220 });
221 }
222
223 preEndpoints.push({
224 endpointKey: `${e.method} ${e.origin}${e.template}`,
225 origin: e.origin,
226 method: e.method,
227 path: e.template,
228 pathParams: e.params.map(p => ({ name: p.name, in: 'path', required: true, schema: p.schema })),
229 queryParams,
230 statusCodes: [...new Set(e.rows.map(r => r.status).filter(s => s != null))].sort((a, b) => a - b),
231 sampleRows: e.rows,
232 sampleCount: e.rows.length,
233 rawPaths: [...e.rawPaths],
234 normalizationFlags: flags,
235 });
236 }
237
238 // Pass 3: classify and decompose
239 const endpoints = [];
240 let noiseCount = 0, pageCount = 0, decomposedCount = 0;
241 for (const ep of preEndpoints) {
242 const category = classifyEndpoint(ep);
243 if (category === 'noise') { noiseCount++; continue; }
244 if (category === 'page') { pageCount++; continue; }
245
246 // Try to decompose multiplexed endpoints
247 const decomposed = decomposeMultiplexed(ep);
248 if (decomposed.length > 1) {
249 decomposedCount += decomposed.length;
250 for (const sub of decomposed) {
251 sub.normalizationFlags = [...(sub.normalizationFlags || [])];
252 const subStatuses = new Set(sub.sampleRows.map(r => r.status).filter(s => s != null));
253 sub.statusCodes = [...subStatuses].sort((a, b) => a - b);
254 if (sub.sampleRows.length === 1) {
255 if (!sub.normalizationFlags.includes('single-sample')) sub.normalizationFlags.push('single-sample');
256 }
257 if (subStatuses.size === 1) {
258 if (!sub.normalizationFlags.includes('single-status')) sub.normalizationFlags.push('single-status');
259 }
260 endpoints.push(sub);
261 }
262 } else {
263 endpoints.push(ep);
264 }
265 }
266
267 // Drop the heavy in-memory rows from the persisted form; infer.mjs needs
268 // them so we keep a parallel sidecar file.
269 const persisted = endpoints.map(({ sampleRows, ...rest }) => rest);
270 writeJsonl(intermediatePath(outDir, 'endpoints.jsonl'), persisted);
271
272 const sidecar = endpoints.map(e => ({ endpointKey: e.endpointKey, samples: e.sampleRows }));
273 writeJsonl(intermediatePath(outDir, 'endpoint-samples.jsonl'), sidecar);
274
275 return { endpoints: endpoints.length, noise: noiseCount, pages: pageCount, decomposed: decomposedCount };
276}
277
278if (import.meta.url === `file://${process.argv[1]}`) {
279 const out = process.argv[2];
280 if (!out) { console.error('usage: normalize.mjs <out-dir>'); process.exit(2); }
281 const stats = normalize(out);
282 console.log(`normalize: ${stats.endpoints} endpoints (${stats.noise} noise, ${stats.pages} pages dropped, ${stats.decomposed} decomposed)`);
283}