Setting the file. One moment.
Link Census · Wix Replatform · wix/skills · Skills Docs
Back to the top of the page lib/ link-census.js
JavaScript · 194 lines · 11 KB
);
7
8 const NAV_KEYS = new Set ([ 'href' , 'url' , 'link' , 'destination' , 'targetUrl' ]);
9 const RESOURCE_KEYS = new Set ([ 'src' , 'srcset' , 'image' , 'media' , 'download' , 'backgroundImage' ]);
10 const IGNORED_SEGMENTS = new Set ([ 'config' , '.git' , 'node_modules' , 'secrets' ]);
11
12 function hash ( value ) { return createHash ( 'sha256' ). update (value). digest ( 'hex' ); }
13
14 function decodeHtmlEntities ( value ) {
15 return value. replace ( /&(#(?:x [0-9a-f] +| [0-9] + ) | amp | quot | apos | lt | gt);/ gi , ( _whole , entity ) => {
16 if (entity[ 0 ] === '#' ) {
17 const number = entity[ 1 ]. toLowerCase () === 'x' ? parseInt (entity. slice ( 2 ), 16 ) : parseInt (entity. slice ( 1 ), 10 );
18 return number > 0 && number <= 0x10ffff ? String. fromCodePoint (number) : _whole;
19 }
20 return { amp: '&' , quot: '"' , apos: "'" , lt: '<' , gt: '>' }[entity. toLowerCase ()];
21 });
22 }
23
24 // Tokenizes tags and quoted attributes. It does not treat arbitrary angle-bracket text as a
25 // successful parse: malformed markup produces an explicit unsupported disposition.
26 function htmlAttributes ( html ) {
27 const found = [];
28 let index = 0 ;
29 while (index < html. length ) {
30 const open = html. indexOf ( '<' , index);
31 if (open < 0 ) break ;
32 if (html. startsWith ( '<!--' , open)) {
33 const close = html. indexOf ( '-->' , open + 4 );
34 if (close < 0 ) throw new Error ( 'unterminated HTML comment' );
35 const comment = html. slice (open + 4 , close). trim ();
36 if (comment. startsWith ( 'wp:' )) {
37 const jsonStart = comment. indexOf ( '{' );
38 const jsonEnd = comment. lastIndexOf ( '}' );
39 if (jsonStart >= 0 && jsonEnd > jsonStart) {
40 try {
41 for ( const link of extractJson ( JSON . parse (comment. slice (jsonStart, jsonEnd + 1 )))) found. push ({ ... link, location: `block:${ open }:${ link . location }` });
42 } catch { throw new Error ( 'unsupported WordPress block attributes' ); }
43 }
44 }
45 index = close + 3 ; continue ;
46 }
47 let cursor = open + 1 ;
48 if ( ! / [A-Za-z/!?] / . test (html[cursor] || '' )) { index = cursor; continue ; }
49 let quote = null ;
50 for (; cursor < html. length ; cursor += 1 ) {
51 const char = html[cursor];
52 if (quote) { if (char === quote) quote = null ; }
53 else if (char === '"' || char === "'" ) quote = char;
54 else if (char === '>' ) break ;
55 }
56 if (cursor >= html. length ) throw new Error ( 'unterminated HTML tag' );
57 const tag = html. slice (open + 1 , cursor);
58 const attr = /(?: ^| \s )(href | action | src | srcset) \s * = \s * (?:"( [ ^ "] * )" | '( [ ^ '] * )' | ( [ ^ \s>] + ))/ gi ;
59 let match;
60 while ((match = attr. exec (tag))) found. push ({ key: match[ 1 ]. toLowerCase (), value: decodeHtmlEntities (match[ 2 ] ?? match[ 3 ] ?? match[ 4 ]), location: `char:${ open + 1 + match . index + match [ 0 ]. indexOf ( match [ 1 ]) }` });
61 index = cursor + 1 ;
62 }
63 return found;
64 }
65
66 function countMetadataHrefs ( value ) {
67 if (Array. isArray (value)) return value. reduce (( count , item ) => count + countMetadataHrefs (item), 0 );
68 if ( ! value || typeof value !== 'object' ) return 0 ;
69 return Object. entries (value). reduce (( count , [ key , child ]) => count + (key === 'href' && typeof child === 'string' ? 1 : countMetadataHrefs (child)), 0 );
70 }
71
72 function extractJson ( value , location = '$' , output = [], contentField = null , stats = null ) {
73 if (Array. isArray (value)) { value. forEach (( item , index ) => extractJson (item, `${ location }[${ index }]` , output, contentField, stats)); return output; }
74 if ( ! value || typeof value !== 'object' ) return output;
75 for ( const [ key , child ] of Object. entries (value)) {
76 if (key === '_links' || key === '_embedded' ) {
77 if (stats) stats.metadataHrefsSkipped = (stats.metadataHrefsSkipped || 0 ) + countMetadataHrefs (child);
78 continue ;
79 }
80 if (key === 'yoast_head' || key === 'yoast_head_json' ) {
81 if (stats && key === 'yoast_head' && typeof child === 'string' ) {
82 stats.metadataHrefsSkipped = (stats.metadataHrefsSkipped || 0 ) + (child. match ( /(?: ^| \s )href \s * =/ gi ) || []). length ;
83 }
84 continue ;
85 }
86 const next = `${ location }.${ key }` ;
87 const owningField = contentField || ([ 'records' , 'items' , 'data' , 'entities' ]. includes (key) ? null : key);
88 if ( typeof child === 'string' ) {
89 const resourceContext = /(?: ^| \. )(?:image | media | attachment | thumbnail | download | logo | icon | background)(?: \. | \[ |$ )/ i . test (location);
90 const navigationContext = /(?: ^| \. )(?:content | body | description | button | menu | recommendation | link | anchor | action | richContent)(?: \. | \[ |$ )/ i . test (location);
91 if ( RESOURCE_KEYS . has (key) || resourceContext && key === 'url' ) output. push ({ key, value: child, location: next, contentField: owningField, resource: true });
92 else if ( NAV_KEYS . has (key) && (key === 'href' || key === 'targetUrl' || key === 'destination' || navigationContext)) output. push ({ key, value: child, location: next, contentField: owningField });
93 else if ( /< [a-z][ ^ >] * >/ i . test (child)) for ( const attr of htmlAttributes (child)) output. push ({ ... attr, contentField: owningField, location: `${ next }:${ attr . location }` });
94 } else extractJson (child, next, output, owningField, stats);
95 }
96 return output;
97 }
98
99 function extractFile ( file , raw , stats = null ) {
100 const ext = path. extname (file). toLowerCase ();
101 if (ext === '.json' ) return extractJson ( JSON . parse (raw), '$' , [], null , stats);
102 if (ext === '.html' || ext === '.htm' ) return htmlAttributes (raw);
103 // CSS URLs are resources, not automatically navigational links.
104 if (ext === '.css' ) return [];
105 throw new Error ( 'unsupported-format' );
106 }
107
108 function containingRecord ( file , location , containingPages ) {
109 const config = containingPages[file];
110 if ( typeof config === 'string' ) return { url: config, recordId: null };
111 if ( ! config || typeof config !== 'object' ) return null ;
112 const prefix = Object. keys (config). filter (( key ) => location. startsWith (key)). sort (( a , b ) => b. length - a. length )[ 0 ];
113 if ( ! prefix) return null ;
114 const row = config[prefix];
115 return typeof row === 'string' ? { url: row, recordId: null } : row;
116 }
117
118 async function walk ( root , relative = '' ) {
119 const entries = await fs. readdir (path. join (root, relative), { withFileTypes: true });
120 const files = [];
121 for ( const entry of entries) {
122 if ( IGNORED_SEGMENTS . has (entry.name) || entry.name. startsWith ( '.' )) continue ;
123 const next = path. join (relative, entry.name);
124 if (entry. isDirectory ()) files. push ( ...await walk (root, next));
125 else if (entry. isFile ()) files. push (next);
126 }
127 return files;
128 }
129
130 async function census ({ capturedRoots , expectedFiles = [], sourceHosts , destinationHosts , containingPages = {}, previous = null }) {
131 if ( ! Array. isArray (capturedRoots) || ! capturedRoots. length ) throw new Error ( 'capturedRoots required' );
132 const started = Date. now ();
133 const manifest = [];
134 const occurrences = [];
135 for ( const root of capturedRoots) {
136 const files = await walk (root);
137 for ( const relative of files) {
138 const file = path. join (root, relative);
139 let raw;
140 try { raw = await fs. readFile (file); }
141 catch (error) { manifest. push ({ file, disposition: 'unreadable' , reason: error.code || 'read-error' }); continue ; }
142 const digest = hash (raw);
143 const base = { file, bytes: raw. length , sha256: digest };
144 const old = previous && previous.censusVersion === 4 && previous.manifest && previous.manifest. find (( row ) => row.file === file && row.sha256 === digest);
145 if (old && previous.occurrences) {
146 manifest. push ({ ... base, disposition: old.disposition, reason: old.reason || null , metadataHrefsSkipped: old.metadataHrefsSkipped || 0 });
147 occurrences. push ( ... previous.occurrences. filter (( row ) => row.sourceFile === file));
148 continue ;
149 }
150 let links;
151 const scanStats = {};
152 try { links = extractFile (file, raw. toString ( 'utf8' ), scanStats); }
153 catch (error) { manifest. push ({ ... base, disposition: 'unsupported' , reason: error.message }); continue ; }
154 const possibleUnparsedLink = ! links. length && /https ? : \/\/ | href \s * = | location \. href/ i . test (raw. toString ( 'utf8' )) && ! scanStats.metadataHrefsSkipped;
155 manifest. push ({ ... base, disposition: links. length ? 'scanned-links' : possibleUnparsedLink ? 'unsupported-construct' : 'scanned-no-links' , metadataHrefsSkipped: scanStats.metadataHrefsSkipped || 0 });
156 for ( const link of links) {
157 if (link.resource || RESOURCE_KEYS . has (link.key)) continue ;
158 const containing = containingRecord (file, link.location, containingPages);
159 const containingUrl = containing && containing.url;
160 if ( ! containingUrl) {
161 // An absolute off-site URL needs no containing page to classify it.
162 if ( / ^ https ? : \/\/ / i . test (link.value) || / ^ \/\/ / . test (link.value)) {
163 const absolute = classifyNavigation (link.value, 'https://link-audit.invalid/' , { sourceHosts, destinationHosts });
164 if (absolute.kind === 'external' ) continue ;
165 }
166 occurrences. push ({ id: occurrenceId ({ sourceFile: file, field: link.key, location: link.location, original: link.value }), sourceFile: file, field: link.key, location: link.location, original: link.value, classification: 'missing-containing-page' });
167 continue ;
168 }
169 const classification = classifyNavigation (link.value, containingUrl, { sourceHosts, destinationHosts });
170 if (classification.kind !== 'internal' && classification.kind !== 'stateful' ) continue ;
171 occurrences. push ({
172 id: occurrenceId ({ sourceFile: file, field: link.key, location: link.location, original: link.value }),
173 sourceFile: file, sourceRecordId: containing.recordId || null , targetRecordId: containing.targetRecordId || null ,
174 recordType: containing.recordType || null , targetContentRevision: containing.targetContentRevision || null ,
175 field: containing.contentField || link.contentField || link.key, location: link.location, containingUrl,
176 original: link.value, classification: classification.kind === 'stateful' ? 'stateful-navigation' : 'internal-navigation' ,
177 originalPath: classification.path, ... (classification.fragmentOnly ? { fragmentOnly: true } : {}),
178 });
179 }
180 }
181 }
182 const scannedFiles = new Set (manifest. map (( row ) => row.file));
183 for ( const file of expectedFiles) if ( ! scannedFiles. has (file)) manifest. push ({ file, disposition: 'missing' , reason: 'capture-file-absent' });
184 return { manifest, occurrences, counts: {
185 files: manifest. length , bytes: manifest. reduce (( sum , row ) => sum + (row.bytes || 0 ), 0 ),
186 internalOccurrences: occurrences. filter (( row ) => [ 'internal-navigation' , 'stateful-navigation' ]. includes (row.classification)). length ,
187 distinctInternalDestinations: new Set (occurrences. filter (( row ) => row.originalPath). map (( row ) => row.originalPath)).size,
188 unsupportedFiles: manifest. filter (( row ) => [ 'unsupported' , 'unsupported-construct' , 'unreadable' , 'missing' ]. includes (row.disposition)). length ,
189 metadataHrefsSkipped: manifest. reduce (( sum , row ) => sum + (row.metadataHrefsSkipped || 0 ), 0 ),
190 scanMs: Date. now () - started,
191 } };
192 }
193
194 module . exports = { decodeHtmlEntities, htmlAttributes, extractJson, extractFile, census };