Setting the file. One moment.
Extract Page · Wix Headless Replatform · wix/skills · Skills Docs
Back to the top of the page 28.10
Workflow
— line 2382
This file
Number 28.21
Position 21 of 89
Type JavaScript
Size 118 KB
Lines 2,461 scripts/ extract-page.mjs
JavaScript · 2,461 lines · 118 KB
}
from
"./lib/common.mjs"
;
16 import { loadPlaywrightFromContext, resolveBrowserToolingContext } from "./lib/browser-tooling.mjs" ;
17 import { extractSeo, extractSectionsFromHtml, inferTokensFromHtml } from "./lib/html-extract.mjs" ;
18
19 async function main () {
20 const args = parseArgs ();
21 const url = normalizeUrl (args._[ 0 ] || args.url). toString ();
22 const outputDir = resolveOutputDir (url, args.out);
23 const page = await extractPage (url, { outputDir, screenshots: args.screenshots !== "false" });
24 const pageDir = path. join ( docsDir (outputDir), "pages" );
25 await writeJson (path. join (pageDir, `${ slugForUrl ( url ) }.json` ), page);
26 if (args.json) process.stdout. write ( `${ JSON . stringify ( page , null , 2 ) } \n ` );
27 }
28
29 export async function extractPage ( url , options = {}) {
30 try {
31 return await extractWithPlaywright (url, options);
32 } catch (error) {
33 throw new Error (
34 `Browser extraction failed for ${ url }. The site clone skill now requires Playwright-based extraction instead of HTML fallback because shared chrome, first-viewport evidence, and repeater detection depend on a real browser. Details: ${ error . message }` ,
35 );
36 }
37 }
38
39 async function extractWithPlaywright ( url , { outputDir , screenshots = true , browserTooling , screenshotDir : suppliedScreenshotDir , screenshotPrefix = "source" } = {}) {
40 const toolingContext = browserTooling || await resolveBrowserToolingContext ({ startDir: process. cwd () });
41 const playwright = await loadPlaywrightFromContext (toolingContext);
42 const browser = await playwright.chromium. launch ({ headless: true });
43 const page = await browser. newPage ();
44 const screenshotMap = {};
45 try {
46 const htmlByViewport = {};
47 const navigationProfiles = [];
48 const responsiveTextGeometry = {};
49 for ( const viewport of DEFAULT_VIEWPORTS ) {
50 await page. setViewportSize ({ width: viewport.width, height: viewport.height });
51 const navigation = await navigateWithAdaptiveProfile (page, url);
52 navigationProfiles. push ({ viewport: viewport.name, ... navigation });
53 responsiveTextGeometry[viewport.name] = await captureImportantTextGeometry (page);
54 if (screenshots && outputDir) {
55 const screenshotDir = suppliedScreenshotDir || path. join ( docsDir (outputDir), "screenshots" );
56 await ensureDir (screenshotDir);
57 const screenshotPath = path. join (screenshotDir, `${ screenshotPrefix }-${ slugForUrl ( url ) }-${ viewport . name }.png` );
58 await page. screenshot ({ path: screenshotPath, fullPage: true });
59 screenshotMap[viewport.name] = screenshotPath;
60 }
61 htmlByViewport[viewport.name] = await page. content ();
62 }
63 const html = htmlByViewport.desktop || Object. values (htmlByViewport)[ 0 ];
64 const seo = extractSeo (html, url);
65 const desktopViewport = DEFAULT_VIEWPORTS . find (( viewport ) => viewport.name === "desktop" ) || DEFAULT_VIEWPORTS [ 0 ];
66 await page. setViewportSize ({ width: desktopViewport.width, height: desktopViewport.height });
67 const desktopNavigation = await navigateWithAdaptiveProfile (page, url);
68 const repeaterPlan = await detectRepeaterSignals (page);
69 const browserData = await page. evaluate ( async ( repeaterPlan ) => {
70 const cleanText = ( value ) => String (value || "" ). replace ( / \s + / g , " " ). trim ();
71 const nonContentSelector = "script,style,noscript,template" ;
72 const semanticText = ( node ) => {
73 if ( ! node || node.nodeType !== Node. ELEMENT_NODE || node. matches (nonContentSelector)) return "" ;
74 const clone = node. cloneNode ( true );
75 clone. querySelectorAll (nonContentSelector). forEach (( child ) => child. remove ());
76 return cleanText (clone.textContent);
77 };
78 const renderedText = ( node ) => {
79 if ( ! node || node.nodeType !== Node. ELEMENT_NODE || node. matches (nonContentSelector)) return "" ;
80 const excludedSelector = `${ nonContentSelector },[hidden],[inert],[aria-hidden='true' i]` ;
81 const walker = document. createTreeWalker (node, NodeFilter. SHOW_TEXT );
82 const values = [];
83 for ( let textNode = walker. nextNode (); textNode; textNode = walker. nextNode ()) {
84 if ( ! cleanText (textNode.nodeValue)) continue ;
85 const parent = textNode.parentElement;
86 if ( ! parent || parent. closest (excludedSelector)) continue ;
87 let current = parent;
88 let excluded = false ;
89 while (current) {
90 const styles = getComputedStyle (current);
91 if (styles.display === "none" || styles.visibility === "hidden" || Number (styles.opacity || 1 ) <= 0 ) {
92 excluded = true ;
93 break ;
94 }
95 if (current === node) break ;
96 current = current.parentElement;
97 }
98 if ( ! excluded) values. push (textNode.nodeValue);
99 }
100 return cleanText (values. join ( " " ));
101 };
102 const rectFor = ( node ) => {
103 const rect = node. getBoundingClientRect ();
104 return { x: rect.x, y: rect.y, width: rect.width, height: rect.height, top: rect.top, bottom: rect.bottom };
105 };
106 const isRawVisible = ( node ) => {
107 const rect = node. getBoundingClientRect ();
108 const styles = getComputedStyle (node);
109 return rect.width > 0 && rect.height > 0 && styles.display !== "none" && styles.visibility !== "hidden" && Number (styles.opacity || 1 ) > 0 ;
110 };
111 const visibleCodeTexts = ( node ) => Array. from (node?. querySelectorAll ?.( "pre,code" ) || [])
112 . filter (( candidate ) => isRawVisible (candidate))
113 . map (( candidate ) => renderedText (candidate))
114 . filter (Boolean)
115 . filter (( value , index , values ) => values. indexOf (value) === index)
116 . slice ( 0 , 12 );
117 const consentSelectors = [
118 "#onetrust-banner-sdk" ,
119 "#onetrust-consent-sdk" ,
120 "[class*='onetrust' i]" ,
121 "[id*='cookie-banner' i]" ,
122 "[class*='cookie-banner' i]" ,
123 "[id*='consent-banner' i]" ,
124 "[class*='consent-banner' i]" ,
125 "[role='dialog']" ,
126 ]. join ( "," );
127 const consentCandidates = Array. from (document. querySelectorAll (consentSelectors)). filter (isRawVisible). filter (( node ) => {
128 const text = renderedText (node). toLowerCase ();
129 const identity = `${ node . id || ""} ${ node . className || ""} ${ node . getAttribute ( "aria-label" ) || ""}` . toLowerCase ();
130 const controls = Array. from (node. querySelectorAll ( "button,a,[role='button']" )). map (( control ) => renderedText (control)). join ( " " ). toLowerCase ();
131 const style = getComputedStyle (node);
132 const rect = node. getBoundingClientRect ();
133 const consentSignal = /cookie | consent | privacy preferences | tracking preferences/ . test ( `${ identity } ${ text }` );
134 const actionSignal = /accept | reject | decline | customi [sz] e | preferences | allow all/ . test (controls);
135 const overlaySignal = style.position === "fixed" || style.position === "sticky" || node. getAttribute ( "role" ) === "dialog" || rect.width >= window.innerWidth * 0.55 ;
136 return /onetrust | cookiebot | quantcast/ . test (identity) || consentSignal && actionSignal && overlaySignal;
137 });
138 const consentRoots = consentCandidates. filter (( node , index , all ) => ! all. some (( candidate , candidateIndex ) => candidateIndex !== index && candidate. contains (node)));
139 const ignoredSurfaces = consentRoots. map (( node , index ) => {
140 node. setAttribute ( "data-rp-ignored-surface" , "consent-management" );
141 const rect = node. getBoundingClientRect ();
142 const identity = `${ node . id || ""} ${ node . className || ""}` . toLowerCase ();
143 return {
144 id: `ignored-surface-${ String ( index + 1 ). padStart ( 3 , "0" ) }` ,
145 kind: "consent-management" ,
146 provider: /onetrust | ot-sdk/ . test (identity) ? "onetrust" : /cookiebot/ . test (identity) ? "cookiebot" : /quantcast/ . test (identity) ? "quantcast" : "unknown" ,
147 creationPolicy: "ignore" ,
148 reason: "Destination consent must be implemented as functional infrastructure, not cloned page content." ,
149 textFingerprint: renderedText (node). slice ( 0 , 240 ),
150 rect: { top: Math. round (rect.top), left: Math. round (rect.left), width: Math. round (rect.width), height: Math. round (rect.height) },
151 domRef: {
152 tag: node.tagName. toLowerCase (),
153 id: node.id || "" ,
154 classTokens: String (node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 10 ),
155 },
156 };
157 });
158 const isVisible = ( node ) => {
159 if ( ! (node instanceof Element ) || node. closest ( "[data-rp-ignored-surface]" )) return false ;
160 return isRawVisible (node);
161 };
162 const headerSelectors = "header,[role='banner'],#header,.header,.site-header,.main-header,.masthead,.sticky-header" ;
163 const navSelectors = "nav,[role='navigation'],.menu,.nav,.navbar,.navigation" ;
164 const visualAssetSelectors = "img,svg" ;
165
166 function pickHeader () {
167 const candidates = Array. from (document. querySelectorAll (headerSelectors))
168 . filter (isVisible)
169 . map (( node ) => ({ node, rect: node. getBoundingClientRect (), styles: getComputedStyle (node) }))
170 . filter (({ rect }) => rect.bottom > - 8 && rect.top < Math. max (window.innerHeight * 0.55 , 260 ));
171 candidates. sort (( a , b ) => {
172 const aTop = Math. abs (a.rect.top);
173 const bTop = Math. abs (b.rect.top);
174 if (aTop !== bTop) return aTop - bTop;
175 return b.rect.height - a.rect.height;
176 });
177 return candidates[ 0 ] || null ;
178 }
179
180 function visualAssetRecord ( node , context = "page" ) {
181 if ( ! (node instanceof Element )) return null ;
182 const rect = node. getBoundingClientRect ();
183 const styles = getComputedStyle (node);
184 const tag = node.tagName. toLowerCase ();
185 const use = tag === "svg" ? node. querySelector ( "use" ) : null ;
186 const useHref = use?. getAttribute ( "href" ) || use?. getAttribute ( "xlink:href" ) || "" ;
187 const sourceUrl = tag === "img" ? (node.currentSrc || node.src || "" ) : useHref;
188 const accessibleName = cleanText (node. getAttribute ( "aria-label" ) || node. getAttribute ( "title" ) || node. getAttribute ( "alt" ) || node. closest ( "a,button" )?. getAttribute ( "aria-label" ) || "" );
189 const identity = `${ node . id || ""} ${ node . className ?. baseVal || node . className || ""} ${ sourceUrl } ${ accessibleName }` . toLowerCase ();
190 if ( /cookielaw | onetrust | ot-sdk | cookiebot | quantcast/ . test (identity)) return null ;
191 const owner = node. closest ( "header,footer,button,a,[role='button']" );
192 const ownerIdentity = `${ owner ?. className || ""} ${ owner ?. id || ""} ${ owner ?. getAttribute ?.( "aria-label" ) || ""} ${ owner ? semanticText ( owner ) : ""}` . toLowerCase ();
193 const homeLink = node. closest ( "a" )?.href && new URL (node. closest ( "a" ).href, location.href).pathname. replace ( / \/ +$ / , "" ) === "" ;
194 const social = /linkedin | instagram | facebook | twitter | youtube | tiktok | x \. com | social/ . test ( `${ identity } ${ ownerIdentity }` );
195 const looksLikeLogo = /logo | wordmark | brandmark | brand-mark/ . test (identity)
196 || ( ! social && homeLink && (rect.width >= 72 || Number (node. getAttribute ( "width" )) >= 72 ))
197 || ( ! social && rect.width >= 120 && rect.width >= rect.height * 1.8 && /header | footer/ . test (context));
198 const kind = looksLikeLogo ? "logo" : "icon" ;
199 const fragmentId = useHref. startsWith ( "#" ) ? useHref. slice ( 1 ) : "" ;
200 const symbol = fragmentId ? document. getElementById (fragmentId) : null ;
201 const variantText = `${ identity } ${ context }` ;
202 const variant = /footer/ . test (variantText) ? "footer"
203 : /solid/ . test (variantText) ? "solid"
204 : /dark | black/ . test (variantText) ? "dark"
205 : /light | white/ . test (variantText) ? "light"
206 : /mobile | menu | drawer/ . test (variantText) ? "menu"
207 : "default" ;
208 return {
209 kind,
210 context,
211 variant,
212 sourceType: tag === "img" ? ( / \. svg(?: $| [?#] )/ i . test (sourceUrl) ? "external-svg" : "image" ) : useHref ? "svg-sprite-use" : "inline-svg" ,
213 sourceUrl,
214 useHref,
215 accessibleName,
216 visible: isRawVisible (node),
217 renderedSize: { width: Math. round (rect.width * 100 ) / 100 , height: Math. round (rect.height * 100 ) / 100 },
218 intrinsicSize: {
219 width: node. getAttribute ( "width" ) || "" ,
220 height: node. getAttribute ( "height" ) || "" ,
221 viewBox: node. getAttribute ( "viewBox" ) || symbol?. getAttribute ?.( "viewBox" ) || "" ,
222 },
223 presentation: {
224 color: styles.color,
225 fill: styles.fill,
226 stroke: styles.stroke,
227 },
228 svgMarkup: tag === "svg" ? node.outerHTML. slice ( 0 , 24000 ) : "" ,
229 symbolMarkup: symbol?.outerHTML?. slice ( 0 , 24000 ) || "" ,
230 domRef: {
231 tag,
232 id: node.id || "" ,
233 classTokens: String (node.className?.baseVal || node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 10 ),
234 },
235 };
236 }
237
238 function collectVisualAssets () {
239 const records = [];
240 const seen = new Set ();
241 const scopes = [
242 ... Array. from (document. querySelectorAll ( "header,[role='banner']" )). map (( scope ) => ({ scope, context: "header" })),
243 ... Array. from (document. querySelectorAll ( "footer,[role='contentinfo']" )). map (( scope ) => ({ scope, context: "footer" })),
244 ... Array. from (document. querySelectorAll ( "button,[role='button'],a[href]" )). map (( scope ) => ({ scope, context: scope. closest ( "footer" ) ? "footer-control" : scope. closest ( "header" ) ? "header-control" : "control" })),
245 ];
246 for ( const { scope , context } of scopes) {
247 const nodes = scope. matches ?.(visualAssetSelectors) ? [scope] : Array. from (scope. querySelectorAll (visualAssetSelectors));
248 for ( const node of nodes) {
249 if (node. closest ( "[data-rp-ignored-surface]" )) continue ;
250 const record = visualAssetRecord (node, context);
251 if ( ! record) continue ;
252 if (record.kind === "icon" && ! record.visible && ! record.useHref && ! record.sourceUrl) continue ;
253 const key = `${ record . kind } \u0000 ${ record . sourceType } \u0000 ${ record . sourceUrl } \u0000 ${ record . domRef . id } \u0000 ${ record . domRef . classTokens . join ( "." ) } \u0000 ${ context }` ;
254 if (seen. has (key)) continue ;
255 seen. add (key);
256 records. push (record);
257 if (records. length >= 120 ) return records;
258 }
259 }
260 return records;
261 }
262
263 function headerVariant ( name ) {
264 const candidate = pickHeader ();
265 if ( ! candidate) return { name, position: "unknown" , height: 0 , visibleAtScrollY: window.scrollY, visible: false };
266 const { node , rect , styles } = candidate;
267 const logo = Array. from (node. querySelectorAll (visualAssetSelectors))
268 . map (( asset ) => ({ asset, record: visualAssetRecord (asset, `header-${ name }` ) }))
269 . find (({ record }) => record?.kind === "logo" ) || null ;
270 const logoNode = logo?.asset || null ;
271 const logoRect = logoNode?. getBoundingClientRect ();
272 return {
273 name,
274 position: styles.position || "static" ,
275 height: Math. round (rect.height),
276 visibleAtScrollY: Math. round (window.scrollY),
277 visible: true ,
278 rect: rectFor (node),
279 logo: logoNode && logoRect ? {
280 ... logo.record,
281 sourceUrl: logo.record.sourceUrl,
282 alt: logoNode. getAttribute ( "alt" ) || logo.record.accessibleName || "" ,
283 renderedWidth: Math. round (logoRect.width),
284 renderedHeight: Math. round (logoRect.height),
285 } : null ,
286 };
287 }
288
289 function linkRecord ( anchor ) {
290 return {
291 label: semanticText (anchor). slice ( 0 , 120 ),
292 href: anchor.href || anchor. getAttribute ( "href" ) || "" ,
293 };
294 }
295
296 function isSameOriginHref ( href ) {
297 try {
298 return new URL (href, window.location.href).origin === window.location.origin;
299 } catch {
300 return false ;
301 }
302 }
303
304 function navScore ( scope , headerNode ) {
305 const anchors = Array. from (scope. querySelectorAll ( "a[href]" )). filter (isVisible);
306 if ( ! anchors. length ) return Number.NEGATIVE_INFINITY;
307 const records = anchors. map (linkRecord);
308 const sameOriginCount = records. filter (( item ) => isSameOriginHref (item.href)). length ;
309 const nonEmptyLabelCount = records. filter (( item ) => item.label). length ;
310 const longLabelCount = records. filter (( item ) => item.label. length >= 2 ). length ;
311 const externalCount = records. length - sameOriginCount;
312 const socialCount = anchors. filter (( anchor ) => {
313 const text = semanticText (anchor). toLowerCase ();
314 const classText = `${ anchor . className || ""} ${ anchor . parentElement ?. className || ""}` . toLowerCase ();
315 const href = String (anchor.href || "" ). toLowerCase ();
316 return /facebook | instagram | pinterest | youtube | tiktok | whatsapp | social | icon-/ . test ( `${ text } ${ classText } ${ href }` );
317 }). length ;
318 const topLevelListItems = Array. from (scope. querySelectorAll ( ":scope > ul > li, :scope > ol > li, :scope > li" ))
319 . filter (isVisible);
320 const nestedListItems = Array. from (scope. querySelectorAll ( "li li" )). filter (isVisible);
321 const uniqueHrefs = new Set (records. map (( item ) => item.href)).size;
322 const inHeader = Boolean (headerNode && headerNode. contains (scope));
323 const ariaLabel = cleanText (scope. getAttribute ( "aria-label" ) || "" ). toLowerCase ();
324 const classText = `${ scope . className || ""} ${ scope . id || ""}` . toLowerCase ();
325 const textSample = semanticText (scope). toLowerCase ();
326
327 let score = 0 ;
328 score += sameOriginCount * 10 ;
329 score += nonEmptyLabelCount * 6 ;
330 score += longLabelCount * 3 ;
331 score += Math. min (topLevelListItems. length , 12 ) * 8 ;
332 score += Math. min (nestedListItems. length , 24 ) * 3 ;
333 score += Math. min (uniqueHrefs, 20 ) * 2 ;
334 if (inHeader) score += 30 ;
335 if ( /main | primary | menu | navigation/ . test ( `${ ariaLabel } ${ classText }` )) score += 20 ;
336 if ( /social/ . test ( `${ ariaLabel } ${ classText }` )) score -= 40 ;
337 if (sameOriginCount <= 1 && externalCount > 0 ) score -= 35 ;
338 score -= externalCount * 6 ;
339 score -= socialCount * 25 ;
340 if (records. length <= 2 && sameOriginCount === 0 ) score -= 50 ;
341 if ( ! topLevelListItems. length && records. length <= 3 ) score -= 15 ;
342 if ( /facebook | instagram/ . test (textSample) && sameOriginCount === 0 ) score -= 50 ;
343 return score;
344 }
345
346 function navigationItems () {
347 const scopes = Array. from (document. querySelectorAll ( `header ${ navSelectors }, ${ navSelectors }` )). filter (isVisible);
348 const headerNode = pickHeader ()?.node || null ;
349 const scoredScopes = scopes
350 . map (( scope ) => ({ scope, score: navScore (scope, headerNode) }))
351 . sort (( a , b ) => b.score - a.score);
352 const scope = scoredScopes[ 0 ]?.scope;
353 if ( ! scope) return [];
354 const listItems = Array. from (scope. querySelectorAll ( "li" )). filter (( item ) => {
355 const parentLi = item.parentElement?. closest ( "li" );
356 return ! parentLi || ! scope. contains (parentLi);
357 });
358 const items = listItems. length ? listItems. map (( item ) => {
359 const anchor = Array. from (item.children). find (( child ) => child. matches ?.( "a[href]" )) || item. querySelector ( "a[href]" );
360 if ( ! anchor) return null ;
361 const children = Array. from (item. querySelectorAll ( "li a[href]" ))
362 . filter (( childAnchor ) => childAnchor !== anchor)
363 . map (linkRecord)
364 . filter (( child ) => child.label || child.href);
365 return { ... linkRecord (anchor), children: dedupeNav (children). slice ( 0 , 24 ) };
366 }) : Array. from (scope. querySelectorAll ( "a[href]" )). map (( anchor ) => ({ ... linkRecord (anchor), children: [] }));
367 return dedupeNav (items. filter (( item ) => item && (item.label || item.href))). slice ( 0 , 40 );
368 }
369
370 function dedupeNav ( items ) {
371 const seen = new Set ();
372 return items. filter (( item ) => {
373 const key = `${ item . label } \u0000 ${ item . href }` ;
374 if (seen. has (key)) return false ;
375 seen. add (key);
376 return true ;
377 });
378 }
379
380 function firstViewportText () {
381 const nodes = Array. from (document. querySelectorAll ( "main *:not(script):not(style), body > *:not(header):not(nav):not(footer):not(script):not(style)" ));
382 const texts = [];
383 const seen = new Set ();
384 for ( const node of nodes) {
385 if ( ! isVisible (node)) continue ;
386 if (node. closest ( "header,nav,footer" )) continue ;
387 const rect = node. getBoundingClientRect ();
388 if (rect.bottom < 0 || rect.top > window.innerHeight) continue ;
389 if (Array. from (node.children). some (( child ) => renderedText (child) === renderedText (node))) continue ;
390 const text = renderedText (node);
391 if ( ! text || text. length < 2 || seen. has (text)) continue ;
392 seen. add (text);
393 texts. push (text. slice ( 0 , 180 ));
394 if (texts. length >= 80 ) break ;
395 }
396 return texts;
397 }
398
399 const initialHeader = headerVariant ( "initial" );
400 const navigation = navigationItems ();
401 const visibleText = firstViewportText ();
402 const scrollTarget = Math. max ( 260 , Math. min (document.documentElement.scrollHeight - window.innerHeight, window.innerHeight * 0.75 ));
403 window. scrollTo ( 0 , scrollTarget);
404 await new Promise (( resolve ) => setTimeout (resolve, 250 ));
405 const scrolledHeader = headerVariant ( "scrolled" );
406 window. scrollTo ( 0 , 0 );
407
408 function domPath ( node , stopNode ) {
409 if ( ! node) return "" ;
410 const parts = [];
411 let current = node;
412 while (current && current.nodeType === Node. ELEMENT_NODE && current !== stopNode && parts. length < 8 ) {
413 const tag = current.tagName. toLowerCase ();
414 const id = current.id ? `#${ current . id }` : "" ;
415 const classTokens = String (current.className || "" )
416 . split ( / \s + / )
417 . filter (Boolean)
418 . slice ( 0 , 2 )
419 . map (( token ) => `.${ token }` )
420 . join ( "" );
421 const siblings = current.parentElement
422 ? Array. from (current.parentElement.children). filter (( sibling ) => sibling.tagName === current.tagName)
423 : [];
424 const nthOfType = siblings. length > 1 ? `:nth-of-type(${ siblings . indexOf ( current ) + 1 })` : "" ;
425 parts. unshift ( `${ tag }${ id }${ classTokens }${ id ? "" : nthOfType }` );
426 current = current.parentElement;
427 }
428 return parts. join ( " > " );
429 }
430
431 function visibleChildren ( node ) {
432 return Array. from (node?.children || []). filter (( child ) => isVisible (child));
433 }
434
435 function visibleHeadingText ( node ) {
436 return Array. from (node. querySelectorAll ( "h1,h2,h3,h4" ))
437 . filter (isVisible)
438 . map (( heading ) => renderedText (heading))
439 . find (Boolean) || "" ;
440 }
441
442 function visibleHeadings ( node ) {
443 return Array. from (node. querySelectorAll ( "h1,h2,h3,h4" ))
444 . filter (isVisible)
445 . map (( heading ) => ({
446 level: Number (heading.tagName. replace ( / ^ H/ i , "" )) || null ,
447 text: renderedText (heading),
448 }))
449 . filter (( heading ) => heading.text);
450 }
451
452 function isTransparentColor ( value ) {
453 const normalized = String (value || "" ). replace ( / \s + / g , "" ). toLowerCase ();
454 return ! normalized || normalized === "transparent" || normalized === "rgba(0,0,0,0)" ;
455 }
456
457 function backgroundInfo ( node ) {
458 const styles = getComputedStyle (node);
459 return {
460 color: styles.backgroundColor,
461 image: styles.backgroundImage && styles.backgroundImage !== "none" ? styles.backgroundImage : "" ,
462 };
463 }
464
465 function roundedBox ( rect ) {
466 return {
467 top: Math. round (rect.top + window.scrollY),
468 left: Math. round (rect.left),
469 width: Math. round (rect.width),
470 height: Math. round (rect.height),
471 };
472 }
473
474 function normalizedBox ( rect , sectionRect ) {
475 return {
476 x: Number (((rect.left - sectionRect.left) / Math. max (sectionRect.width, 1 )). toFixed ( 3 )),
477 y: Number (((rect.top - sectionRect.top) / Math. max (sectionRect.height, 1 )). toFixed ( 3 )),
478 width: Number ((rect.width / Math. max (sectionRect.width, 1 )). toFixed ( 3 )),
479 height: Number ((rect.height / Math. max (sectionRect.height, 1 )). toFixed ( 3 )),
480 };
481 }
482
483 function textGeometry ( node , rect ) {
484 const styles = getComputedStyle (node);
485 const lineTops = [];
486 const walker = document. createTreeWalker (node, NodeFilter. SHOW_TEXT , {
487 acceptNode ( textNode ) {
488 return cleanText (textNode.nodeValue). length ? NodeFilter. FILTER_ACCEPT : NodeFilter. FILTER_REJECT ;
489 },
490 });
491 for ( let textNode = walker. nextNode (); textNode; textNode = walker. nextNode ()) {
492 const range = document. createRange ();
493 range. selectNodeContents (textNode);
494 for ( const lineRect of range. getClientRects ()) {
495 if (lineRect.width > 0 && lineRect.height > 0 && ! lineTops. some (( top ) => Math. abs (top - lineRect.top) <= 2 )) lineTops. push (lineRect.top);
496 }
497 }
498 const lineCount = Math. max ( 1 , lineTops. length );
499 const maxWidth = styles.maxWidth;
500 return {
501 inlineSize: Math. round (rect.width * 100 ) / 100 ,
502 blockSize: Math. round (rect.height * 100 ) / 100 ,
503 lineCount,
504 wrapPolicy: lineCount === 1 ? "single-line" : styles.whiteSpace === "nowrap" ? "clipped-or-overflowing" : "wrapped" ,
505 whiteSpace: styles.whiteSpace,
506 maxWidth,
507 minWidth: styles.minWidth,
508 fontSize: styles.fontSize,
509 lineHeight: styles.lineHeight,
510 letterSpacing: styles.letterSpacing,
511 };
512 }
513
514 function layoutRole ( node ) {
515 const tag = node.tagName. toLowerCase ();
516 const role = node. getAttribute ( "role" ) || "" ;
517 const classText = `${ node . className || ""} ${ node . id || ""}` . toLowerCase ();
518 const peerControls = Array. from (node.parentElement?.children || []). filter (( candidate ) => {
519 const candidateTag = candidate.tagName?. toLowerCase ?.();
520 return candidateTag === "button" || candidate. getAttribute ?.( "role" ) === "tab" ;
521 });
522 const childControls = Array. from (node.children || []). filter (( candidate ) => {
523 const candidateTag = candidate.tagName?. toLowerCase ?.();
524 return candidateTag === "button" || candidate. getAttribute ?.( "role" ) === "tab" ;
525 });
526 if (role === "tablist" || / \b tabs ?\b| tab-list | tablist/ . test (classText) || childControls. length >= 3 ) return "tabs" ;
527 if (role === "tab" || /tab__btn | tab-button/ . test (classText) || (tag === "button" && peerControls. length >= 3 )) return "tab" ;
528 if ( / ^ h [1-6] $ / . test (tag)) return "heading" ;
529 if (tag === "p" || tag === "blockquote" ) return "body-copy" ;
530 if (tag === "video" || tag === "iframe" || tag === "picture" || tag === "img" ) return "media" ;
531 if (tag === "button" || role === "button" || (tag === "a" && /button | cta | contact | learn | read | discover | explore/ . test (classText))) return "action" ;
532 if (tag === "nav" || role === "navigation" ) return "navigation" ;
533 return "content" ;
534 }
535
536 const STYLEABLE_CATEGORY_PROPERTIES = {
537 text: [ "color" , "font-family" , "font-size" , "font-style" , "font-weight" , "letter-spacing" , "line-height" , "text-align" , "text-decoration" , "text-transform" , "white-space" ],
538 action: [ "color" , "background-color" , "border-color" , "border-radius" , "border-style" , "border-width" , "box-shadow" , "cursor" , "min-height" , "padding" , "transition" ],
539 media: [ "aspect-ratio" , "border-radius" , "height" , "object-fit" , "object-position" , "opacity" , "width" ],
540 container: [ "align-items" , "background-color" , "background-image" , "border-color" , "border-radius" , "box-shadow" , "display" , "gap" , "grid-template-columns" , "justify-content" , "overflow" , "padding" , "position" , "z-index" ],
541 };
542
543 function styleableCategories ( role ) {
544 if ([ "heading" , "body-copy" , "tab" ]. includes (role)) return role === "tab" ? [ "text" , "action" ] : [ "text" ];
545 if (role === "action" ) return [ "text" , "action" ];
546 if ([ "media" , "background-media" ]. includes (role)) return [ "media" ];
547 return [ "container" ];
548 }
549
550 function computedPropertyUnion ( styles , categories ) {
551 const properties = [ ...new Set (categories. flatMap (( category ) => STYLEABLE_CATEGORY_PROPERTIES [category] || []))]. sort ();
552 return Object. fromEntries (properties. map (( property ) => [property, styles. getPropertyValue (property)]));
553 }
554
555 function authoredCssContext ( node , categories ) {
556 const allowed = new Set (categories. flatMap (( category ) => STYLEABLE_CATEGORY_PROPERTIES [category] || []));
557 const matches = [];
558 for ( const sheet of Array. from (document.styleSheets). slice ( 0 , 80 )) {
559 let rules;
560 try { rules = Array. from (sheet.cssRules || []); } catch { continue ; }
561 for ( const rule of rules. slice ( 0 , 800 )) {
562 if ( ! rule.selectorText || ! rule.style) continue ;
563 let matched = false ;
564 try { matched = node. matches (rule.selectorText); } catch { continue ; }
565 if ( ! matched) continue ;
566 const declarations = {};
567 for ( const property of Array. from (rule.style)) if (allowed. has (property)) declarations[property] = rule.style. getPropertyValue (property);
568 if (Object. keys (declarations). length ) matches. push ({ selector: rule.selectorText, href: sheet.href || "inline" , declarations });
569 if (matches. length >= 20 ) return matches;
570 }
571 }
572 return matches;
573 }
574
575 function collectLayoutEvidence ( sectionRoot ) {
576 const sectionRect = sectionRoot. getBoundingClientRect ();
577 const sectionStyles = getComputedStyle (sectionRoot);
578 const selector = [
579 "h1" , "h2" , "h3" , "h4" , "h5" , "h6" , "p" , "blockquote" ,
580 "a[href]" , "button" , "[role='button']" , "[role='tablist']" , "[role='tab']" ,
581 "nav" , "[class*='tabs' i]" , "[class*='tab-list' i]" ,
582 "video" , "iframe" , "picture" , "img" ,
583 ]. join ( "," );
584 const semanticCandidates = Array. from (sectionRoot. querySelectorAll (selector));
585 const repeatedControlGroups = [ ...new Set (Array. from (sectionRoot. querySelectorAll ( "button, [role='tab']" ))
586 . map (( control ) => control.parentElement)
587 . filter (Boolean)
588 . filter (( parent ) => Array. from (parent.children). filter (( child ) => child.tagName?. toLowerCase ?.() === "button" || child. getAttribute ?.( "role" ) === "tab" ). length >= 3 ))];
589 const regionCandidates = [ ...new Set ([ ... semanticCandidates, ... repeatedControlGroups])]
590 . filter (isVisible)
591 . filter (( node ) => {
592 const rect = node. getBoundingClientRect ();
593 if (rect.width < 20 || rect.height < 12 ) return false ;
594 if ([ "p" , "blockquote" ]. includes (node.tagName. toLowerCase ()) && renderedText (node). length < 16 ) return false ;
595 return true ;
596 });
597 const regions = [];
598 for ( const node of regionCandidates) {
599 const role = layoutRole (node);
600 if (role === "tab" && regionCandidates. some (( candidate ) => candidate !== node && layoutRole (candidate) === "tabs" && candidate. contains (node))) continue ;
601 let rect = node. getBoundingClientRect ();
602 if (role === "tabs" ) {
603 const tabRects = Array. from (node. querySelectorAll ( "[role='tab'], button, a" ))
604 . filter (isVisible)
605 . filter (( candidate ) => layoutRole (candidate) === "tab" )
606 . map (( candidate ) => candidate. getBoundingClientRect ());
607 if (tabRects. length >= 2 ) {
608 const left = Math. min ( ... tabRects. map (( candidate ) => candidate.left));
609 const top = Math. min ( ... tabRects. map (( candidate ) => candidate.top));
610 const right = Math. max ( ... tabRects. map (( candidate ) => candidate.right));
611 const bottom = Math. max ( ... tabRects. map (( candidate ) => candidate.bottom));
612 rect = { left, top, right, bottom, width: right - left, height: bottom - top };
613 }
614 }
615 const styles = getComputedStyle (node);
616 const categories = styleableCategories (role);
617 regions. push ({
618 role,
619 text: [ "heading" , "body-copy" , "action" , "tab" , "tabs" ]. includes (role) ? renderedText (node). slice ( 0 , 180 ) : "" ,
620 rect: roundedBox (rect),
621 normalizedRect: normalizedBox (rect, sectionRect),
622 position: styles.position,
623 display: styles.display,
624 textAlign: styles.textAlign,
625 zIndex: styles.zIndex,
626 styleableCategories: categories,
627 computedStyle: computedPropertyUnion (styles, categories),
628 authoredCss: authoredCssContext (node, categories),
629 ... ([ "heading" , "body-copy" , "action" , "tab" , "tabs" ]. includes (role) ? { textGeometry: textGeometry (node, rect) } : {}),
630 domRef: {
631 tag: node.tagName. toLowerCase (),
632 id: node.id || "" ,
633 classTokens: String (node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 6 ),
634 },
635 });
636 if (regions. length >= 36 ) break ;
637 }
638
639 const layerCandidates = Array. from (sectionRoot. querySelectorAll ( "video, iframe, picture, img, [class*='background' i], [class*='overlay' i], [class*='curtain' i], [class*='mask' i], [class*='panel' i], [class*='frame' i], [class*='border' i]" ))
640 . filter (isVisible)
641 . map (( node ) => ({ node, rect: node. getBoundingClientRect (), styles: getComputedStyle (node) }))
642 . filter (({ rect , styles }) => {
643 const areaRatio = (rect.width * rect.height) / Math. max (sectionRect.width * Math. min (sectionRect.height, window.innerHeight), 1 );
644 return areaRatio >= 0.18 || [ "absolute" , "fixed" , "sticky" ]. includes (styles.position);
645 })
646 . slice ( 0 , 20 )
647 . map (({ node , rect , styles }) => {
648 const classText = String (node.className || "" ). toLowerCase ();
649 const kind = [ "video" , "iframe" ]. includes (node.tagName. toLowerCase ()) ? "video" : [ "picture" , "img" ]. includes (node.tagName. toLowerCase ()) ? "image" : "decorative-layer" ;
650 const role = /partial . * border | border . * frame |\b frame \b / . test (classText) ? "partial-border-frame"
651 : /copy . * panel | content . * panel/ . test (classText) ? "copy-panel"
652 : /overlay | curtain | mask | scrim/ . test (classText) ? "overlay"
653 : /background |\b bg \b / . test (classText) ? "background"
654 : kind === "image" || kind === "video" ? "media" : "decoration" ;
655 return {
656 kind,
657 role,
658 rect: roundedBox (rect),
659 normalizedRect: normalizedBox (rect, sectionRect),
660 position: styles.position,
661 zIndex: styles.zIndex,
662 opacity: styles.opacity,
663 objectFit: styles.objectFit,
664 overflow: styles.overflow,
665 background: backgroundInfo (node),
666 src: node.currentSrc || node.src || "" ,
667 classTokens: String (node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 8 ),
668 };
669 });
670
671 const pseudoLayers = [ "::before" , "::after" ]. map (( pseudo ) => {
672 const styles = getComputedStyle (sectionRoot, pseudo);
673 if ( ! styles || styles.content === "none" || ( isTransparentColor (styles.backgroundColor) && ( ! styles.backgroundImage || styles.backgroundImage === "none" ))) return null ;
674 return {
675 kind: "pseudo-layer" ,
676 pseudo,
677 position: styles.position,
678 zIndex: styles.zIndex,
679 opacity: styles.opacity,
680 background: {
681 color: styles.backgroundColor,
682 image: styles.backgroundImage && styles.backgroundImage !== "none" ? styles.backgroundImage : "" ,
683 },
684 };
685 }). filter (Boolean);
686
687 return {
688 vocabularyVersion: 1 ,
689 viewport: { width: window.innerWidth, height: window.innerHeight },
690 canvas: {
691 display: sectionStyles.display,
692 position: sectionStyles.position,
693 overflowX: sectionStyles.overflowX,
694 overflowY: sectionStyles.overflowY,
695 minHeight: sectionStyles.minHeight,
696 background: backgroundInfo (sectionRoot),
697 },
698 regions,
699 layers: [ ... layerCandidates, ... pseudoLayers],
700 };
701 }
702
703 function nodeFingerprint ( node ) {
704 const heading = visibleHeadingText (node);
705 const textPrefix = renderedText (node). slice ( 0 , 160 );
706 return {
707 heading,
708 textPrefix,
709 imageCount: node. querySelectorAll ( "img" ). length ,
710 linkCount: node. querySelectorAll ( "a[href]" ). length ,
711 };
712 }
713
714 function findContentRoot () {
715 const candidateDefs = [
716 { selector: "main" , label: "main" },
717 { selector: "[role='main']" , label: "role-main" },
718 { selector: "article" , label: "article" },
719 { selector: "#primary" , label: "primary-id" },
720 { selector: ".content-area" , label: "content-area" },
721 { selector: ".site-main" , label: "site-main" },
722 { selector: "body" , label: "body" },
723 ];
724 const candidates = [];
725 for ( const def of candidateDefs) {
726 for ( const node of Array. from (document. querySelectorAll (def.selector)). filter (isVisible). slice ( 0 , 3 )) {
727 const rect = node. getBoundingClientRect ();
728 const headingCount = node. querySelectorAll ( "h1,h2,h3,h4" ). length ;
729 const imageCount = node. querySelectorAll ( "img" ). length ;
730 const linkCount = node. querySelectorAll ( "a[href]" ). length ;
731 let score = 0 ;
732 score += 40 ;
733 score += Math. min (rect.width / Math. max (window.innerWidth, 1 ), 1.2 ) * 35 ;
734 score += Math. min (rect.height / Math. max (window.innerHeight, 1 ), 4 ) * 10 ;
735 score += Math. min (headingCount, 8 ) * 4 ;
736 score += Math. min (imageCount, 20 ) * 1.5 ;
737 if (linkCount > 3 ) score += 5 ;
738 if (def.label === "main" ) score += 20 ;
739 else if (def.label === "role-main" ) score += 18 ;
740 else if (def.label === "article" ) score += 10 ;
741 else if (def.label === "body" ) score -= 30 ;
742 if (rect.height > document.documentElement.scrollHeight * 0.9 ) score -= 25 ;
743 candidates. push ({ node, label: def.label, score });
744 }
745 }
746 candidates. sort (( a , b ) => b.score - a.score);
747 return candidates[ 0 ] || { node: document.body, label: "body" , score: 0 };
748 }
749
750 function collectModuleStats ( node , root ) {
751 const rect = node. getBoundingClientRect ();
752 const children = visibleChildren (node);
753 const headings = visibleHeadings (node);
754 const images = Array. from (node. querySelectorAll ( "img" )). filter (( img ) => isVisible (img));
755 const links = Array. from (node. querySelectorAll ( "a[href]" )). filter (( link ) => isVisible (link));
756 const buttons = Array. from (node. querySelectorAll ( "button" )). filter (( button ) => isVisible (button));
757 const paragraphs = Array. from (node. querySelectorAll ( "p, li, blockquote, figcaption" )). filter (( item ) => isVisible (item));
758 const childRects = children. map (( child ) => child. getBoundingClientRect ());
759 const rowCountEstimate = new Set (childRects. map (( item ) => Math. round (item.top / 24 ))).size;
760 const background = backgroundInfo (node);
761 const role = node. getAttribute ( "role" ) || "" ;
762 const classText = `${ node . className || ""} ${ node . id || ""}` . toLowerCase ();
763 const ancestorText = [];
764 let ancestor = node.parentElement;
765 for ( let depth = 0 ; ancestor && depth < 4 ; depth += 1 ) {
766 ancestorText. push ( `${ ancestor . className || ""} ${ ancestor . id || ""}` . toLowerCase ());
767 if (ancestor === root) break ;
768 ancestor = ancestor.parentElement;
769 }
770 const ancestryText = ancestorText. join ( " " );
771 const reviewPattern = /review | testimonial | verified customer | productreview | based on [\d,.\s] + reviews | what our customers/ i ;
772 const reviewLike = reviewPattern. test (classText)
773 || headings. some (( heading ) => reviewPattern. test (heading.text))
774 || reviewPattern. test ( renderedText (node))
775 || reviewPattern. test (ancestryText);
776 const pricePattern = / \$ \s ? \d |\b price \b| original price | current price | sale \b / i ;
777 const addToCartPattern = /add to cart | quantity | buy now | shop now/ i ;
778 const productLike = pricePattern. test ( renderedText (node))
779 || addToCartPattern. test ( renderedText (node))
780 || /product | products | woocommerce | add-to-cart | price | shopify-buy/ . test ( `${ classText } ${ ancestryText }` );
781 const reviewWidgetLike = /reviewsio | judge \. me | stamped | loox | yotpo | trustpilot | productreview/ . test ( `${ classText } ${ ancestryText }` );
782 const galleryLike = /gallery | slider | embla | swiper | carousel | featured/ . test (classText);
783 const logoLikeChildren = images. filter (( image ) => {
784 const imageRect = image. getBoundingClientRect ();
785 return imageRect.width <= 240 && imageRect.height <= 140 ;
786 }). length ;
787 return {
788 node,
789 tag: node.tagName. toLowerCase (),
790 role,
791 rect,
792 heading: headings[ 0 ]?.text || "" ,
793 headings,
794 text: renderedText (node). slice ( 0 , 1800 ) || "" ,
795 textLength: renderedText (node). length ,
796 visibleCodeTexts: visibleCodeTexts (node),
797 imageCount: images. length ,
798 linkCount: links. length ,
799 buttonCount: buttons. length ,
800 paragraphCount: paragraphs. length ,
801 childCount: children. length ,
802 cardLikeChildren: children. filter (( child ) => {
803 const childRect = child. getBoundingClientRect ();
804 return childRect.width >= 140 && childRect.height >= 90 ;
805 }). length ,
806 rowCountEstimate,
807 background,
808 hasBackgroundChange: ! isTransparentColor (background.color) || Boolean (background.image),
809 reviewLike,
810 reviewWidgetLike,
811 productLike,
812 galleryLike,
813 logoLikeChildren,
814 path: domPath (node, root?.parentElement || document.body.parentElement),
815 parentPath: domPath (node.parentElement, root?.parentElement || document.body.parentElement),
816 nthOfType: node.parentElement
817 ? Array. from (node.parentElement.children). filter (( sibling ) => sibling.tagName === node.tagName). indexOf (node) + 1
818 : 1 ,
819 fingerprint: nodeFingerprint (node),
820 };
821 }
822
823 function statsClassText ( stats ) {
824 return `${ stats . node ?. className || ""} ${ stats . node ?. id || ""} ${ stats . path || ""}` . toLowerCase ();
825 }
826
827 function sectionCapabilities ({ hasCta , hasMedia , repeatingItems = false , isCarouselLike = false }) {
828 return { hasCta, hasMedia, repeatingItems, isCarouselLike };
829 }
830
831 const SECTION_DETECTORS = [
832 {
833 kind: "header" ,
834 match : ({ stats , classText , hasCta }) => (stats.tag === "header" || / \b header \b| masthead | top-bar | site-header/ . test (classText))
835 ? {
836 kind: "header" ,
837 variant: /top-bar/ . test (classText) ? "top-bar" : "generic" ,
838 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 }),
839 }
840 : null ,
841 },
842 {
843 kind: "footer" ,
844 match : ({ stats , classText , hasCta }) => (stats.tag === "footer" || / \b footer \b| site-info | copyright | privacy | terms | site-map/ . test (classText))
845 ? {
846 kind: "footer" ,
847 variant: "site-footer" ,
848 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 }),
849 }
850 : null ,
851 },
852 {
853 kind: "hero" ,
854 match : ({ stats , classText , hasCta , atTopOfPage , hasStrongHeroMedia }) => (
855 hasStrongHeroMedia
856 && hasCta
857 && stats.rect.height >= Math. max (window.innerHeight * 0.22 , 220 )
858 && (atTopOfPage || / \b hero \b| banner | masthead | billboard | home-section/ . test (classText) || stats.headings. some (( heading ) => heading.level === 1 ))
859 )
860 ? {
861 kind: "hero" ,
862 variant: stats.galleryLike || stats.cardLikeChildren >= 2 ? "carousel-hero" : "generic-hero" ,
863 capabilities: sectionCapabilities ({ hasCta: true , hasMedia: true , repeatingItems: stats.cardLikeChildren >= 2 , isCarouselLike: stats.galleryLike }),
864 }
865 : null ,
866 },
867 {
868 kind: "reviews" ,
869 match : ({ stats , hasCta }) => (
870 (stats.reviewWidgetLike || stats.reviewLike)
871 && ! (stats.productLike && ! stats.reviewWidgetLike)
872 && (stats.cardLikeChildren >= 1 || stats.imageCount >= 2 || stats.textLength >= 80 )
873 )
874 ? {
875 kind: "reviews" ,
876 variant: stats.galleryLike || stats.cardLikeChildren >= 2 ? "review-carousel" : "review-list" ,
877 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 , repeatingItems: true , isCarouselLike: stats.galleryLike }),
878 }
879 : null ,
880 },
881 {
882 kind: "category-strip" ,
883 match : ({ stats }) => (
884 stats.imageCount === 0
885 && stats.linkCount === 0
886 && stats.buttonCount === 0
887 && stats.paragraphCount >= 4
888 && stats.textLength >= 40
889 && stats.textLength <= 280
890 )
891 ? {
892 kind: "category-strip" ,
893 variant: "text-category-grid" ,
894 capabilities: sectionCapabilities ({ hasCta: false , hasMedia: false , repeatingItems: true }),
895 }
896 : null ,
897 },
898 {
899 kind: "card-collection" ,
900 match : ({ stats , hasCta }) => (
901 (stats.logoLikeChildren >= 4 && stats.imageCount >= 4 && stats.textLength < 420 )
902 || (stats.galleryLike && stats.imageCount >= 5 && stats.textLength < 180 )
903 )
904 ? {
905 kind: "card-collection" ,
906 variant: "logo-gallery" ,
907 capabilities: sectionCapabilities ({ hasCta, hasMedia: true , repeatingItems: true , isCarouselLike: stats.galleryLike }),
908 }
909 : null ,
910 },
911 {
912 kind: "card-collection" ,
913 match : ({ stats }) => (stats.productLike && (stats.cardLikeChildren >= 3 || stats.imageCount >= 4 ) && (stats.buttonCount >= 2 || stats.linkCount >= 6 ))
914 ? {
915 kind: "card-collection" ,
916 variant: stats.galleryLike ? "product-carousel" : "product-grid" ,
917 capabilities: sectionCapabilities ({ hasCta: true , hasMedia: true , repeatingItems: true , isCarouselLike: stats.galleryLike }),
918 }
919 : null ,
920 },
921 {
922 kind: "card-collection" ,
923 match : ({ stats , hasCta }) => (stats.cardLikeChildren >= 3 && (stats.imageCount >= 3 || stats.linkCount >= 3 ))
924 ? {
925 kind: "card-collection" ,
926 variant: stats.galleryLike ? "card-carousel" : "card-grid" ,
927 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 , repeatingItems: true , isCarouselLike: stats.galleryLike }),
928 }
929 : null ,
930 },
931 {
932 kind: "text-media" ,
933 match : ({ stats , classText , hasCta }) => (stats.imageCount >= 1 && stats.headings. length >= 1 && stats.paragraphCount >= 1 )
934 ? {
935 kind: "text-media" ,
936 variant: /has-media-on-the-right | media-right/ . test (classText) ? "media-right" : "media-left" ,
937 capabilities: sectionCapabilities ({ hasCta, hasMedia: true }),
938 }
939 : null ,
940 },
941 {
942 kind: "stat-group" ,
943 match : ({ stats }) => (stats.headings. length >= 3 && stats.imageCount === 0 && stats.linkCount === 0 && stats.buttonCount === 0 )
944 ? {
945 kind: "stat-group" ,
946 variant: "step-strip" ,
947 capabilities: sectionCapabilities ({ hasCta: false , hasMedia: false , repeatingItems: true }),
948 }
949 : null ,
950 },
951 {
952 kind: "promo-band" ,
953 match : ({ stats , hasCta }) => (
954 stats.imageCount <= 1
955 && stats.textLength >= 40
956 && stats.textLength <= 220
957 && stats.paragraphCount <= 2
958 && stats.headings. length <= 1
959 && (stats.hasBackgroundChange || stats.imageCount >= 1 )
960 )
961 ? {
962 kind: "promo-band" ,
963 variant: hasCta ? "value-prop-cta" : "value-prop-band" ,
964 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 }),
965 }
966 : null ,
967 },
968 {
969 kind: "cta-strip" ,
970 match : ({ stats , hasCta }) => (
971 stats.imageCount === 0
972 && hasCta
973 && stats.textLength <= 220
974 && stats.paragraphCount <= 1
975 && stats.headings. length <= 2
976 )
977 ? {
978 kind: "cta-strip" ,
979 variant: "inline-cta-strip" ,
980 capabilities: sectionCapabilities ({ hasCta: true , hasMedia: false }),
981 }
982 : null ,
983 },
984 {
985 kind: "cta-strip" ,
986 match : ({ stats , hasCta , atTopOfPage }) => ((stats.buttonCount + stats.linkCount) >= 2 && stats.imageCount >= 1 && stats.rect.height >= 120 )
987 ? {
988 kind: "cta-strip" ,
989 variant: atTopOfPage ? "promo-hero-strip" : "promo-strip" ,
990 capabilities: sectionCapabilities ({ hasCta: true , hasMedia: true }),
991 }
992 : null ,
993 },
994 {
995 kind: "cta-strip" ,
996 match : ({ stats , hasCta }) => (stats.headings. length === 1 && stats.imageCount === 0 && stats.paragraphCount === 0 )
997 ? {
998 kind: "cta-strip" ,
999 variant: "centered-heading-strip" ,
1000 capabilities: sectionCapabilities ({ hasCta, hasMedia: false }),
1001 }
1002 : null ,
1003 },
1004 {
1005 kind: "rich-text" ,
1006 match : ({ stats , hasCta }) => (stats.textLength >= 60 || stats.headings. length || stats.imageCount)
1007 ? {
1008 kind: "rich-text" ,
1009 variant: "generic" ,
1010 capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 }),
1011 }
1012 : null ,
1013 },
1014 ];
1015
1016 function classifySection ( stats ) {
1017 const classText = statsClassText (stats);
1018 const atTopOfPage = (stats.rect.top + window.scrollY) < Math. max (window.innerHeight * 0.35 , 320 );
1019 const hasCta = (stats.buttonCount + stats.linkCount) >= 1 ;
1020 const hasStrongHeroMedia = stats.imageCount >= 1 || stats.hasBackgroundChange;
1021 for ( const detector of SECTION_DETECTORS ) {
1022 const matched = detector. match ({ stats, classText, atTopOfPage, hasCta, hasStrongHeroMedia });
1023 if (matched) return matched;
1024 }
1025 return { kind: "unknown" , variant: "generic" , capabilities: sectionCapabilities ({ hasCta, hasMedia: stats.imageCount >= 1 }) };
1026 }
1027
1028 function sectionScore ( stats , root ) {
1029 let score = 0 ;
1030 score += 30 ;
1031 score += Math. min (stats.rect.width / Math. max (root. getBoundingClientRect ().width, 1 ), 1.2 ) * 30 ;
1032 score += Math. min (stats.rect.height / Math. max (window.innerHeight, 1 ), 1.5 ) * 20 ;
1033 score += Math. min (stats.headings. length , 4 ) * 10 ;
1034 score += Math. min (stats.imageCount, 6 ) * 4 ;
1035 score += Math. min (stats.cardLikeChildren, 6 ) * 5 ;
1036 if (stats.hasBackgroundChange) score += 10 ;
1037 if (stats.rect.width < root. getBoundingClientRect ().width * 0.55 ) score -= 20 ;
1038 if (stats.rect.height < 80 ) score -= 40 ;
1039 if (stats.textLength < 15 && stats.imageCount === 0 ) score -= 60 ;
1040 if ( /nav | menu | breadcrumb | popup | modal | drawer | cookie/ . test ( `${ stats . role } ${ stats . path }` . toLowerCase ())) score -= 35 ;
1041 if (stats.tag === "main" || stats.tag === "article" ) score -= 20 ;
1042 if (stats.rect.height > root. getBoundingClientRect ().height * 0.8 ) score -= 50 ;
1043 if (stats.childCount === 1 && stats.cardLikeChildren <= 1 && stats.headings. length <= 1 && ! stats.hasBackgroundChange) score -= 40 ;
1044 if (stats.textLength > 3200 && stats.rect.height > root. getBoundingClientRect ().height * 0.55 && ! stats.hasBackgroundChange) score -= 45 ;
1045 return score;
1046 }
1047
1048 function isWrapperLike ( stats , root ) {
1049 return (
1050 stats.childCount <= 2
1051 && stats.cardLikeChildren <= 1
1052 && stats.headings. length <= 1
1053 && ! stats.hasBackgroundChange
1054 && stats.rect.height > Math. max (window.innerHeight * 1.25 , root. getBoundingClientRect ().height * 0.45 )
1055 );
1056 }
1057
1058 function findSegmentationRoot ( contentRoot ) {
1059 const warnings = [];
1060 let current = contentRoot;
1061 for ( let depth = 0 ; depth < 10 ; depth += 1 ) {
1062 const scopedContentChild = Array. from (current.children || []). find (( child ) => {
1063 if ( ! isVisible (child)) return false ;
1064 const classText = `${ child . className || ""} ${ child . id || ""}` . toLowerCase ();
1065 return /single-content | entry-content | post-content | article-content | main-content/ . test (classText);
1066 });
1067 if (scopedContentChild) {
1068 const childStats = collectModuleStats (scopedContentChild, contentRoot);
1069 current = scopedContentChild;
1070 warnings. push ( `segmentation-root-descended:${ childStats . path }` );
1071 continue ;
1072 }
1073 const children = visibleChildren (current). filter (( child ) => {
1074 const rect = child. getBoundingClientRect ();
1075 return rect.height >= 24 && ( renderedText (child). length >= 20 || child. querySelector ( "img,svg,a[href],button" ) || rect.height >= 120 );
1076 });
1077 if ( ! children. length ) break ;
1078 if (children. length === 1 ) {
1079 const childStats = collectModuleStats (children[ 0 ], contentRoot);
1080 if (childStats.rect.width >= current. getBoundingClientRect ().width * 0.72 ) {
1081 current = children[ 0 ];
1082 warnings. push ( `segmentation-root-descended:${ childStats . path }` );
1083 continue ;
1084 }
1085 }
1086 const dominant = children. find (( child ) => {
1087 const rect = child. getBoundingClientRect ();
1088 return rect.width >= current. getBoundingClientRect ().width * 0.72 && rect.height >= current. getBoundingClientRect ().height * 0.72 ;
1089 });
1090 if (children. length <= 3 && dominant) {
1091 const dominantStats = collectModuleStats (dominant, contentRoot);
1092 if ( ! dominantStats.hasBackgroundChange && dominantStats.cardLikeChildren <= 1 ) {
1093 current = dominant;
1094 warnings. push ( `segmentation-root-descended:${ dominantStats . path }` );
1095 continue ;
1096 }
1097 }
1098 const dominantCandidate = children
1099 . map (( child ) => ({ child, rect: child. getBoundingClientRect () }))
1100 . sort (( a , b ) => b.rect.height - a.rect.height)[ 0 ];
1101 if (dominantCandidate && children. length <= 5 ) {
1102 const otherHeight = children
1103 . filter (( child ) => child !== dominantCandidate.child)
1104 . reduce (( sum , child ) => sum + child. getBoundingClientRect ().height, 0 );
1105 const dominantClassText = `${ dominantCandidate . child . className || ""} ${ dominantCandidate . child . id || ""}` . toLowerCase ();
1106 if (dominantCandidate.rect.height >= current. getBoundingClientRect ().height * 0.6 && otherHeight <= dominantCandidate.rect.height * 0.35 ) {
1107 const dominantStats = collectModuleStats (dominantCandidate.child, contentRoot);
1108 if ( ! dominantStats.hasBackgroundChange) {
1109 current = dominantCandidate.child;
1110 warnings. push ( `segmentation-root-descended:${ dominantStats . path }` );
1111 continue ;
1112 }
1113 }
1114 if (
1115 dominantCandidate.rect.width >= current. getBoundingClientRect ().width * 0.68
1116 && dominantCandidate.rect.height >= current. getBoundingClientRect ().height * 0.45
1117 && otherHeight <= dominantCandidate.rect.height * 0.8
1118 && /content | entry | article | post | main/ . test (dominantClassText)
1119 ) {
1120 const dominantStats = collectModuleStats (dominantCandidate.child, contentRoot);
1121 current = dominantCandidate.child;
1122 warnings. push ( `segmentation-root-descended:${ dominantStats . path }` );
1123 continue ;
1124 }
1125 }
1126 break ;
1127 }
1128 return { node: current, warnings };
1129 }
1130
1131 function isStructuralBand ( stats , segmentationRoot ) {
1132 const lowerTag = stats.tag. toLowerCase ();
1133 if ( / ^ h [1-6] $ / . test (lowerTag) || lowerTag === "p" ) return false ;
1134 if (lowerTag === "div" && stats.rect.height < 100 && ! stats.hasBackgroundChange && stats.imageCount === 0 && stats.cardLikeChildren === 0 ) return false ;
1135 if (stats.rect.width < segmentationRoot. getBoundingClientRect ().width * 0.55 && stats.imageCount === 0 && stats.headings. length === 0 ) return false ;
1136 return true ;
1137 }
1138
1139 function overlaps ( a , b ) {
1140 const top = Math. max (a.rect.top, b.rect.top);
1141 const bottom = Math. min (a.rect.top + a.rect.height, b.rect.top + b.rect.height);
1142 const overlapHeight = Math. max ( 0 , bottom - top);
1143 return overlapHeight >= Math. min (a.rect.height, b.rect.height) * 0.6 ;
1144 }
1145
1146 function classifyModule ( stats , sectionType = null ) {
1147 const classText = statsClassText (stats);
1148 const text = String (stats.text || "" ). toLowerCase ();
1149 const hasCta = stats.linkCount + stats.buttonCount >= 1 ;
1150 const sectionKey = sectionType ? `${ sectionType . kind }:${ sectionType . variant || "*"}` : "" ;
1151 const moduleDetectors = {
1152 "reviews:*" : () => ({ kind: "review-card" , variant: "customer-review" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } }),
1153 "stat-group:*" : () => ({ kind: "stat-item" , variant: "step-or-stat" , capabilities: { hasCta: false , hasMedia: false } }),
1154 "category-strip:*" : () => ({ kind: "text-block" , variant: "category-item" , capabilities: { hasCta, hasMedia: false } }),
1155 "promo-band:*" : () => (stats.imageCount >= 1 && stats.headings. length === 0 && stats.paragraphCount === 0
1156 ? { kind: "media-item" , variant: "promo-media" , capabilities: { hasCta, hasMedia: true } }
1157 : { kind: "text-block" , variant: "promo-copy" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } }),
1158 "cta-strip:*" : () => ({ kind: "text-block" , variant: hasCta ? "cta-copy" : "strip-copy" , capabilities: { hasCta, hasMedia: false } }),
1159 "text-media:*" : () => {
1160 if (stats.imageCount >= 1 && stats.paragraphCount === 0 && stats.headings. length === 0 ) return { kind: "media-item" , variant: "image-tile" , capabilities: { hasCta, hasMedia: true } };
1161 if (stats.headings. length >= 1 || stats.paragraphCount >= 1 ) return { kind: "text-block" , variant: "copy-block" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1162 return null ;
1163 },
1164 "card-collection:logo-gallery" : () => ({ kind: "logo-item" , variant: "brand-mark" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } }),
1165 "card-collection:*" : () => ({ kind: "card" , variant: /product | price | add to cart | book rental | rent/ . test (text) ? "product-card" : "content-card" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } }),
1166 "footer:*" : () => ({ kind: "link-group" , variant: "footer-links" , capabilities: { hasCta: stats.linkCount >= 1 , hasMedia: false } }),
1167 "hero:*" : () => {
1168 if (stats.headings. length >= 1 || stats.paragraphCount >= 1 ) return { kind: "text-block" , variant: "hero-copy" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1169 if (stats.imageCount >= 1 ) return { kind: "media-item" , variant: "hero-media" , capabilities: { hasCta, hasMedia: true } };
1170 return null ;
1171 },
1172 };
1173 const specialized = (moduleDetectors[sectionKey] || moduleDetectors[ `${ sectionType ?. kind }:*` ])?.();
1174 if (specialized) return specialized;
1175 if (stats.reviewLike || /review | testimonial | verified customer/ . test ( `${ classText } ${ text }` )) return { kind: "review-card" , variant: "customer-review" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1176 if (stats.productLike) return { kind: "card" , variant: "product-card" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1177 if (stats.logoLikeChildren >= 1 || (stats.imageCount >= 1 && stats.textLength < 40 && stats.rect.width <= 260 )) return { kind: "logo-item" , variant: "brand-mark" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1178 if (stats.imageCount >= 1 && stats.headings. length === 0 && stats.paragraphCount === 0 ) return { kind: "media-item" , variant: "image-tile" , capabilities: { hasCta, hasMedia: true } };
1179 if (stats.headings. length >= 1 || stats.paragraphCount >= 1 ) return { kind: "text-block" , variant: "copy-block" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1180 return { kind: "unknown" , variant: "generic" , capabilities: { hasCta, hasMedia: stats.imageCount >= 1 } };
1181 }
1182
1183 function moduleSummary ( stats , sectionRoot , sectionType = null ) {
1184 const moduleType = classifyModule (stats, sectionType);
1185 return {
1186 id: "" ,
1187 kind: moduleType.kind,
1188 variant: moduleType.variant,
1189 capabilities: moduleType.capabilities,
1190 heading: stats.heading,
1191 visibleCodeTexts: stats.visibleCodeTexts,
1192 order: 0 ,
1193 confidence: Number (Math. max ( 0.2 , Math. min ( 0.95 , 0.45 + ((stats.imageCount + stats.headings. length + stats.cardLikeChildren) * 0.04 ))). toFixed ( 2 )),
1194 rect: {
1195 top: Math. round (stats.rect.top + window.scrollY),
1196 left: Math. round (stats.rect.left),
1197 width: Math. round (stats.rect.width),
1198 height: Math. round (stats.rect.height),
1199 },
1200 counts: {
1201 headings: stats.headings. length ,
1202 paragraphs: stats.paragraphCount,
1203 images: stats.imageCount,
1204 links: stats.linkCount,
1205 buttons: stats.buttonCount,
1206 },
1207 domRef: {
1208 tag: stats.tag,
1209 id: stats.node.id || "" ,
1210 classTokens: String (stats.node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 6 ),
1211 path: domPath (stats.node, sectionRoot.parentElement || document.body.parentElement),
1212 parentPath: domPath (stats.node.parentElement, sectionRoot.parentElement || document.body.parentElement),
1213 nthOfType: stats.nthOfType,
1214 fingerprint: stats.fingerprint,
1215 },
1216 warnings: [],
1217 };
1218 }
1219
1220 function moduleCandidatesFrom ( containerRoot , sectionRoot , filters = {}) {
1221 const minHeight = filters.minHeight || 60 ;
1222 const minWidth = filters.minWidth || Math. min (sectionRoot. getBoundingClientRect ().width * 0.22 , 220 );
1223 return visibleChildren (containerRoot)
1224 . map (( child ) => collectModuleStats (child, sectionRoot))
1225 . filter (( stats ) => stats.rect.height >= minHeight)
1226 . filter (( stats ) => stats.rect.width >= minWidth || stats.imageCount >= 1 || stats.headings. length >= 1 )
1227 . filter (( stats ) => stats.textLength >= 10 || stats.imageCount >= 1 )
1228 . slice ( 0 , 36 );
1229 }
1230
1231 function resolveRepeatingContainer ( sectionRoot ) {
1232 let current = sectionRoot;
1233 for ( let depth = 0 ; depth < 3 ; depth += 1 ) {
1234 const children = visibleChildren (current);
1235 if ( ! children. length ) return current;
1236 const repeatedChild = children. find (( child ) => {
1237 const classText = `${ child . className || ""} ${ child . id || ""}` . toLowerCase ();
1238 return /slider | carousel | embla | swiper | viewport | container | grid | gallery | reviews | products | list/ . test (classText);
1239 });
1240 if (repeatedChild) {
1241 current = repeatedChild;
1242 continue ;
1243 }
1244 if (children. length === 1 ) {
1245 current = children[ 0 ];
1246 continue ;
1247 }
1248 return current;
1249 }
1250 return current;
1251 }
1252
1253 function extractRepeatingItemModules ( sectionRoot , sectionType ) {
1254 const container = resolveRepeatingContainer (sectionRoot);
1255 const children = moduleCandidatesFrom (container, sectionRoot, { minHeight: 80 , minWidth: 120 });
1256 const accepted = [];
1257 for ( const stats of children) {
1258 if (stats.rect.height > sectionRoot. getBoundingClientRect ().height * 0.95 && stats.cardLikeChildren <= 1 ) continue ;
1259 if (accepted. some (( existing ) => overlaps (existing, stats))) continue ;
1260 accepted. push (stats);
1261 }
1262 return accepted. slice ( 0 , 12 ). map (( stats , index ) => {
1263 const module = moduleSummary (stats, sectionRoot, sectionType);
1264 module . id = `module-${ index + 1 }` ;
1265 module .order = index + 1 ;
1266 return module ;
1267 }). filter (( module ) => module .kind !== "unknown" );
1268 }
1269
1270 function dedupeModuleStats ( statsList ) {
1271 const accepted = [];
1272 for ( const stats of statsList) {
1273 if (accepted. some (( existing ) => overlaps (existing, stats))) continue ;
1274 accepted. push (stats);
1275 }
1276 return accepted;
1277 }
1278
1279 function extractLogoModules ( sectionRoot , sectionType ) {
1280 const imageNodes = uniqueElements (
1281 Array. from (sectionRoot. querySelectorAll ( "img" ))
1282 . filter (isVisible)
1283 . map (( image ) => image. closest ( "a, figure, li, div" ) || image),
1284 );
1285 const statsList = imageNodes
1286 . map (( node ) => collectModuleStats (node, sectionRoot))
1287 . filter (( stats ) => stats.imageCount >= 1 )
1288 . filter (( stats ) => stats.rect.width >= 40 && stats.rect.height >= 24 )
1289 . sort (( a , b ) => (a.rect.width * a.rect.height) - (b.rect.width * b.rect.height));
1290 return dedupeModuleStats (statsList). slice ( 0 , 16 ). map (( stats , index ) => {
1291 const module = moduleSummary (stats, sectionRoot, sectionType);
1292 module . id = `module-${ index + 1 }` ;
1293 module .order = index + 1 ;
1294 return module ;
1295 }). filter (( module ) => module .kind !== "unknown" );
1296 }
1297
1298 function extractReviewModules ( sectionRoot , sectionType ) {
1299 const reviewNodes = uniqueElements (
1300 Array. from (sectionRoot. querySelectorAll ( "[class*='review'], [id*='review'], [class*='testimonial'], [class*='customer']" ))
1301 . filter (isVisible),
1302 );
1303 const statsList = reviewNodes
1304 . map (( node ) => collectModuleStats (node, sectionRoot))
1305 . filter (( stats ) => stats.textLength >= 30 || stats.imageCount >= 1 )
1306 . filter (( stats ) => stats.rect.height >= 60 )
1307 . sort (( a , b ) => (a.rect.width * a.rect.height) - (b.rect.width * b.rect.height));
1308 const accepted = dedupeModuleStats (statsList)
1309 . filter (( stats ) => ! stats.node. contains (sectionRoot))
1310 . slice ( 0 , 12 );
1311 return accepted. map (( stats , index ) => {
1312 const module = moduleSummary (stats, sectionRoot, sectionType);
1313 module . id = `module-${ index + 1 }` ;
1314 module .order = index + 1 ;
1315 return module ;
1316 }). filter (( module ) => module .kind !== "unknown" );
1317 }
1318
1319 function resolveTextMediaScope ( sectionRoot ) {
1320 let current = sectionRoot;
1321 for ( let depth = 0 ; depth < 3 ; depth += 1 ) {
1322 const children = visibleChildren (current). filter (( child ) => child. getBoundingClientRect ().height >= 24 );
1323 if (children. length !== 1 ) return current;
1324 const onlyChild = children[ 0 ];
1325 const onlyRect = onlyChild. getBoundingClientRect ();
1326 if (onlyRect.width < current. getBoundingClientRect ().width * 0.72 ) return current;
1327 current = onlyChild;
1328 }
1329 return current;
1330 }
1331
1332 function extractTextMediaModules ( sectionRoot , sectionType ) {
1333 const scope = resolveTextMediaScope (sectionRoot);
1334 const children = moduleCandidatesFrom (scope, sectionRoot, { minHeight: 90 , minWidth: 120 })
1335 . sort (( a , b ) => (a.rect.width * a.rect.height) - (b.rect.width * b.rect.height));
1336 const accepted = [];
1337 for ( const stats of children) {
1338 const moduleType = classifyModule (stats, sectionType);
1339 if (moduleType.kind === "unknown" ) continue ;
1340 if (accepted. some (( existing ) => existing.node. contains (stats.node) && classifyModule (existing, sectionType).kind === moduleType.kind)) continue ;
1341 if (accepted. some (( existing ) => overlaps (existing, stats))) continue ;
1342 accepted. push (stats);
1343 }
1344 return accepted. slice ( 0 , 6 ). map (( stats , index ) => {
1345 const module = moduleSummary (stats, sectionRoot, sectionType);
1346 module . id = `module-${ index + 1 }` ;
1347 module .order = index + 1 ;
1348 return module ;
1349 });
1350 }
1351
1352 function extractStatModules ( sectionRoot , sectionType ) {
1353 const children = visibleChildren (sectionRoot)
1354 . map (( child ) => collectModuleStats (child, sectionRoot))
1355 . filter (( stats ) => stats.headings. length >= 1 || stats.textLength >= 20 )
1356 . filter (( stats ) => stats.rect.height >= 40 )
1357 . slice ( 0 , 12 );
1358 return children. map (( stats , index ) => {
1359 const module = moduleSummary (stats, sectionRoot, sectionType);
1360 module . id = `module-${ index + 1 }` ;
1361 module .order = index + 1 ;
1362 return module ;
1363 }). filter (( module ) => module .kind !== "unknown" );
1364 }
1365
1366 function extractGenericModules ( sectionRoot , sectionType ) {
1367 const children = visibleChildren (sectionRoot)
1368 . map (( child ) => collectModuleStats (child, sectionRoot))
1369 . filter (( stats ) => stats.rect.width >= Math. min (sectionRoot. getBoundingClientRect ().width * 0.22 , 220 ) || stats.imageCount >= 1 )
1370 . filter (( stats ) => stats.rect.height >= 90 )
1371 . filter (( stats ) => stats.textLength >= 10 || stats.imageCount >= 1 )
1372 . slice ( 0 , 16 );
1373 const accepted = [];
1374 for ( const stats of children) {
1375 if (stats.rect.height > sectionRoot. getBoundingClientRect ().height * 0.92 && stats.cardLikeChildren <= 1 ) continue ;
1376 if (accepted. some (( existing ) => overlaps (existing, stats))) continue ;
1377 accepted. push (stats);
1378 }
1379 return accepted. slice ( 0 , 8 ). map (( stats , index ) => {
1380 const module = moduleSummary (stats, sectionRoot, sectionType);
1381 module . id = `module-${ index + 1 }` ;
1382 module .order = index + 1 ;
1383 return module ;
1384 }). filter (( module ) => module .kind !== "unknown" );
1385 }
1386
1387 const MODULE_EXTRACTORS = {
1388 "reviews:*" : extractReviewModules,
1389 "card-collection:logo-gallery" : extractLogoModules,
1390 "card-collection:*" : extractRepeatingItemModules,
1391 "text-media:*" : extractTextMediaModules,
1392 "hero:*" : extractTextMediaModules,
1393 "stat-group:*" : extractStatModules,
1394 "footer:*" : extractStatModules,
1395 "category-strip:*" : extractStatModules,
1396 "cta-strip:*" : extractStatModules,
1397 "promo-band:*" : extractStatModules,
1398 };
1399
1400 function extractSectionModules ( sectionRoot , sectionType ) {
1401 const exact = MODULE_EXTRACTORS [ `${ sectionType . kind }:${ sectionType . variant }` ];
1402 const family = MODULE_EXTRACTORS [ `${ sectionType . kind }:*` ];
1403 const extractor = exact || family || extractGenericModules;
1404 return extractor (sectionRoot, sectionType);
1405 }
1406
1407 function extractSections () {
1408 const contentRootChoice = findContentRoot ();
1409 const contentRoot = contentRootChoice.node || document.body;
1410 const segmentationRootChoice = findSegmentationRoot (contentRoot);
1411 const segmentationRoot = segmentationRootChoice.node || contentRoot;
1412 const diagnostics = {
1413 source: "dom-layout+a11y" ,
1414 contentRoot: contentRootChoice.label || "body" ,
1415 segmentationRootPath: domPath (segmentationRoot, contentRoot.parentElement || document.body.parentElement),
1416 accessibilitySnapshotAvailable: Boolean (repeaterPlan?.accessibilitySnapshotAvailable),
1417 accessibilityRoot: repeaterPlan?.accessibilityRoot || null ,
1418 candidateCount: 0 ,
1419 acceptedSectionCount: 0 ,
1420 acceptedModuleCount: 0 ,
1421 warnings: [],
1422 };
1423 if (contentRootChoice.label !== "main" && contentRootChoice.label !== "role-main" ) {
1424 diagnostics.warnings. push ( `content-root-fallback-used:${ contentRootChoice . label }` );
1425 }
1426 diagnostics.warnings. push ( ... segmentationRootChoice.warnings);
1427 const primaryChildren = visibleChildren (segmentationRoot)
1428 . map (( node ) => collectModuleStats (node, segmentationRoot))
1429 . filter (( stats ) => stats.rect.height >= 24 )
1430 . filter (( stats ) => stats.textLength >= 10 || stats.imageCount >= 1 || stats.buttonCount >= 1 || stats.linkCount >= 1 );
1431 const primaryBandStats = primaryChildren
1432 . filter (( stats ) => isStructuralBand (stats, segmentationRoot))
1433 . map (( stats ) => ({
1434 ... stats,
1435 score: sectionScore (stats, segmentationRoot) + 22 ,
1436 isDirectChild: true ,
1437 }))
1438 . filter (( stats ) => stats.score >= 30 )
1439 . filter (( stats ) => stats.rect.height >= 40 )
1440 . sort (( a , b ) => a.rect.top - b.rect.top);
1441 const candidateNodes = uniqueElements ([
1442 ... primaryBandStats. map (( stats ) => stats.node),
1443 ... Array. from (segmentationRoot. querySelectorAll ( "section, article, aside, [role='region'], [data-testid], div" ))
1444 . filter (isVisible)
1445 . slice ( 0 , 260 ),
1446 ]). filter (( node ) => node !== contentRoot && node !== segmentationRoot);
1447 const candidateStats = candidateNodes
1448 . map (( node ) => collectModuleStats (node, segmentationRoot))
1449 . filter (( stats ) => stats.rect.width >= Math. min (segmentationRoot. getBoundingClientRect ().width * 0.45 , 520 ))
1450 . filter (( stats ) => stats.rect.height >= 80 )
1451 . filter (( stats ) => stats.textLength >= 20 || stats.imageCount >= 1 )
1452 . map (( stats ) => {
1453 let score = sectionScore (stats, segmentationRoot);
1454 if (stats.node.parentElement === segmentationRoot) score += 18 ;
1455 if ( isStructuralBand (stats, segmentationRoot)) score += 8 ;
1456 return { ... stats, score, isDirectChild: stats.node.parentElement === segmentationRoot };
1457 })
1458 . sort (( a , b ) => a.rect.top - b.rect.top || b.score - a.score);
1459 diagnostics.candidateCount = candidateStats. length ;
1460 const accepted = [ ... primaryBandStats];
1461 for ( const stats of candidateStats) {
1462 if (stats.score < 35 ) continue ;
1463 if ( isWrapperLike (stats, segmentationRoot)) continue ;
1464 const duplicate = accepted. find (( existing ) => overlaps (existing, stats));
1465 if ( ! duplicate) {
1466 accepted. push (stats);
1467 continue ;
1468 }
1469 if (duplicate.isDirectChild && ! stats.isDirectChild && duplicate.score >= stats.score - 15 ) continue ;
1470 if (stats.isDirectChild && ! duplicate.isDirectChild && stats.score >= duplicate.score - 15 ) {
1471 accepted. splice (accepted. indexOf (duplicate), 1 , stats);
1472 continue ;
1473 }
1474 const statsContainsDuplicate = stats.node. contains (duplicate.node);
1475 const duplicateContainsStats = duplicate.node. contains (stats.node);
1476 if ( isWrapperLike (duplicate, segmentationRoot) && duplicateContainsStats) {
1477 accepted. splice (accepted. indexOf (duplicate), 1 , stats);
1478 } else if ( isWrapperLike (stats, segmentationRoot) && statsContainsDuplicate) {
1479 diagnostics.warnings. push ( `nested-candidates-collapsed:${ stats . path }` );
1480 continue ;
1481 } else if (duplicateContainsStats && duplicate.rect.height > stats.rect.height * 1.5 && stats.score >= duplicate.score - 8 ) {
1482 accepted. splice (accepted. indexOf (duplicate), 1 , stats);
1483 } else if (statsContainsDuplicate && stats.rect.height > duplicate.rect.height * 1.5 ) {
1484 diagnostics.warnings. push ( `nested-candidates-collapsed:${ stats . path }` );
1485 continue ;
1486 } else if (stats.score > duplicate.score) {
1487 accepted. splice (accepted. indexOf (duplicate), 1 , stats);
1488 }
1489 }
1490 if ( ! accepted. length ) {
1491 diagnostics.warnings. push ( "no-strong-section-boundaries" );
1492 }
1493 const sections = accepted. slice ( 0 , 18 ). map (( stats , index ) => {
1494 const sectionRoot = stats.node;
1495 const sectionType = classifySection (stats);
1496 const modules = extractSectionModules (sectionRoot, sectionType). map (( module , moduleIndex ) => ({
1497 ... module ,
1498 id: `section-${ String ( index + 1 ). padStart ( 3 , "0" ) }-module-${ String ( moduleIndex + 1 ). padStart ( 3 , "0" ) }` ,
1499 order: moduleIndex + 1 ,
1500 }));
1501 diagnostics.acceptedModuleCount += modules. length ;
1502 return {
1503 id: `section-${ String ( index + 1 ). padStart ( 3 , "0" ) }` ,
1504 kind: sectionType.kind,
1505 variant: sectionType.variant,
1506 capabilities: sectionType.capabilities,
1507 source: "dom-layout+a11y" ,
1508 tag: stats.tag,
1509 idAttr: sectionRoot.id || "" ,
1510 className: sectionRoot.className?. toString ?.() || "" ,
1511 heading: stats.heading,
1512 text: stats.text,
1513 visibleCodeTexts: stats.visibleCodeTexts,
1514 order: index + 1 ,
1515 confidence: Number (Math. max ( 0.3 , Math. min ( 0.98 , stats.score / 100 )). toFixed ( 2 )),
1516 rect: {
1517 x: Math. round (stats.rect.x),
1518 y: Math. round (stats.rect.y),
1519 width: Math. round (stats.rect.width),
1520 height: Math. round (stats.rect.height),
1521 top: Math. round (stats.rect.top + window.scrollY),
1522 left: Math. round (stats.rect.left),
1523 },
1524 layoutHints: {
1525 fullWidth: stats.rect.width >= segmentationRoot. getBoundingClientRect ().width * 0.85 ,
1526 columnCount: Math. max ( 1 , Math. min (stats.rowCountEstimate > 1 ? Math. ceil (stats.cardLikeChildren / stats.rowCountEstimate) : stats.cardLikeChildren, 4 )) || 1 ,
1527 hasGrid: stats.cardLikeChildren >= 3 && stats.rowCountEstimate >= 1 ,
1528 hasRepeatingChildren: stats.cardLikeChildren >= 3 ,
1529 },
1530 counts: {
1531 headings: stats.headings. length ,
1532 paragraphs: stats.paragraphCount,
1533 images: stats.imageCount,
1534 links: stats.linkCount,
1535 buttons: stats.buttonCount,
1536 },
1537 background: stats.background,
1538 layoutEvidence: collectLayoutEvidence (sectionRoot),
1539 domRef: {
1540 tag: stats.tag,
1541 id: sectionRoot.id || "" ,
1542 classTokens: String (sectionRoot.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 6 ),
1543 path: stats.path,
1544 parentPath: stats.parentPath,
1545 nthOfType: stats.nthOfType,
1546 fingerprint: stats.fingerprint,
1547 },
1548 a11y: {
1549 role: stats.role || null ,
1550 headingLevels: stats.headings. map (( heading ) => heading.level). filter (Boolean),
1551 containsList: sectionRoot. querySelectorAll ( "ul,ol,[role='list']" ). length > 0 ,
1552 },
1553 modules,
1554 warnings: [],
1555 };
1556 });
1557 diagnostics.acceptedSectionCount = sections. length ;
1558 return { sections, diagnostics };
1559 }
1560
1561 const sectionResult = extractSections ();
1562 const footer = (() => {
1563 const node = Array. from (document. querySelectorAll ( "footer, .site-footer, [role='contentinfo']" )). find (isVisible);
1564 if ( ! node) return null ;
1565 const links = Array. from (node. querySelectorAll ( "a[href]" )). filter (isVisible). map (linkRecord);
1566 const cleanNodeText = ( source ) => renderedText (source);
1567 const text = cleanNodeText (node). slice ( 0 , 6000 );
1568 const legalText = Array. from (node. querySelectorAll ( "p, small, [class*='legal' i], [class*='disclaimer' i]" ))
1569 . filter (isVisible)
1570 . map (cleanNodeText)
1571 . filter (( value ) => value. length >= 120 || /disclaimer | portfolio companies | copyright | ©/ i . test (value))
1572 . filter (( value , index , values ) => values. indexOf (value) === index)
1573 . join ( " \n\n " ) || text;
1574 return {
1575 text,
1576 legalText: legalText. slice ( 0 , 6000 ),
1577 links: dedupeNav (links). slice ( 0 , 80 ),
1578 domRef: {
1579 tag: node.tagName. toLowerCase (),
1580 id: node.id || "" ,
1581 classTokens: String (node.className || "" ). split ( / \s + / ). filter (Boolean). slice ( 0 , 8 ),
1582 path: domPath (node, document.body.parentElement),
1583 },
1584 };
1585 })();
1586 const computedTokens = Array. from (document. querySelectorAll ( "body, h1, h2, h3, p, a, button" )). slice ( 0 , 80 ). map (( node ) => {
1587 const styles = getComputedStyle (node);
1588 return {
1589 tag: node.tagName. toLowerCase (),
1590 text: renderedText (node). slice ( 0 , 120 ) || "" ,
1591 color: styles.color,
1592 backgroundColor: styles.backgroundColor,
1593 fontFamily: styles.fontFamily,
1594 fontSize: styles.fontSize,
1595 fontWeight: styles.fontWeight,
1596 borderRadius: styles.borderRadius,
1597 boxShadow: styles.boxShadow,
1598 };
1599 });
1600 const direction = document.documentElement.dir || document.body.dir || getComputedStyle (document.body).direction || "ltr" ;
1601 function nextVisibleNonHeadingSibling ( node ) {
1602 let current = node?.nextElementSibling || null ;
1603 while (current) {
1604 if ( isVisible (current) && ! / ^ H [1-6] $ / . test (current.tagName)) return current;
1605 current = current.nextElementSibling;
1606 }
1607 return null ;
1608 }
1609
1610 function findContentContainerForHeading ( heading ) {
1611 const direct = nextVisibleNonHeadingSibling (heading);
1612 if (direct) return direct;
1613 const parentDirect = nextVisibleNonHeadingSibling (heading.parentElement);
1614 if (parentDirect) return parentDirect;
1615 return null ;
1616 }
1617
1618 function nearestCommonAncestor ( a , b , limit ) {
1619 let current = a || null ;
1620 while (current) {
1621 if (current. contains (b)) return current;
1622 if (limit && current === limit) break ;
1623 current = current.parentElement;
1624 }
1625 return null ;
1626 }
1627
1628 function uniqueElements ( nodes ) {
1629 const seen = new Set ();
1630 const result = [];
1631 for ( const node of nodes || []) {
1632 if ( ! node || seen. has (node)) continue ;
1633 seen. add (node);
1634 result. push (node);
1635 }
1636 return result;
1637 }
1638
1639 function queryScopeNodes ( scopeNodes , selector ) {
1640 const seen = new Set ();
1641 const matches = [];
1642 for ( const node of scopeNodes || []) {
1643 if (node?. matches ?.(selector) && ! seen. has (node)) {
1644 seen. add (node);
1645 matches. push (node);
1646 }
1647 for ( const child of node?. querySelectorAll ?.(selector) || []) {
1648 if (seen. has (child)) continue ;
1649 seen. add (child);
1650 matches. push (child);
1651 }
1652 }
1653 return matches;
1654 }
1655
1656 function collectContentNodes ( content , nextHeadingNode ) {
1657 if ( ! content) return [];
1658 const nodes = [];
1659 if (nextHeadingNode && content.parentElement && nextHeadingNode.parentElement === content.parentElement) {
1660 let current = content;
1661 while (current && current !== nextHeadingNode) {
1662 if ( isVisible (current)) nodes. push (current);
1663 current = current.nextElementSibling;
1664 }
1665 }
1666 if ( ! nodes. length ) nodes. push (content);
1667 return uniqueElements (nodes);
1668 }
1669
1670 function nodeSignature ( node ) {
1671 if ( ! node) return "" ;
1672 const tag = node.tagName. toLowerCase ();
1673 const classTokens = String (node.className || "" )
1674 . split ( / \s + / )
1675 . filter (Boolean)
1676 . slice ( 0 , 4 )
1677 . sort ()
1678 . join ( "." );
1679 const childTags = Array. from (node.children). slice ( 0 , 8 ). map (( child ) => child.tagName. toLowerCase ()). join ( "," );
1680 return `${ tag }:${ classTokens }:${ childTags }` ;
1681 }
1682
1683 function normalized ( value ) {
1684 return cleanText (value). toLowerCase ();
1685 }
1686
1687 function extractParagraphs ( scopeNodes , heading ) {
1688 const seen = new Set ();
1689 const paragraphs = [];
1690 const candidates = queryScopeNodes (scopeNodes, "p, li, blockquote, figcaption" ). filter (isVisible);
1691 for ( const node of candidates) {
1692 const text = renderedText (node);
1693 if ( ! text || text. length < 20 ) continue ;
1694 const key = normalized (text);
1695 if (seen. has (key)) continue ;
1696 seen. add (key);
1697 paragraphs. push (text);
1698 }
1699 if ( ! paragraphs. length ) {
1700 const fallbackBlocks = queryScopeNodes (scopeNodes, "div, span" ). filter (( node ) => {
1701 if ( ! isVisible (node) || node === heading || heading?. contains (node)) return false ;
1702 if (node. querySelector ( "p, li, blockquote, figcaption" )) return false ;
1703 if (node. querySelector ( "img" )) return false ;
1704 const text = renderedText (node);
1705 if ( ! text || text. length < 30 ) return false ;
1706 if (Array. from (node.children). some (( child ) => renderedText (child) === text)) return false ;
1707 const display = getComputedStyle (node).display;
1708 return /block | flex | grid | inline-block | table/ . test (display);
1709 });
1710 for ( const node of fallbackBlocks) {
1711 const text = renderedText (node);
1712 const key = normalized (text);
1713 if (seen. has (key)) continue ;
1714 seen. add (key);
1715 paragraphs. push (text);
1716 }
1717 }
1718 if (paragraphs. length ) return paragraphs;
1719 const fallback = cleanText (scopeNodes. map (( node ) => renderedText (node)). join ( " " ));
1720 return fallback ? [fallback] : [];
1721 }
1722
1723 function extractImages ( scopeNodes ) {
1724 function bestSrcFromSet ( value ) {
1725 const candidates = String (value || "" )
1726 . split ( "," )
1727 . map (( item ) => item. trim ())
1728 . map (( item ) => {
1729 const match = item. match ( / ^ ( \S + )(?: \s + ( \d + )w) ? / );
1730 return match ? { url: match[ 1 ], width: Number (match[ 2 ] || 0 ) } : null ;
1731 })
1732 . filter (Boolean)
1733 . sort (( a , b ) => b.width - a.width);
1734 return candidates[ 0 ]?.url || "" ;
1735 }
1736
1737 function bestImageSrc ( img ) {
1738 return (
1739 img. getAttribute ( "data-src" )
1740 || img. getAttribute ( "data-lazy-src" )
1741 || bestSrcFromSet (img. getAttribute ( "data-srcset" ))
1742 || bestSrcFromSet (img. getAttribute ( "srcset" ))
1743 || img.currentSrc
1744 || img.src
1745 || ""
1746 );
1747 }
1748
1749 const seen = new Set ();
1750 return queryScopeNodes (scopeNodes, "img" )
1751 . filter (( img ) => {
1752 const rect = img. getBoundingClientRect ();
1753 return ( isVisible (img) || Boolean ( bestImageSrc (img))) && rect.width >= 24 && rect.height >= 24 ;
1754 })
1755 . map (( img ) => {
1756 const rect = img. getBoundingClientRect ();
1757 return {
1758 src: bestImageSrc (img),
1759 alt: cleanText (img. getAttribute ( "alt" ) || "" ),
1760 width: Math. round (rect.width),
1761 height: Math. round (rect.height),
1762 rect: {
1763 left: rect.left,
1764 top: rect.top,
1765 width: rect.width,
1766 height: rect.height,
1767 },
1768 };
1769 })
1770 . filter (( image ) => image.src && ! image.src. startsWith ( "data:image/svg+xml" ))
1771 . filter (( image ) => {
1772 if (seen. has (image.src)) return false ;
1773 seen. add (image.src);
1774 return true ;
1775 });
1776 }
1777
1778 function extractLinksFromContainer ( scopeNodes ) {
1779 const seen = new Set ();
1780 return queryScopeNodes (scopeNodes, "a[href]" )
1781 . map (( anchor ) => ({
1782 href: anchor.href || anchor. getAttribute ( "href" ) || "" ,
1783 label: semanticText (anchor). slice ( 0 , 160 ),
1784 }))
1785 . filter (( link ) => link.href)
1786 . filter (( link ) => {
1787 const key = `${ link . label } \u0000 ${ link . href }` ;
1788 if (seen. has (key)) return false ;
1789 seen. add (key);
1790 return true ;
1791 })
1792 . slice ( 0 , 24 );
1793 }
1794
1795 function inferItemLayoutHints ( heading , scopeNodes , images , paragraphs ) {
1796 const descendants = queryScopeNodes (scopeNodes, "img, p, li, blockquote, figcaption, div, span" ). filter (( node ) => {
1797 const text = renderedText (node);
1798 if (node.tagName === "IMG" ) return true ;
1799 return text. length >= 20 && isVisible (node);
1800 });
1801 const firstKind = descendants[ 0 ]?.tagName === "IMG" ? "media-first" : descendants[ 0 ] ? "text-first" : "unknown" ;
1802 const textNodes = queryScopeNodes (scopeNodes, "p, li, blockquote, figcaption, div, span" ). filter (( node ) => {
1803 if ( ! isVisible (node) || node === heading || heading?. contains (node)) return false ;
1804 const text = renderedText (node);
1805 if ( ! text || text. length < 20 ) return false ;
1806 if (Array. from (node.children). some (( child ) => renderedText (child) === text)) return false ;
1807 return true ;
1808 });
1809 const averageCenter = ( nodes ) => {
1810 if ( ! nodes. length ) return null ;
1811 const centers = nodes. map (( node ) => {
1812 const rect = node. getBoundingClientRect ();
1813 return rect.left + (rect.width / 2 );
1814 });
1815 return centers. reduce (( sum , value ) => sum + value, 0 ) / centers. length ;
1816 };
1817 const textCenter = averageCenter (textNodes);
1818 const imageCenter = images. length
1819 ? images. reduce (( sum , image ) => sum + (image.rect.left + (image.rect.width / 2 )), 0 ) / images. length
1820 : null ;
1821 let textPosition = "stacked" ;
1822 let mediaPosition = "stacked" ;
1823 if (textCenter != null && imageCenter != null ) {
1824 const delta = imageCenter - textCenter;
1825 const scopeWidth = Math. max ( ... scopeNodes. map (( node ) => node. getBoundingClientRect ().width), 0 );
1826 const threshold = Math. max (scopeWidth * 0.12 , 60 );
1827 if (Math. abs (delta) > threshold) {
1828 mediaPosition = delta < 0 ? "left" : "right" ;
1829 textPosition = delta < 0 ? "right" : "left" ;
1830 }
1831 }
1832 return {
1833 mediaCount: images. length ,
1834 paragraphCount: paragraphs. length ,
1835 firstContentKind: firstKind,
1836 headingLevel: Number (heading.tagName. replace ( / ^ H/ i , "" )) || null ,
1837 textPosition,
1838 mediaPosition,
1839 };
1840 }
1841
1842 function scoreItemContainer ( node , { heading , content , main }) {
1843 if ( ! node || ! main. contains (node)) return Number.NEGATIVE_INFINITY;
1844 if ( ! node. contains (heading) || ! node. contains (content)) return Number.NEGATIVE_INFINITY;
1845 const rect = node. getBoundingClientRect ();
1846 if (rect.width < 180 || rect.height < 80 ) return Number.NEGATIVE_INFINITY;
1847 const classText = `${ node . className || ""} ${ node . id || ""}` . toLowerCase ();
1848 const imageCount = node. querySelectorAll ( "img" ). length ;
1849 const paragraphCount = node. querySelectorAll ( "p, li, blockquote, figcaption" ). length ;
1850 const headingCount = node. querySelectorAll ( "h1,h2,h3,h4,h5,h6" ). length ;
1851 let score = 0 ;
1852 score += 40 ;
1853 score += Math. min (imageCount, 6 ) * 6 ;
1854 score += Math. min (paragraphCount, 6 ) * 5 ;
1855 if (headingCount === 1 ) score += 18 ;
1856 else score -= Math. max ( 0 , headingCount - 1 ) * 12 ;
1857 if (node === content) score -= 10 ;
1858 if ( /slider | carousel | gallery | nav | menu | footer | share/ . test (classText)) score -= 35 ;
1859 if (rect.height > window.innerHeight * 2.5 ) score -= 25 ;
1860 if (rect.width < Math. min (window.innerWidth * 0.45 , 320 )) score -= 20 ;
1861 return score;
1862 }
1863
1864 function findItemContainerForHeading ( heading , main ) {
1865 const content = findContentContainerForHeading (heading);
1866 if ( ! content) return null ;
1867 const common = nearestCommonAncestor (heading, content, main) || nearestCommonAncestor (heading, content, document.body);
1868 const candidates = uniqueElements ([
1869 common,
1870 content,
1871 content.parentElement,
1872 heading.parentElement,
1873 heading.parentElement?.parentElement,
1874 heading. closest ( "section,article,li,[data-testid],[class]" ),
1875 content. closest ( "section,article,li,[data-testid],[class]" ),
1876 ]). filter (( node ) => node && main. contains (node));
1877 const scored = candidates
1878 . map (( node ) => ({ node, score: scoreItemContainer (node, { heading, content, main }) }))
1879 . filter (( entry ) => Number. isFinite (entry.score))
1880 . sort (( a , b ) => b.score - a.score);
1881 const best = scored[ 0 ];
1882 if ( ! best) return null ;
1883 return { container: best.node, content, score: best.score, signature: nodeSignature (best.node) };
1884 }
1885
1886 function dominantParentGroup ( matches ) {
1887 const groups = [];
1888 for ( const match of matches) {
1889 const parent = match.container.parentElement;
1890 if ( ! parent) continue ;
1891 const existing = groups. find (( group ) => group.parent === parent);
1892 if (existing) existing.items. push (match);
1893 else groups. push ({ parent, items: [match] });
1894 }
1895 groups. sort (( a , b ) => b.items. length - a.items. length );
1896 return groups[ 0 ] || null ;
1897 }
1898
1899 function extractRepeaters ( plan ) {
1900 const signals = Array. isArray (plan?.signals) ? plan.signals : [];
1901 const main = document. querySelector ( "main" );
1902 const diagnostics = {
1903 source: "accessibility+dom" ,
1904 accessibilitySnapshotAvailable: Boolean (plan?.accessibilitySnapshotAvailable),
1905 signalCount: signals. length ,
1906 candidateCount: plan?.candidateCount || signals. length ,
1907 acceptedRepeaterCount: 0 ,
1908 signals: [],
1909 warnings: [ ... (plan?.warnings || [])],
1910 };
1911 if ( ! main) {
1912 diagnostics.warnings. push ( "main-content-missing: manual repeater review required" );
1913 return { repeaters: [], diagnostics };
1914 }
1915 const headingNodes = Array. from (main. querySelectorAll ( "h1,h2,h3,h4,h5,h6" )). filter (( node ) => renderedText (node));
1916 const headingRegistry = headingNodes. map (( node , index ) => ({
1917 node,
1918 index,
1919 text: renderedText (node),
1920 normalized: normalized ( renderedText (node)),
1921 level: Number (node.tagName. replace ( / ^ H/ i , "" )) || 0 ,
1922 }));
1923 const repeaters = signals. map (( signal , signalIndex ) => {
1924 let minIndex = - 1 ;
1925 const matched = [];
1926 for ( const expected of signal.items || []) {
1927 const expectedText = normalized (expected.heading);
1928 const match = headingRegistry. find (( entry ) => entry.index > minIndex && entry.level === signal.headingLevel && entry.normalized === expectedText);
1929 if ( ! match) continue ;
1930 minIndex = match.index;
1931 const resolved = findItemContainerForHeading (match.node, main);
1932 if ( ! resolved) continue ;
1933 matched. push ({ ... resolved, heading: match });
1934 }
1935 const dominantParent = dominantParentGroup (matched);
1936 const resolvedItems = dominantParent && dominantParent.items. length >= 3 ? dominantParent.items : matched;
1937 const items = resolvedItems. map (( resolved , resolvedIndex ) => {
1938 const nextHeadingNode = resolvedItems[resolvedIndex + 1 ]?.heading?.node || null ;
1939 const scopeNodes = collectContentNodes (resolved.content, nextHeadingNode);
1940 const paragraphs = extractParagraphs (scopeNodes, resolved.heading.node);
1941 const images = extractImages (scopeNodes);
1942 return {
1943 heading: resolved.heading.text,
1944 paragraphs,
1945 images,
1946 links: extractLinksFromContainer (scopeNodes),
1947 layoutHints: inferItemLayoutHints (resolved.heading.node, scopeNodes, images, paragraphs),
1948 };
1949 }). filter (( item ) => item.paragraphs. length && item.images. length );
1950 const signalDiagnostics = {
1951 label: signal.label || `main-repeater-${ signalIndex + 1 }` ,
1952 roleSignature: signal.roleSignature || null ,
1953 headingLevel: signal.headingLevel || null ,
1954 expectedItemCount: signal.items?. length || 0 ,
1955 matchedHeadingCount: matched. length ,
1956 resolvedItemCount: items. length ,
1957 dominantParentTag: dominantParent?.parent?.tagName?. toLowerCase () || null ,
1958 dominantParentClass: dominantParent?.parent?.className?. toString ?.() || "" ,
1959 containerSignatures: Array. from ( new Set (resolvedItems. map (( item ) => item.signature))). slice ( 0 , 6 ),
1960 warnings: [],
1961 };
1962 diagnostics.signals. push (signalDiagnostics);
1963 if (items. length < 3 ) {
1964 signalDiagnostics.warnings. push ( "dom-resolution-ambiguous" );
1965 diagnostics.warnings. push ( `repeater-resolution-ambiguous:${ signalDiagnostics . label }` );
1966 return null ;
1967 }
1968 const mediaCounts = items. map (( item ) => item.images. length );
1969 const repeater = {
1970 kind: "content-repeater" ,
1971 source: "accessibility+dom" ,
1972 label: signal.label || `main-repeater-${ signalIndex + 1 }` ,
1973 itemCount: items. length ,
1974 schema: {
1975 hasHeading: items. every (( item ) => Boolean (item.heading)),
1976 hasParagraphs: items. every (( item ) => item.paragraphs. length > 0 ),
1977 hasImages: items. every (( item ) => item.images. length > 0 ),
1978 maxImagesPerItem: Math. max ( ... mediaCounts),
1979 },
1980 layoutHints: {
1981 alternating: new Set (items. map (( item ) => item.layoutHints.textPosition || item.layoutHints.firstContentKind)).size > 1 ,
1982 },
1983 items,
1984 };
1985 diagnostics.acceptedRepeaterCount += 1 ;
1986 return repeater;
1987 }). filter (Boolean);
1988 if ( ! signals. length ) {
1989 const mainHeadingCount = main. querySelectorAll ( "h2,h3,h4,h5,h6" ). length ;
1990 const mainImageCount = main. querySelectorAll ( "img" ). length ;
1991 if (mainHeadingCount >= 4 && mainImageCount >= 4 ) {
1992 diagnostics.warnings. push ( "possible-repeated-content-without-high-confidence-structure: manual review required" );
1993 }
1994 } else if ( ! repeaters. length ) {
1995 diagnostics.warnings. push ( "accessibility-signals-found-but-no-structured-repeaters-resolved: manual review required" );
1996 }
1997 return { repeaters, diagnostics };
1998 }
1999
2000 function extractAccordionRepeaters ( main ) {
2001 if ( ! main) return [];
2002 const groups = [];
2003 const candidates = Array. from (main. querySelectorAll ( "details" )). filter (( node ) => renderedText (node. querySelector ( "summary" )));
2004 for ( const node of candidates) {
2005 const parent = node.parentElement;
2006 if ( ! parent) continue ;
2007 const existing = groups. find (( group ) => group.parent === parent);
2008 if (existing) existing.items. push (node);
2009 else groups. push ({ parent, items: [node] });
2010 }
2011 const nativeRecords = groups. filter (( group ) => group.items. length >= 2 ). map (( group , groupIndex ) => {
2012 const items = group.items. map (( node ) => {
2013 const summary = node. querySelector ( "summary" );
2014 const clone = node. cloneNode ( true );
2015 clone. querySelector ( "summary" )?. remove ();
2016 const paragraphs = Array. from (clone. querySelectorAll ( "p,li" )). map (( child ) => semanticText (child)). filter (Boolean);
2017 const fallbackText = semanticText (clone);
2018 const images = Array. from (clone. querySelectorAll ( "img" )). map (( image ) => ({ src: image.currentSrc || image.src || "" , alt: image.alt || "" })). filter (( image ) => image.src && ! /(emoji | twemoji | wp-smiley)/ i . test (image.src));
2019 const links = Array. from (clone. querySelectorAll ( "a[href]" )). map (( link ) => ({ text: semanticText (link), url: link.href })). filter (( link ) => link.url);
2020 const content = paragraphs. length ? paragraphs : fallbackText ? [fallbackText] : [];
2021 return {
2022 heading: renderedText (summary),
2023 paragraphs: content,
2024 images,
2025 links,
2026 initialExpanded: Boolean (node.open),
2027 itemStructure: images. length ? content. length ? "media-and-text" : "media-only" : "text-only" ,
2028 layoutHints: { mediaCount: images. length , paragraphCount: content. length , firstContentKind: images. length ? "media-first" : "text-first" , headingLevel: null },
2029 };
2030 });
2031 const mediaCounts = items. map (( item ) => item.images. length );
2032 const expanded = items. map (( item , index ) => item.initialExpanded ? index : null ). filter (( index ) => index !== null );
2033 return {
2034 kind: "accordion-repeater" ,
2035 source: "dom:details" ,
2036 label: cleanText (group.parent. getAttribute ( "aria-label" )) || `accordion-${ groupIndex + 1 }` ,
2037 itemCount: items. length ,
2038 schema: { hasHeading: true , hasParagraphs: items. every (( item ) => item.paragraphs. length > 0 ), hasImages: items. some (( item ) => item.images. length > 0 ), maxImagesPerItem: Math. max ( ... mediaCounts, 0 ) },
2039 state: { initialExpandedIndices: expanded, multipleOpen: expanded. length > 1 , openBehavior: "requires-interaction-probe" },
2040 items,
2041 };
2042 });
2043 const customRoots = Array. from (main. querySelectorAll ( ".tatsu-accordion, [class*='accordion' i]" )). filter (( root , index , all ) =>
2044 root. querySelectorAll ( ".accordion-head" ). length >= 2 && ! all. some (( other ) => other !== root && other. contains (root) && other. querySelectorAll ( ".accordion-head" ). length >= 2 ),
2045 );
2046 const customRecords = customRoots. map (( root , rootIndex ) => {
2047 const heads = Array. from (root. querySelectorAll ( ".accordion-head" )). filter (( head ) => renderedText (head));
2048 const items = heads. map (( head ) => {
2049 const contentRoot = head.nextElementSibling?. matches ?.( ".accordion-content,[role='region']" ) ? head.nextElementSibling : document. getElementById (head. getAttribute ( "aria-controls" ) || "" );
2050 const paragraphs = Array. from (contentRoot?. querySelectorAll ?.( "p,li" ) || []). map (( child ) => semanticText (child)). filter (Boolean);
2051 const fallbackText = contentRoot ? semanticText (contentRoot) : "" ;
2052 const content = paragraphs. length ? paragraphs : fallbackText ? [fallbackText] : [];
2053 const images = Array. from (contentRoot?. querySelectorAll ?.( "img" ) || []). map (( image ) => ({ src: image.currentSrc || image.src || "" , alt: image.alt || "" })). filter (( image ) => image.src && ! /(emoji | twemoji | wp-smiley)/ i . test (image.src));
2054 const links = Array. from (contentRoot?. querySelectorAll ?.( "a[href]" ) || []). map (( link ) => ({ text: semanticText (link), url: link.href })). filter (( link ) => link.url);
2055 const expanded = head. getAttribute ( "aria-expanded" ) === "true" || contentRoot && getComputedStyle (contentRoot).display !== "none" && getComputedStyle (contentRoot).visibility !== "hidden" && contentRoot. getBoundingClientRect ().height > 0 ;
2056 return { heading: renderedText (head), paragraphs: content, images, links, initialExpanded: Boolean (expanded), itemStructure: images. length ? content. length ? "media-and-text" : "media-only" : "text-only" , layoutHints: { mediaCount: images. length , paragraphCount: content. length , firstContentKind: images. length ? "media-first" : "text-first" , headingLevel: Number (head.tagName. slice ( 1 )) || null } };
2057 });
2058 const expanded = items. map (( item , index ) => item.initialExpanded ? index : null ). filter (( index ) => index !== null );
2059 return { kind: "accordion-repeater" , source: "dom:accordion-head" , label: cleanText (root. getAttribute ( "aria-label" )) || `accordion-${ rootIndex + 1 }` , itemCount: items. length , schema: { hasHeading: true , hasParagraphs: items. every (( item ) => item.paragraphs. length > 0 ), hasImages: items. some (( item ) => item.images. length > 0 ), maxImagesPerItem: Math. max ( 0 , ... items. map (( item ) => item.images. length )) }, state: { initialExpandedIndices: expanded, multipleOpen: expanded. length > 1 , openBehavior: root.classList. contains ( "tatsu-accordion" ) ? "single" : "requires-interaction-probe" }, items };
2060 }). filter (( record ) => record.itemCount >= 2 && record.items. every (( item ) => item.paragraphs. length ));
2061 return [ ... nativeRecords, ... customRecords];
2062 }
2063
2064 const repeaterResult = extractRepeaters (repeaterPlan);
2065 const accordionRepeaters = extractAccordionRepeaters (document. querySelector ( "main, [role='main'], #main, #content" ) || document.body);
2066 const visualAssets = collectVisualAssets ();
2067 const visibleLinks = dedupeNav (Array. from (document. querySelectorAll ( "a[href]" )). filter (isVisible). map (linkRecord)). slice ( 0 , 240 )
2068 . map (( link ) => ({ text: link.label, url: link.href }));
2069 return {
2070 sections: sectionResult.sections,
2071 sectionDiagnostics: sectionResult.diagnostics,
2072 footer,
2073 computedTokens,
2074 repeaters: [ ... repeaterResult.repeaters, ... accordionRepeaters],
2075 repeaterDiagnostics: repeaterResult.diagnostics,
2076 ignoredSurfaces,
2077 visualAssets,
2078 links: visibleLinks,
2079 chrome: {
2080 direction,
2081 viewport: { width: window.innerWidth, height: window.innerHeight },
2082 header: {
2083 variants: [initialHeader, scrolledHeader],
2084 maxStickyViewportRatio: 0.16 ,
2085 },
2086 navigation,
2087 heroEvidence: {
2088 visibleText,
2089 forbiddenSeoOnlyText: [],
2090 },
2091 },
2092 };
2093 }, repeaterPlan);
2094 const normalizedDesktopDom = await page. content ();
2095 browserData.chrome.heroEvidence.forbiddenSeoOnlyText = seoOnlyText ({ seo, visibleText: browserData.chrome.heroEvidence.visibleText });
2096 const extractionWarnings = [
2097 ... ((desktopNavigation.warnings || []). map (( warning ) => `navigation: ${ warning }` )),
2098 ... (browserData.sectionDiagnostics?.warnings || []),
2099 ... (browserData.repeaterDiagnostics?.warnings || []),
2100 ];
2101 return {
2102 url,
2103 path: new URL (url).pathname || "/" ,
2104 area: classifyUrl (url),
2105 title: seo.title,
2106 seo,
2107 navigation: desktopNavigation,
2108 sections: browserData.sections. length ? browserData.sections : extractSectionsFromHtml (html),
2109 sectionDiagnostics: browserData.sectionDiagnostics || {
2110 source: "dom-layout+a11y" ,
2111 contentRoot: "unknown" ,
2112 accessibilitySnapshotAvailable: Boolean (repeaterPlan?.accessibilitySnapshotAvailable),
2113 accessibilityRoot: repeaterPlan?.accessibilityRoot || null ,
2114 candidateCount: 0 ,
2115 acceptedSectionCount: 0 ,
2116 acceptedModuleCount: 0 ,
2117 warnings: [ "no-strong-section-boundaries" ],
2118 },
2119 repeaters: browserData.repeaters || [],
2120 repeaterDiagnostics: browserData.repeaterDiagnostics || { source: "accessibility+dom" , signalCount: 0 , acceptedRepeaterCount: 0 , warnings: [] },
2121 footer: browserData.footer || null ,
2122 ignoredSurfaces: browserData.ignoredSurfaces || [],
2123 normalizationActions: (browserData.ignoredSurfaces || []). flatMap (( surface ) => [
2124 {
2125 phase: "before-scroll-sweep" ,
2126 action: "quarantine-obstructive-surface" ,
2127 surfaceId: surface.id,
2128 kind: surface.kind,
2129 reason: surface.reason,
2130 reversible: true ,
2131 },
2132 {
2133 phase: "after-scroll-sweep" ,
2134 action: "verify-surface-remains-quarantined" ,
2135 surfaceId: surface.id,
2136 kind: surface.kind,
2137 reversible: true ,
2138 },
2139 ]),
2140 visualAssets: browserData.visualAssets || [],
2141 links: browserData.links?. length ? browserData.links : extractLinks (html, url),
2142 tokens: { ... inferTokensFromHtml (html), computed: browserData.computedTokens },
2143 chrome: browserData.chrome,
2144 responsiveTextGeometry,
2145 screenshots: screenshotMap,
2146 navigationProfiles,
2147 rawDomSnapshots: htmlByViewport,
2148 normalizedDomSnapshots: { desktop: normalizedDesktopDom },
2149 accessibilityEvidence: {
2150 snapshotAvailable: Boolean (repeaterPlan?.accessibilitySnapshotAvailable),
2151 root: repeaterPlan?.accessibilityRoot || null ,
2152 signalCount: repeaterPlan?.signals?. length || 0 ,
2153 },
2154 extractionWarnings: extractionWarnings. length ? extractionWarnings : undefined ,
2155 extractedAt: new Date (). toISOString (),
2156 };
2157 } finally {
2158 await browser. close ();
2159 }
2160 }
2161
2162 async function captureImportantTextGeometry ( page ) {
2163 return page. evaluate (() => {
2164 const clean = ( value ) => String (value || "" ). replace ( / \s + / g , " " ). trim ();
2165 const visible = ( node ) => {
2166 const rect = node. getBoundingClientRect ();
2167 const style = getComputedStyle (node);
2168 return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && Number (style.opacity || 1 ) > 0 ;
2169 };
2170 const lineCount = ( node ) => {
2171 const tops = [];
2172 const walker = document. createTreeWalker (node, NodeFilter. SHOW_TEXT );
2173 for ( let textNode = walker. nextNode (); textNode; textNode = walker. nextNode ()) {
2174 if ( ! clean (textNode.nodeValue)) continue ;
2175 const range = document. createRange ();
2176 range. selectNodeContents (textNode);
2177 for ( const rect of range. getClientRects ()) {
2178 if (rect.width > 0 && rect.height > 0 && ! tops. some (( top ) => Math. abs (top - rect.top) <= 2 )) tops. push (rect.top);
2179 }
2180 }
2181 return Math. max ( 1 , tops. length );
2182 };
2183 return Array. from (document. querySelectorAll ( "h1,h2,h3" ))
2184 . filter (visible)
2185 . map (( node ) => {
2186 const rect = node. getBoundingClientRect ();
2187 const style = getComputedStyle (node);
2188 const lines = lineCount (node);
2189 return {
2190 text: clean (node.textContent). slice ( 0 , 240 ),
2191 tag: node.tagName. toLowerCase (),
2192 inlineSize: Math. round (rect.width * 100 ) / 100 ,
2193 blockSize: Math. round (rect.height * 100 ) / 100 ,
2194 lineCount: lines,
2195 wrapPolicy: lines === 1 ? "single-line" : style.whiteSpace === "nowrap" ? "clipped-or-overflowing" : "wrapped" ,
2196 fontSize: style.fontSize,
2197 lineHeight: style.lineHeight,
2198 maxWidth: style.maxWidth,
2199 };
2200 })
2201 . filter (( item ) => item.text)
2202 . slice ( 0 , 48 );
2203 });
2204 }
2205
2206 async function extractWithFetch ( url ) {
2207 const html = await fetchText (url);
2208 const seo = extractSeo (html, url);
2209 return {
2210 url,
2211 path: new URL (url).pathname || "/" ,
2212 area: classifyUrl (url),
2213 title: seo.title,
2214 seo,
2215 sections: extractSectionsFromHtml (html),
2216 sectionDiagnostics: {
2217 source: "none" ,
2218 contentRoot: "none" ,
2219 accessibilitySnapshotAvailable: false ,
2220 accessibilityRoot: null ,
2221 candidateCount: 0 ,
2222 acceptedSectionCount: 0 ,
2223 acceptedModuleCount: 0 ,
2224 warnings: [ "browser-unavailable: section extraction requires browser-based DOM, layout, and accessibility evidence" ],
2225 },
2226 repeaters: [],
2227 repeaterDiagnostics: {
2228 source: "none" ,
2229 accessibilitySnapshotAvailable: false ,
2230 signalCount: 0 ,
2231 acceptedRepeaterCount: 0 ,
2232 warnings: [ "browser-unavailable: repeater extraction requires browser-based accessibility and DOM evidence" ],
2233 },
2234 footer: null ,
2235 links: extractLinks (html, url),
2236 tokens: inferTokensFromHtml (html),
2237 chrome: {
2238 direction: "unknown" ,
2239 header: { variants: [], maxStickyViewportRatio: 0.16 },
2240 navigation: [],
2241 heroEvidence: {
2242 visibleText: [],
2243 forbiddenSeoOnlyText: seoOnlyText ({ seo, visibleText: [] }),
2244 },
2245 warnings: [ "browser-unavailable: header behavior, dropdown hierarchy, logo footprint, and hero visible-text evidence require manual screenshot review" ],
2246 },
2247 screenshots: {},
2248 extractionWarnings: [ "browser-unavailable: repeater extraction requires browser-based accessibility and DOM evidence" ],
2249 extractedAt: new Date (). toISOString (),
2250 };
2251 }
2252
2253 async function detectRepeaterSignals ( page ) {
2254 const candidates = [
2255 { selector: 'main, [role="main"]' , label: "main" },
2256 { selector: "article" , label: "article" },
2257 { selector: "body" , label: "body" },
2258 ];
2259 const failures = [];
2260 for ( const candidate of candidates) {
2261 const locator = page. locator (candidate.selector). first ();
2262 const count = await locator. count ();
2263 if (count === 0 ) {
2264 failures. push ( `${ candidate . label }: selector did not match any element` );
2265 continue ;
2266 }
2267 try {
2268 const snapshot = await locator. ariaSnapshot ({ mode: "ai" });
2269 const detection = detectRepeaterSignalsFromAria (snapshot);
2270 return {
2271 accessibilitySnapshotAvailable: true ,
2272 accessibilityRoot: candidate.label,
2273 candidateCount: detection.candidateCount,
2274 signals: detection.signals,
2275 warnings: detection.signals. length
2276 ? []
2277 : [ `accessibility-snapshot-captured-on-${ candidate . label }-but-no-high-confidence-repeater-patterns-found` ],
2278 };
2279 } catch (error) {
2280 failures. push ( `${ candidate . label }: ${ error . message }` );
2281 }
2282 }
2283 return {
2284 accessibilitySnapshotAvailable: false ,
2285 accessibilityRoot: null ,
2286 candidateCount: 0 ,
2287 signals: [],
2288 warnings: [ `accessibility-snapshot-unavailable: ${ failures . join ( "; " ) }` ],
2289 };
2290 }
2291
2292 async function navigateWithAdaptiveProfile ( page , url ) {
2293 const profiles = [
2294 { name: "normal" , waitUntil: "networkidle" , timeoutMs: 45000 , settleMs: 0 },
2295 { name: "slow-site" , waitUntil: "domcontentloaded" , timeoutMs: 90000 , settleMs: 2000 },
2296 ];
2297 const warnings = [];
2298 let lastError = null ;
2299 for ( const profile of profiles) {
2300 try {
2301 await page. goto (url, { waitUntil: profile.waitUntil, timeout: profile.timeoutMs });
2302 if (profile.settleMs > 0 ) {
2303 await page. waitForTimeout (profile.settleMs);
2304 }
2305 if (profile.name !== "normal" ) {
2306 warnings. push ( `used ${ profile . name } navigation profile after the default profile did not settle cleanly` );
2307 }
2308 return {
2309 profile: profile.name,
2310 waitUntil: profile.waitUntil,
2311 timeoutMs: profile.timeoutMs,
2312 warnings,
2313 };
2314 } catch (error) {
2315 lastError = error;
2316 warnings. push ( `${ profile . name } failed: ${ error . message }` );
2317 }
2318 }
2319 throw lastError;
2320 }
2321
2322 function detectRepeaterSignalsFromAria ( snapshot ) {
2323 const tree = parseAriaSnapshot (snapshot);
2324 if ( ! tree) return { signals: [], candidateCount: 0 };
2325 const mains = [];
2326 walkAriaTree (tree, ( node ) => {
2327 if (node.role === "main" ) mains. push (node);
2328 });
2329 const root = mains. at ( - 1 ) || tree;
2330 const candidates = [];
2331 walkAriaTree (root, ( node ) => {
2332 const grouped = groupHeadingContentPairs (node.children || []);
2333 for ( const group of grouped) {
2334 if (group.items. length >= 3 ) candidates. push (group);
2335 }
2336 });
2337 candidates. sort (( a , b ) => {
2338 if (b.items. length !== a.items. length ) return b.items. length - a.items. length ;
2339 return averageMediaCount (b.items) - averageMediaCount (a.items);
2340 });
2341 return {
2342 candidateCount: candidates. length ,
2343 signals: candidates. slice ( 0 , 3 ). map (( candidate , index ) => ({
2344 label: `main-repeated-content-${ index + 1 }` ,
2345 headingLevel: candidate.headingLevel,
2346 roleSignature: `heading-${ candidate . headingLevel }+generic` ,
2347 items: candidate.items. map (( item ) => ({
2348 heading: item.heading,
2349 paragraphCount: item.paragraphCount,
2350 imageCount: item.imageCount,
2351 })),
2352 })),
2353 };
2354 }
2355
2356 function averageMediaCount ( items ) {
2357 return items. reduce (( sum , item ) => sum + item.imageCount, 0 ) / Math. max (items. length , 1 );
2358 }
2359
2360 function groupHeadingContentPairs ( children = []) {
2361 const grouped = new Map ();
2362 for ( let index = 0 ; index < children. length - 1 ; index += 1 ) {
2363 const heading = children[index];
2364 const content = children[index + 1 ];
2365 if (heading.role !== "heading" || content.role !== "generic" ) continue ;
2366 const headingLevel = heading.level || 0 ;
2367 if (headingLevel < 2 ) continue ;
2368 const paragraphCount = countAriaRole (content, "paragraph" );
2369 const imageCount = countAriaRole (content, "img" );
2370 if ( ! heading.name || paragraphCount < 1 || imageCount < 1 ) continue ;
2371 const key = String (headingLevel);
2372 if ( ! grouped. has (key)) grouped. set (key, { headingLevel, items: [] });
2373 grouped. get (key).items. push ({
2374 heading: heading.name,
2375 paragraphCount,
2376 imageCount,
2377 });
2378 }
2379 return Array. from (grouped. values ());
2380 }
2381
2382 function countAriaRole ( node , role ) {
2383 let count = 0 ;
2384 walkAriaTree (node, ( current ) => {
2385 if (current.role === role) count += 1 ;
2386 });
2387 return count;
2388 }
2389
2390 function walkAriaTree ( node , visit ) {
2391 if ( ! node) return ;
2392 visit (node);
2393 for ( const child of node.children || []) walkAriaTree (child, visit);
2394 }
2395
2396 function parseAriaSnapshot ( snapshot ) {
2397 const lines = String (snapshot || "" )
2398 . split ( " \n " )
2399 . map (( line ) => line. match ( / ^ ( \s * )- ( . * ) $ / ))
2400 . filter (Boolean)
2401 . map (([, indent , content ]) => ({ depth: Math. floor (indent. length / 2 ), content }));
2402 if ( ! lines. length ) return null ;
2403 const root = { role: "root" , children: [] };
2404 const stack = [root];
2405 for ( const line of lines) {
2406 const node = parseAriaLine (line.content);
2407 if ( ! node) continue ;
2408 while (stack. length > line.depth + 1 ) stack. pop ();
2409 const parent = stack[stack. length - 1 ];
2410 parent.children. push (node);
2411 stack. push (node);
2412 }
2413 return root;
2414 }
2415
2416 function parseAriaLine ( content ) {
2417 const trimmed = String (content || "" ). trim ();
2418 if ( ! trimmed) return null ;
2419 if (trimmed. startsWith ( "text:" )) {
2420 return { role: "text" , name: trimmed. slice ( 5 ). trim (), children: [] };
2421 }
2422 const roleMatch = trimmed. match ( / ^ ( [a-zA-Z0-9-] + )/ );
2423 if ( ! roleMatch) return null ;
2424 const role = roleMatch[ 1 ];
2425 const nameMatch = trimmed. match ( /"( [ ^ "] + )"/ );
2426 const levelMatch = trimmed. match ( / \[ level=( \d + ) \] / );
2427 return {
2428 role,
2429 name: nameMatch?.[ 1 ] || "" ,
2430 level: levelMatch ? Number (levelMatch[ 1 ]) : null ,
2431 raw: trimmed,
2432 children: [],
2433 };
2434 }
2435
2436 function seoOnlyText ({ seo , visibleText }) {
2437 const visible = new Set ((visibleText || []). map (normalizeEvidenceText));
2438 return [
2439 seo?.title,
2440 seo?.openGraph?.[ "og:title" ],
2441 seo?.jsonLd?.name,
2442 seo?.jsonLd?.description,
2443 seo?.description,
2444 ]
2445 . flat ()
2446 . filter (Boolean)
2447 . map (( value ) => String (value). replace ( /'/ g , "'" ). replace ( /"/ g , " \" " ). replace ( /&/ g , "&" ). trim ())
2448 . filter (( value , index , values ) => values. indexOf (value) === index)
2449 . filter (( value ) => ! visible. has ( normalizeEvidenceText (value)));
2450 }
2451
2452 function normalizeEvidenceText ( value ) {
2453 return String (value || "" ). replace ( / \s + / g , " " ). trim (). toLowerCase ();
2454 }
2455
2456 if ( import . meta .url === `file://${ process . argv [ 1 ] }` ) {
2457 main (). catch (( error ) => {
2458 console. error (error.stack || error.message);
2459 process. exit ( 1 );
2460 });
2461 }