Setting the file. One moment.
HTML Extract · Wix Headless Replatform · wix/skills · Skills Docs
Back to the top of the page 28.10
Workflow
This file
Number 28.48
Position 48 of 89
Type JavaScript
Size 6 KB
Lines 168 scripts/lib/ html-extract.mjs
JavaScript · 168 lines · 6 KB
extractMeta
(
html
) {
8 const metas = [];
9 const pattern = /<meta \b ( [ ^ >] + )>/ gi ;
10 let match;
11 while ((match = pattern. exec (html))) {
12 const attrs = parseAttrs (match[ 1 ]);
13 metas. push (attrs);
14 }
15 return metas;
16 }
17
18 export function extractSeo ( html , url ) {
19 const metas = extractMeta (html);
20 const metaBy = ( field , value ) => metas. find (( meta ) => meta[field] === value)?.content || "" ;
21 const canonical = matchAttr (html, /<link \b [ ^ >] * rel \s * = \s * ["'] canonical ["'][ ^ >] * >/ i , "href" );
22 const icons = [];
23 const linkPattern = /<link \b ( [ ^ >] + )>/ gi ;
24 let linkMatch;
25 while ((linkMatch = linkPattern. exec (html))) {
26 const attrs = parseAttrs (linkMatch[ 1 ]);
27 if ((attrs.rel || "" ). toLowerCase (). includes ( "icon" )) icons. push (attrs.href);
28 }
29 const schema = [];
30 const schemaPattern = /<script \b [ ^ >] * type \s * = \s * ["'] application \/ ld \+ json ["'][ ^ >] * >( [\s\S] *? )< \/ script>/ gi ;
31 let schemaMatch;
32 while ((schemaMatch = schemaPattern. exec (html))) schema. push (schemaMatch[ 1 ]. trim ());
33 return {
34 url,
35 title: extractTitle (html),
36 description: metaBy ( "name" , "description" ),
37 robots: metaBy ( "name" , "robots" ),
38 canonical,
39 openGraph: Object. fromEntries (metas. filter (( m ) => m.property?. startsWith ( "og:" )). map (( m ) => [m.property, m.content || "" ])),
40 twitter: Object. fromEntries (metas. filter (( m ) => m.name?. startsWith ( "twitter:" )). map (( m ) => [m.name, m.content || "" ])),
41 icons,
42 schema,
43 };
44 }
45
46 export function extractSectionsFromHtml ( html ) {
47 const body = rawMatchContent (html, /<body [ ^ >] * >( [\s\S] *? )< \/ body>/ i ) || html;
48 const sectionPattern = /<(header | main | section | article | footer | nav) \b ( [ ^ >] * )>( [\s\S] *? )< \/ \1 >/ gi ;
49 const sections = [];
50 let match;
51 while ((match = sectionPattern. exec (body)) && sections. length < 40 ) {
52 const text = visibleHtmlText (match[ 3 ]);
53 if ( ! text) continue ;
54 sections. push ({
55 tag: match[ 1 ]. toLowerCase (),
56 id: parseAttrs (match[ 2 ]).id || "" ,
57 className: parseAttrs (match[ 2 ]).class || "" ,
58 text: text. slice ( 0 , 1600 ),
59 heading: firstHeading (match[ 3 ]),
60 });
61 }
62 if ( ! sections. length ) {
63 const text = visibleHtmlText (body);
64 sections. push ({ tag: "body" , id: "" , className: "" , heading: "" , text: text. slice ( 0 , 3000 ) });
65 }
66 return sections;
67 }
68
69 export function extractAssetsFromHtml ( html , baseUrl ) {
70 const assets = [];
71 const add = ( value , type ) => {
72 try {
73 const url = new URL (value, baseUrl);
74 if ( / ^ https ? : $ / . test (url.protocol)) assets. push ({ sourceUrl: url. toString (), type });
75 } catch {
76 // Ignore invalid asset URLs.
77 }
78 };
79 for ( const pattern of [
80 { re: /<img \b [ ^ >] * src \s * = \s * ["'] ( [ ^ "'] + ) ["'][ ^ >] * >/ gi , type: "image" },
81 { re: /<source \b [ ^ >] * srcset \s * = \s * ["'] ( [ ^ "'] + ) ["'][ ^ >] * >/ gi , type: "image" },
82 { re: /<video \b [ ^ >] * src \s * = \s * ["'] ( [ ^ "'] + ) ["'][ ^ >] * >/ gi , type: "video" },
83 { re: /<link \b [ ^ >] * href \s * = \s * ["'] ( [ ^ "'] + ) ["'][ ^ >] * >/ gi , type: "linked" },
84 ]) {
85 let match;
86 while ((match = pattern.re. exec (html))) {
87 const candidates = String (match[ 1 ]). split ( "," ). map (( part ) => part. trim (). split ( / \s + / )[ 0 ]);
88 for ( const candidate of candidates) add (candidate, pattern.type);
89 }
90 }
91 const bgPattern = /url \( ( ['"] ? )( [ ^ '")] + ) \1 \) / gi ;
92 let bgMatch;
93 while ((bgMatch = bgPattern. exec (html))) add (bgMatch[ 2 ], "background" );
94 return dedupeAssets (assets);
95 }
96
97 export function extractStylesheetUrls ( html , baseUrl ) {
98 const urls = [];
99 const pattern = /<link \b ( [ ^ >] + )>/ gi ;
100 let match;
101 while ((match = pattern. exec (html))) {
102 const attrs = parseAttrs (match[ 1 ]);
103 if ( ! / \b stylesheet \b / i . test (attrs.rel || "" )) continue ;
104 if ( ! attrs.href) continue ;
105 try {
106 urls. push ( new URL (attrs.href, baseUrl). toString ());
107 } catch {
108 // Ignore invalid stylesheet URLs.
109 }
110 }
111 return Array. from ( new Set (urls));
112 }
113
114 export function inferTokensFromHtml ( html ) {
115 const colors = Array. from ( new Set ((html. match ( /# [0-9a-fA-F] {3,8}\b / g ) || []). map (( c ) => c. toLowerCase ()))). slice ( 0 , 24 );
116 const fontFamilies = Array. from (
117 new Set (
118 (html. match ( /font-family \s * : \s * ( [ ^ ;"}] + )/ gi ) || [])
119 . map (( value ) => value. split ( ":" ). slice ( 1 ). join ( ":" ). trim (). replace ( / ^ ['"] | ['"] $ / g , "" )),
120 ),
121 ). slice ( 0 , 12 );
122 const radii = Array. from ( new Set (html. match ( /border-radius \s * : \s * [ ^ ;"}] + / gi ) || [])). slice ( 0 , 12 );
123 const shadows = Array. from ( new Set (html. match ( /box-shadow \s * : \s * [ ^ ;"}] + / gi ) || [])). slice ( 0 , 12 );
124 return { colors, fontFamilies, radii, shadows };
125 }
126
127 export function parseAttrs ( raw ) {
128 const attrs = {};
129 const pattern = /( [:@\w-] + )(?: \s * = \s * (?:"( [ ^ "] * )" | '( [ ^ '] * )' | ( [ ^ \s"'=<>`] + ))) ? / g ;
130 let match;
131 while ((match = pattern. exec (raw || "" ))) {
132 attrs[match[ 1 ]. toLowerCase ()] = match[ 2 ] ?? match[ 3 ] ?? match[ 4 ] ?? "" ;
133 }
134 return attrs;
135 }
136
137 function matchContent ( html , pattern ) {
138 const match = pattern. exec (html);
139 return match ? visibleHtmlText (match[ 1 ]) : "" ;
140 }
141
142 function rawMatchContent ( html , pattern ) {
143 return pattern. exec (html)?.[ 1 ] || "" ;
144 }
145
146 function visibleHtmlText ( html ) {
147 const withoutNonContent = String (html || "" ). replace ( /<(script | style | noscript | template) \b [ ^ >] * > [\s\S] *? < \/ \1 \s * >/ gi , " " );
148 return stripTags (withoutNonContent). replace ( / \s + / g , " " ). trim ();
149 }
150
151 function matchAttr ( html , tagPattern , attr ) {
152 const match = tagPattern. exec (html);
153 if ( ! match) return "" ;
154 return parseAttrs (match[ 0 ])[attr] || "" ;
155 }
156
157 function firstHeading ( html ) {
158 return matchContent (html, /<h [1-3][ ^ >] * >( [\s\S] *? )< \/ h [1-3] >/ i );
159 }
160
161 function dedupeAssets ( assets ) {
162 const seen = new Set ();
163 return assets. filter (( asset ) => {
164 if (seen. has (asset.sourceUrl)) return false ;
165 seen. add (asset.sourceUrl);
166 return true ;
167 });
168 }