Setting the file. One moment.
Validate Stagehand · Add Webmcp · browserbase/skills · Skills Docs
Back to the top of the page scripts/ validate-stagehand.mjs
JavaScript · 232 lines · 10 KB
"consequential"
]);
9
10 function usage () {
11 return [
12 "Usage: validate-stagehand.mjs --url <url> --config <file> (--local | --browserbase)" ,
13 " [--init-script <file>] [--executable-path <file>] [--headless] [--no-sandbox]" ,
14 " [--allow-consequential]" ,
15 ]. join ( " \n " );
16 }
17
18 function parseArgs ( argv ) {
19 const args = {
20 mode: null ,
21 headed: null ,
22 noSandbox: false ,
23 allowConsequential: false ,
24 };
25 for ( let index = 0 ; index < argv. length ; index += 1 ) {
26 const value = argv[index];
27 if (value === "--url" || value === "--config" || value === "--init-script" || value === "--executable-path" ) {
28 const key = value. slice ( 2 ). replace ( /-( [a-z] )/ g , ( _ , letter ) => letter. toUpperCase ());
29 const next = argv[index + 1 ];
30 if ( ! next) throw new Error ( `${ value } requires a value` );
31 args[key] = next;
32 index += 1 ;
33 } else if (value === "--local" || value === "--browserbase" ) {
34 const mode = value. slice ( 2 );
35 if (args.mode && args.mode !== mode) throw new Error ( "Choose exactly one of --local or --browserbase" );
36 args.mode = mode;
37 } else if (value === "--headed" ) {
38 args.headed = true ;
39 } else if (value === "--headless" ) {
40 args.headed = false ;
41 } else if (value === "--no-sandbox" ) {
42 args.noSandbox = true ;
43 } else if (value === "--allow-consequential" ) {
44 args.allowConsequential = true ;
45 } else {
46 throw new Error ( `Unknown argument: ${ value } \n ${ usage () }` );
47 }
48 }
49 if ( ! args.url || ! args.config || ! args.mode) throw new Error ( usage ());
50 if (args.headed !== null && args.mode !== "local" ) {
51 throw new Error ( "--headed and --headless are only valid with --local" );
52 }
53 if (args.headed === null ) args.headed = true ;
54 if (args.noSandbox && args.mode !== "local" ) throw new Error ( "--no-sandbox is only valid with --local" );
55 if (args.executablePath && args.mode !== "local" ) throw new Error ( "--executable-path is only valid with --local" );
56 return args;
57 }
58
59 function assertPlainObject ( value , label ) {
60 if ( ! value || typeof value !== "object" || Array. isArray (value)) {
61 throw new Error ( `${ label } must be an object` );
62 }
63 }
64
65 function validateConfig ( config ) {
66 assertPlainObject (config, "Config" );
67 if ( ! Array. isArray (config.tools) || config.tools. length === 0 ) {
68 throw new Error ( "Config tools must be a non-empty array" );
69 }
70 if (config.timeoutMs !== undefined && ( ! Number. isFinite (config.timeoutMs) || config.timeoutMs < 0 )) {
71 throw new Error ( "Config timeoutMs must be a non-negative number" );
72 }
73 if (config.expectedDom !== undefined ) {
74 if ( ! Array. isArray (config.expectedDom)) throw new Error ( "Config expectedDom must be an array" );
75 for ( const [ index , expectation ] of config.expectedDom. entries ()) {
76 assertPlainObject (expectation, `expectedDom[${ index }]` );
77 if ( typeof expectation.selector !== "string" || ! expectation.selector. trim ()) {
78 throw new Error ( `expectedDom[${ index }].selector is required` );
79 }
80 if ( typeof expectation.text !== "string" ) throw new Error ( `expectedDom[${ index }].text must be a string` );
81 }
82 }
83
84 const names = new Set ();
85 for ( const [ index , tool ] of config.tools. entries ()) {
86 assertPlainObject (tool, `tools[${ index }]` );
87 if ( typeof tool.name !== "string" || ! tool.name. trim ()) throw new Error ( `tools[${ index }].name is required` );
88 if (names. has (tool.name)) throw new Error ( `Duplicate expected tool: ${ tool . name }` );
89 names. add (tool.name);
90 if ( ! RISKS . has (tool.risk)) {
91 throw new Error ( `Tool ${ tool . name } risk must be read-only, reversible, or consequential` );
92 }
93 if (tool.expectedAnnotations !== undefined ) assertPlainObject (tool.expectedAnnotations, `${ tool . name }.expectedAnnotations` );
94 if (tool.expectedOutputSubset !== undefined ) assertPlainObject (tool.expectedOutputSubset, `${ tool . name }.expectedOutputSubset` );
95 if (tool.input !== undefined ) assertPlainObject (tool.input, `${ tool . name }.input` );
96 }
97 }
98
99 function deepSubset ( actual , expected , location = "value" ) {
100 if (Array. isArray (expected)) {
101 if ( ! Array. isArray (actual)) return `${ location } is not an array` ;
102 if (actual. length < expected. length ) return `${ location } has ${ actual . length } items; expected at least ${ expected . length }` ;
103 for ( let index = 0 ; index < expected. length ; index += 1 ) {
104 const failure = deepSubset (actual[index], expected[index], `${ location }[${ index }]` );
105 if (failure) return failure;
106 }
107 return null ;
108 }
109 if (expected && typeof expected === "object" ) {
110 if ( ! actual || typeof actual !== "object" || Array. isArray (actual)) return `${ location } is not an object` ;
111 for ( const [ key , expectedValue ] of Object. entries (expected)) {
112 if ( ! (key in actual)) return `${ location }.${ key } is missing` ;
113 const failure = deepSubset (actual[key], expectedValue, `${ location }.${ key }` );
114 if (failure) return failure;
115 }
116 return null ;
117 }
118 return Object. is (actual, expected) ? null : `${ location } was ${ JSON . stringify ( actual ) }; expected ${ JSON . stringify ( expected ) }` ;
119 }
120
121 async function launch ( args ) {
122 if (args.mode === "browserbase" ) {
123 if ( ! process.env. BROWSERBASE_API_KEY ) throw new Error ( "BROWSERBASE_API_KEY is required for --browserbase" );
124 return browserbase. launch ({
125 apiKey: process.env. BROWSERBASE_API_KEY ,
126 userMetadata: { suite: "add-webmcp-validator" },
127 });
128 }
129 return localBrowser. launch ({
130 headless: ! args.headed,
131 ... (args.noSandbox ? { chromiumSandbox: false } : {}),
132 ... (args.executablePath ? { executablePath: path. resolve (args.executablePath) } : {}),
133 });
134 }
135
136 async function run () {
137 const args = parseArgs (process.argv. slice ( 2 ));
138 const configPath = path. resolve (args.config);
139 const config = JSON . parse ( await readFile (configPath, "utf8" ));
140 validateConfig (config);
141
142 const unsafe = config.tools. filter (( tool ) => tool.risk === "consequential" && Object. hasOwn (tool, "input" ));
143 if (unsafe. length > 0 && ! args.allowConsequential) {
144 throw new Error (
145 `Refusing consequential invocation(s): ${ unsafe . map (( tool ) => tool . name ). join ( ", " ) }. ` +
146 "Remove input for discovery-only validation or use --allow-consequential in an explicitly authorized sandbox." ,
147 );
148 }
149
150 const browser = await launch (args);
151 let stagehand;
152 const failures = [];
153 try {
154 stagehand = await Stagehand. create ({ browser });
155 const pages = await stagehand.browser.context. pages ();
156 const page = pages[ 0 ] ?? ( await stagehand.browser.context. newPage ());
157 if (args.initScript) {
158 await page. addInitScript ({ path: path. resolve (args.initScript) });
159 }
160 await page. goto (args.url, { waitUntil: "load" });
161
162 const timeout = config.timeoutMs ?? 5_000 ;
163 const discovered = await page. tools ({ timeout });
164 console. log ( `Stagehand discovered ${ discovered . length } WebMCP tool(s)` );
165
166 for ( const expected of config.tools) {
167 const tool = discovered. find (( candidate ) => candidate.name === expected.name);
168 if ( ! tool) {
169 failures. push ( `${ expected . name }: not discovered` );
170 console. log ( `FAIL ${ expected . name }: not discovered` );
171 continue ;
172 }
173 const failureCountBeforeValidation = failures. length ;
174 if ( ! tool.description?. trim ()) failures. push ( `${ expected . name }: description is empty` );
175 if ( ! tool.inputSchema || typeof tool.inputSchema !== "object" || Array. isArray (tool.inputSchema)) {
176 failures. push ( `${ expected . name }: input schema is missing or not an object` );
177 }
178 if (expected.expectedAnnotations) {
179 const mismatch = deepSubset (tool.annotations, expected.expectedAnnotations, `${ expected . name }.annotations` );
180 if (mismatch) failures. push (mismatch);
181 }
182
183 if ( ! Object. hasOwn (expected, "input" )) {
184 const status = failures. length === failureCountBeforeValidation ? "PASS" : "FAIL" ;
185 console. log ( `${ status } ${ expected . name }: discovered (discovery-only, risk=${ expected . risk })` );
186 continue ;
187 }
188
189 const invocation = await tool. invoke ({ input: expected.input });
190 const response = await invocation. result ({ timeout });
191 const expectedStatus = expected.expectedStatus ?? "Completed" ;
192 if (response.status !== expectedStatus) {
193 failures. push ( `${ expected . name }: status ${ response . status }; expected ${ expectedStatus }` );
194 }
195 if (expected.expectedOutputSubset) {
196 const mismatch = deepSubset (response.output, expected.expectedOutputSubset, `${ expected . name }.output` );
197 if (mismatch) failures. push (mismatch);
198 }
199 const status = failures. length === failureCountBeforeValidation ? "PASS" : "FAIL" ;
200 console. log ( `${ status } ${ expected . name }: invoked status=${ response . status } risk=${ expected . risk }` );
201 if (expected.expectedOutputSubset) {
202 console. log ( ` verified output subset: ${ JSON . stringify ( expected . expectedOutputSubset ) }` );
203 }
204 }
205
206 for ( const expected of config.expectedDom ?? []) {
207 const actualText = await page. locator (expected.selector). textContent ();
208 if (actualText !== expected.text) {
209 failures. push ( `DOM ${ expected . selector }: text was ${ JSON . stringify ( actualText ) }; expected ${ JSON . stringify ( expected . text ) }` );
210 console. log ( `FAIL DOM ${ expected . selector }` );
211 } else {
212 console. log ( `PASS DOM ${ expected . selector }: ${ JSON . stringify ( expected . text ) }` );
213 }
214 }
215 } finally {
216 await stagehand?. close (). catch (() => {});
217 await browser. close (). catch (() => {});
218 }
219
220 if (failures. length > 0 ) {
221 console. error ( "Validation failed:" );
222 for ( const failure of failures) console. error ( `- ${ failure }` );
223 process.exitCode = 1 ;
224 } else {
225 console. log ( `Validation passed: ${ config . tools . length }/${ config . tools . length } expected tool(s)` );
226 }
227 }
228
229 run (). catch (( error ) => {
230 console. error (error instanceof Error ? error.message : String (error));
231 process.exitCode = 1 ;
232 });