Setting the file. One moment. Bundled file Gitignore
scripts/validate.mjs
JavaScript·173 lines·5 KB
artifactDir
) {
11 return JSON.parse(await readFile(path.join(artifactDir, "manifest.json"), "utf8"));
12}
13
14function outputLooksFailed(output) {
15 return output !== null && typeof output === "object" && "success" in output && output.success === false;
16}
17
18async function writeEvalResult(artifactDir, result) {
19 await writeFile(path.join(artifactDir, "eval.json"), `${JSON.stringify(result, null, 2)}\n`, "utf8");
20 const lines = [
21 "# WebMCP Eval Report",
22 "",
23 `Status: ${result.status}`,
24 `URL: ${result.url}`,
25 "",
26 "## Tools",
27 ""
28 ];
29 for (const tool of result.tools) {
30 lines.push(`- ${tool.name}: found=${tool.found} invoked=${tool.invoked} status=${tool.status ?? "n/a"}`);
31 if (tool.error) lines.push(` - error: ${tool.error}`);
32 }
33 if (result.errors.length) {
34 lines.push("", "## Errors", "");
35 for (const error of result.errors) lines.push(`- ${error}`);
36 }
37 await writeFile(path.join(artifactDir, "eval-report.md"), `${lines.join("\n")}\n`, "utf8");
38}
39
40async function validateArtifact(artifactDir) {
41 const manifest = await readManifest(artifactDir);
42 const scriptPath = path.join(artifactDir, "webmcp.init.js");
43 const errors = [];
44 const tools = [];
45
46 if (!Array.isArray(manifest.tools)) {
47 const result = {
48 artifactDir,
49 url: manifest.url,
50 status: "failed",
51 tools,
52 errors: ["Manifest `tools` must be an array."]
53 };
54 await writeEvalResult(artifactDir, result);
55 return result;
56 }
57
58 try {
59 await access(scriptPath);
60 } catch {
61 const result = {
62 artifactDir,
63 url: manifest.url,
64 status: "failed",
65 tools,
66 errors: [`Missing webmcp.init.js. Run compile.mjs ${artifactDir} first.`]
67 };
68 await writeEvalResult(artifactDir, result);
69 return result;
70 }
71
72 const stagehand = new Stagehand({
73 env: "LOCAL",
74 verbose: 1,
75 localBrowserLaunchOptions: {
76 args: ["--enable-features=WebMCPTesting,DevToolsWebMCPSupport"]
77 }
78 });
79
80 try {
81 await stagehand.init();
82 const page = stagehand.context.pages()[0] ?? (await stagehand.context.newPage());
83 await page.addInitScript({ path: scriptPath });
84 await page.goto(manifest.url, { waitUntil: "load" });
85
86 const registeredTools = await page.listWebMCPTools({ timeoutMs: 5000 });
87
88 for (const expectedTool of manifest.tools) {
89 const foundTool = registeredTools.find((tool) => tool.name === expectedTool.name);
90 if (!foundTool) {
91 tools.push({
92 name: expectedTool.name,
93 found: false,
94 invoked: false,
95 error: "Tool was not registered."
96 });
97 continue;
98 }
99
100 try {
101 const invocation = await page.invokeWebMCPTool(
102 expectedTool.name,
103 expectedTool.fixtureInput ?? {},
104 { frameId: foundTool.frameId }
105 );
106 const result = await invocation.result;
107 const failed = Boolean(result.errorText) || outputLooksFailed(result.output);
108 const error = failed
109 ? result.errorText || "Tool returned an output with success=false."
110 : undefined;
111 tools.push({
112 name: expectedTool.name,
113 found: true,
114 invoked: true,
115 status: result.status,
116 output: result.output,
117 ...(error ? { error } : {})
118 });
119 } catch (error) {
120 tools.push({
121 name: expectedTool.name,
122 found: true,
123 invoked: false,
124 error: error instanceof Error ? error.message : String(error)
125 });
126 }
127 }
128 } catch (error) {
129 errors.push(error instanceof Error ? error.message : String(error));
130 } finally {
131 await stagehand.close().catch(() => {});
132 }
133
134 if (manifest.tools.length === 0) {
135 errors.push("Manifest declares no tools; nothing to validate.");
136 }
137
138 const status = errors.length === 0 &&
139 manifest.tools.length > 0 &&
140 tools.length === manifest.tools.length &&
141 tools.every((tool) => tool.found && tool.invoked && tool.status === "Completed" && !tool.error)
142 ? "passed"
143 : "failed";
144
145 const result = {
146 artifactDir,
147 url: manifest.url,
148 status,
149 tools,
150 errors
151 };
152 await writeEvalResult(artifactDir, result);
153 return result;
154}
155
156async function main() {
157 const artifactDir = process.argv[2];
158 if (!artifactDir) throw new Error(usage());
159 const resolvedArtifactDir = path.resolve(process.cwd(), artifactDir);
160 const result = await validateArtifact(resolvedArtifactDir);
161 console.log(`Validation ${result.status}: ${resolvedArtifactDir}`);
162 for (const tool of result.tools) {
163 console.log(`- ${tool.name}: found=${tool.found} invoked=${tool.invoked} status=${tool.status ?? "n/a"}`);
164 if (tool.error) console.log(` error=${tool.error}`);
165 }
166 for (const error of result.errors) console.log(`Error: ${error}`);
167 if (result.status !== "passed") process.exitCode = 1;
168}
169
170main().catch((error) => {
171 console.error(error instanceof Error ? error.message : String(error));
172 process.exitCode = 1;
173});