Setting the file. One moment.
Kit · N8n:testmax Assistant UI · n8n-io/n8n · Skills Docs
Back to the top of the page scripts/ kit.mjs
JavaScript · 351 lines · 13 KB
'node:fs'
;
14 import { join } from 'node:path' ;
15
16 import { expect } from '../../testmax-n8n-ai/scripts/n8n_browser.mjs' ;
17
18 const THREAD_URL = / \/ assistant \/ ( [0-9a-f] {8} - [0-9a-f] {4} - [0-9a-f] {4} - [0-9a-f] {4} - [0-9a-f] {12} )/ ;
19 const RUN_GRACE_MS = 10_000 ;
20
21 /**
22 * Replace the accessibility subtree of each masked element with one line. Masked texts alone keep
23 * the structure, so a list with a different number of rows still shows as a change. Each subtree
24 * replaces only its first unmasked match, so an unmasked element with the same content stays.
25 */
26 function maskSubtrees ( tree , subtrees ) {
27 const lines = tree. split ( ' \n ' );
28 for ( const subtree of subtrees) {
29 const sub = subtree. split ( ' \n ' ). filter (( l ) => l. trim ());
30 if ( ! sub. length || sub[ 0 ]. startsWith ( '#' )) continue ;
31 for ( let i = 0 ; i + sub. length <= lines. length ; i ++ ) {
32 if ( ! lines[i]. endsWith (sub[ 0 ])) continue ;
33 const indent = lines[i]. slice ( 0 , lines[i]. length - sub[ 0 ]. length );
34 if (indent. trim () || ! sub. every (( l , j ) => lines[i + j] === indent + l)) continue ;
35 lines. splice (i, sub. length , `${ indent }- text: "[masked]"` );
36 break ;
37 }
38 }
39 return lines. join ( ' \n ' );
40 }
41
42 /** The accessibility tree without the masked texts and the instance port, so branch and master compare. */
43 function maskTree ( tree , texts ) {
44 let out = tree. replace ( /localhost(: | %3A) \d + / g , 'localhost$1PORT' );
45 for ( const text of [ ... texts]. sort (( a , b ) => b. length - a. length ))
46 out = out. split (text). join ( '[masked]' );
47 return out;
48 }
49
50 export class StepFailed extends Error {}
51
52 /** Locators of the Instance AI page. They mirror pages/InstanceAiPage.ts in the Playwright tests. */
53 function locators ( page ) {
54 const container = () => page. getByTestId ( 'instance-ai-container' );
55 return {
56 container,
57 chatInput : () => container (). getByRole ( 'textbox' ). or ( container (). getByRole ( 'combobox' )),
58 composer : () => container (). getByTestId ( 'instance-ai-composer' ),
59 sendButton : () => container (). getByTestId ( 'instance-ai-send-button' ),
60 stopButton : () => container (). getByTestId ( 'instance-ai-stop-button' ),
61 userMessages : () => container (). getByTestId ( 'instance-ai-user-message' ),
62 assistantMessages : () => container (). getByTestId ( 'instance-ai-assistant-message' ),
63 assistantText : ( text ) =>
64 container (). getByTestId ( 'instance-ai-assistant-message' ). getByText (text),
65 panelText : ( text ) => container (). getByText (text),
66 emptyState : () => container (). getByTestId ( 'instance-ai-empty-state' ),
67 statusBar : () => container (). getByTestId ( 'instance-ai-status-bar' ),
68 sidebarToggle : () => container (). getByTestId ( 'instance-ai-sidebar-toggle' ),
69 threadList : () => page. getByTestId ( 'instance-ai-thread-list' ),
70 newThreadButton : () =>
71 page
72 . getByTestId ( 'project-instance-ai-menu-item' )
73 . getByRole ( 'menuitem' , { name: 'Assistant' , exact: true }),
74 mentionButton : () => container (). getByTestId ( 'instance-ai-mention-button' ),
75 mentionMenu : () => page. getByTestId ( 'instance-ai-mention-menu-content' ),
76 fileInput : () => container (). locator ( 'input[type="file"]' ),
77 approveButton : () => container (). getByTestId ( 'approval-card-allow-once' ),
78 alwaysAllowButton : () => container (). getByTestId ( 'approval-card-always-allow' ),
79 denyButton : () => container (). getByTestId ( 'approval-card-deny' ),
80 domainAllowOnce : () => container (). getByTestId ( 'domain-access-allow-once' ),
81 planApproveButton : () => container (). getByTestId ( 'instance-ai-plan-approve' ),
82 credentialContinue : () => container (). getByTestId ( 'instance-ai-credential-continue-button' ),
83 previewPanel : () => container (). getByTestId ( 'instance-ai-preview-panel' ),
84 previewCanvasNodes : () =>
85 container (). getByTestId ( 'workflow-canvas-host' ). locator ( '[data-test-id="canvas-node"]' ),
86 previewToggle : () => container (). getByTestId ( 'instance-ai-artifacts-preview-toggle' ),
87 artifactsSidebar : () => container (). getByTestId ( 'instance-ai-artifacts-sidebar' ),
88 outOfCredits : () => container (). getByTestId ( 'instance-ai-out-of-credits' ),
89 };
90 }
91
92 export function createKit ({ page , out , baseUrl , autoConfirm , emit }) {
93 const started = Date. now ();
94 const steps = [];
95 const threads = [];
96 const prompts = [];
97 const ai = locators (page);
98 let stopped = false ;
99 mkdirSync ( join (out, 'captures' ), { recursive: true });
100
101 const seconds = () => (Date. now () - started) / 1000 ;
102 function save () {
103 const partial = join (out, 'steps.tmp.json' );
104 writeFileSync (partial, JSON . stringify (steps, null , 1 ));
105 renameSync (partial, join (out, 'steps.json' ));
106 }
107 async function shot ( file ) {
108 try {
109 await page. screenshot ({ path: join (out, file) });
110 return file;
111 } catch {
112 return null ;
113 }
114 }
115 function message ( error ) {
116 return String (error?.message ?? error)
117 . replace ( / \u001b \[ [0-9;] * m/ g , '' )
118 . slice ( 0 , 2000 );
119 }
120
121 async function record ( kind , title , fn ) {
122 if (stopped) throw new StepFailed ( 'The scenario has stopped.' );
123 const entry = { i: steps. length + 1 , kind, title, status: 'running' , t: seconds () };
124 steps. push (entry);
125 save ();
126 emit ( `${ kind } ${ title }` );
127 let value;
128 let failed = false ;
129 try {
130 value = typeof fn === 'function' ? await fn () : fn;
131 if (value === false ) throw new Error ( 'The check returned false.' );
132 } catch (error) {
133 failed = true ;
134 entry.error = message (error);
135 }
136 // After a stop, run_scenario.mjs owns the steps. A late result must not change them.
137 if (stopped) throw new StepFailed ( 'The scenario has stopped.' );
138 entry.status = failed ? 'failed' : 'passed' ;
139 if (failed) emit ( `${ kind } failed: ${ entry . error . split ( ' \n ' )[ 0 ] }` );
140 entry.duration = seconds () - entry.t;
141 entry.shot = await shot ( `${ kind }-${ String ( entry . i ). padStart ( 2 , '0' ) }.png` );
142 if ( ! stopped) save ();
143 if (failed && kind === 'step' ) throw new StepFailed (entry.error);
144 return failed ? undefined : value;
145 }
146
147 async function api ( method , path , body ) {
148 return await page. evaluate (
149 async ({ method , path , body }) => {
150 const res = await fetch (path, {
151 method,
152 headers: {
153 'content-type' : 'application/json' ,
154 'browser-id' : localStorage. getItem ( 'n8n-browserId' ) ?? '' ,
155 },
156 body: body === undefined ? undefined : JSON . stringify (body),
157 });
158 const text = await res. text ();
159 if ( ! res.ok) throw new Error ( `${ method } ${ path } -> ${ res . status }: ${ text . slice ( 0 , 300 ) }` );
160 const json = text ? JSON . parse (text) : null ;
161 return json && typeof json === 'object' && Object. keys (json). length === 1 && 'data' in json
162 ? json.data
163 : json;
164 },
165 { method, path, body },
166 );
167 }
168
169 function currentThread () {
170 return page. url (). match ( THREAD_URL )?.[ 1 ] ?? null ;
171 }
172
173 function noteThread ( id ) {
174 if (id && ! threads. includes (id)) {
175 threads. push (id);
176 emit ( `thread ${ id }` );
177 }
178 }
179
180 async function threadStatus ( id ) {
181 const s = await api ( 'GET' , `/rest/instance-ai/threads/${ id }/status` );
182 const background = (s?.backgroundTasks ?? []). filter (( x ) => x.status === 'running' ). length ;
183 return { active: Boolean (s?.hasActiveRun), suspended: Boolean (s?.isSuspended), background };
184 }
185
186 const t = {
187 page,
188 ai,
189 expect,
190 baseUrl,
191 threads,
192 prompts,
193
194 /** A step. When it throws, the scenario ends as failed. */
195 step : ( title , fn ) => record ( 'step' , title, fn),
196 /** A soft check. A failure is recorded and the scenario goes on. fn can return false to fail. */
197 check : ( title , fn ) => record ( 'check' , title, fn),
198 /** A line in the step list, with a screenshot. */
199 note : ( title ) => record ( 'note' , title, () => true ),
200
201 /** Open a page of the instance. Default: the Assistant, with the composer ready. */
202 async goto ( path = '/assistant' ) {
203 await page. goto (baseUrl + path, { waitUntil: 'domcontentloaded' });
204 if (path. startsWith ( '/assistant' )) {
205 const enable = page. getByRole ( 'dialog' ). filter ({ hasText: 'Try new n8n Assistant' });
206 if ( await enable. isVisible ({ timeout: 1500 }). catch (() => false )) {
207 await enable
208 . getByRole ( 'button' , { name: /Enable n8n Assistant on this instance/ })
209 . click ();
210 await enable. getByRole ( 'button' , { name: / ^ (Continue | Enable) $ / }). click ();
211 }
212 await expect (ai. chatInput ()). toBeVisible ({ timeout: 30_000 });
213 }
214 noteThread ( currentThread ());
215 },
216
217 /** Type a message in the composer and send it. Returns the thread id. */
218 async send ( text ) {
219 const before = await ai. assistantMessages (). count ();
220 await expect (ai. chatInput ()). toBeVisible ({ timeout: 30_000 });
221 await ai. chatInput (). fill (text);
222 await expect (ai. sendButton ()). toBeEnabled ({ timeout: 30_000 });
223 await ai. sendButton (). click ();
224 await page. waitForURL ( THREAD_URL , { timeout: 30_000 });
225 const id = currentThread ();
226 noteThread (id);
227 prompts. push ({
228 t: seconds (),
229 text,
230 thread: id,
231 assistantMessagesBefore: before,
232 sentAt: Date. now (),
233 });
234 return id;
235 },
236
237 /**
238 * Wait until the thread has no active run, no open confirmation, and no running background task.
239 * Returns 'idle', or 'suspended' when a confirmation card waits for the scenario
240 * (autoConfirm is off, or the card stays open for 20 seconds).
241 */
242 async waitForIdle ({ timeout = 600_000 , settle = 3000 } = {}) {
243 const id = currentThread () ?? threads. at ( - 1 );
244 if ( ! id) throw new Error ( 'No thread is open. Call t.send() first.' );
245 const last = prompts. filter (( p ) => p.thread === id). at ( - 1 );
246 const deadline = Date. now () + timeout;
247 let seenRun = false ;
248 let idleSince = null ;
249 let suspendedSince = null ;
250 while ( ! stopped && Date. now () < deadline) {
251 let s;
252 try {
253 s = await threadStatus (id);
254 } catch {
255 await page. waitForTimeout ( 1000 );
256 continue ;
257 }
258 // The status can show no run for a moment after the message is sent. A run that the user
259 // stops at once can end before it shows, so after a grace period "no run" means "done".
260 if ( ! seenRun && Date. now () - (last?.sentAt ?? 0 ) > RUN_GRACE_MS ) {
261 seenRun = true ;
262 }
263 if ( ! seenRun) {
264 const replies = await ai
265 . assistantMessages ()
266 . count ()
267 . catch (() => 0 );
268 seenRun =
269 s.active ||
270 s.suspended ||
271 s.background > 0 ||
272 replies > (last?.assistantMessagesBefore ?? 0 );
273 }
274 if (s.suspended) {
275 suspendedSince ??= Date. now ();
276 if ( ! autoConfirm || Date. now () - suspendedSince > 20_000 ) return 'suspended' ;
277 } else {
278 suspendedSince = null ;
279 }
280 const idle = seenRun && ! s.active && ! s.suspended && s.background === 0 ;
281 if ( ! idle) {
282 idleSince = null ;
283 } else {
284 idleSince ??= Date. now ();
285 if (Date. now () - idleSince >= settle) {
286 await ai
287 . sendButton ()
288 . waitFor ({ state: 'visible' , timeout: 10_000 })
289 . catch (() => {});
290 return 'idle' ;
291 }
292 }
293 await page. waitForTimeout ( 1000 );
294 }
295 throw new Error ( `The thread did not go idle in ${ Math . round ( timeout / 1000 ) }s.` );
296 },
297
298 /**
299 * Save a screen for the visual check. Visual scenarios run on this branch and on master, and the
300 * report compares the screens with the same name. mask hides changing parts, such as LLM text.
301 * root is the part of the page for the accessibility tree diff (default: the whole page).
302 */
303 async capture ( name , { mask = [], maskText = [], root , fullPage = false } = {}) {
304 if ( ! / ^ [a-z0-9][a-z0-9-] *$ / . test (name))
305 throw new Error ( `Capture name "${ name }": use a-z, 0-9 and "-".` );
306 return await record ( 'capture' , `Capture ${ name }` , async () => {
307 await page. waitForLoadState ( 'networkidle' , { timeout: 5000 }). catch (() => {});
308 await page. waitForTimeout ( 500 );
309 const texts = new Set (maskText. filter (Boolean));
310 for ( const locator of mask) {
311 for ( const inner of await locator. allInnerTexts (). catch (() => [])) {
312 for ( const line of inner. split ( ' \n ' ))
313 if (line. trim (). length > 1 ) texts. add (line. trim ());
314 }
315 }
316 const textMasks = maskText. filter (Boolean). map (( s ) => page. getByText (s, { exact: true }));
317 await page. screenshot ({
318 path: join (out, 'captures' , `${ name }.png` ),
319 mask: [ ... mask, ... textMasks],
320 maskColor: '#FF00FF' ,
321 animations: 'disabled' ,
322 caret: 'hide' ,
323 fullPage,
324 });
325 const tree = await (root ?? page. locator ( 'body' ))
326 . ariaSnapshot ({ timeout: 10_000 })
327 . catch (( e ) => `# ${ message ( e ) }` );
328 const subtrees = [];
329 for ( const locator of mask) {
330 for ( const element of await locator. all (). catch (() => [])) {
331 subtrees. push ( await element. ariaSnapshot ({ timeout: 5_000 }). catch (() => '' ));
332 }
333 }
334 writeFileSync (
335 join (out, 'captures' , `${ name }.aria.yml` ),
336 maskTree ( maskSubtrees (tree, subtrees), texts) + ' \n ' ,
337 );
338 });
339 },
340
341 /** A REST call as the signed-in user. Returns the `data` field of the answer when it has one. */
342 api,
343 threadStatus,
344 sleep : ( ms ) => page. waitForTimeout (ms),
345 };
346 /** End the scenario: later steps throw, and a running step does not change the steps. */
347 const stop = () => {
348 stopped = true ;
349 };
350 return { t, steps, threads, prompts, stop };
351 }