Setting the file. One moment.
Fetch Codex Manual · OpenAI Docs · openai/skills · Skills Docs
Back to the top of the page const readHeaderSha
— line 216
This file
Number 25.4
Position 4 of 8
Type JavaScript
Size 16 KB
Lines 598 scripts/ fetch-codex-manual.mjs
JavaScript · 598 lines · 16 KB
;
13 import { createHash } from "node:crypto" ;
14 import path from "node:path" ;
15 import process from "node:process" ;
16 import { pathToFileURL } from "node:url" ;
17 import { inspect, promisify } from "node:util" ;
18
19 const DEFAULT_MANUAL_URL = "https://developers.openai.com/codex/codex-manual.md" ;
20 const DEFAULT_CACHE_DIR_NAME = "openai-docs-cache" ;
21 const CACHE_FILE_NAME = "codex-manual.md" ;
22 const OUTLINE_FILE_NAME = "codex-manual.outline.md" ;
23 const HASH_HEADER = "x-content-sha256" ;
24 const USER_AGENT = "codex-openai-docs" ;
25 const execFileAsync = promisify (execFile);
26
27 class ManualFetchError extends Error {
28 constructor ( message , options ) {
29 super (message, options);
30 this .name = "ManualFetchError" ;
31 }
32 }
33
34 const sha256 = ( value ) => createHash ( "sha256" ). update (value). digest ( "hex" );
35
36 const withTimeout = async ( promiseFactory , timeoutMs ) => {
37 const controller = new AbortController ();
38 const timeout = setTimeout (() => controller. abort (), timeoutMs);
39 try {
40 return await promiseFactory (controller.signal);
41 } finally {
42 clearTimeout (timeout);
43 }
44 };
45
46 const proxyConfigured = () =>
47 process.env. HTTP_PROXY ||
48 process.env. HTTPS_PROXY ||
49 process.env.http_proxy ||
50 process.env.https_proxy;
51
52 const responseHeaders = ( headers ) => ({
53 get ( name ) {
54 return headers. get (name. toLowerCase ()) ?? null ;
55 },
56 });
57
58 const makeResponse = ({ body , headers , status }) => ({
59 headers: responseHeaders (headers),
60 ok: status >= 200 && status < 300 ,
61 status,
62 async text () {
63 return body;
64 },
65 });
66
67 const parseCurlHeaders = ( rawHeaders ) => {
68 const normalized = rawHeaders. replace ( / \r\n / g , " \n " ). trim ();
69 const blocks = normalized. split ( / \n\n + / ). filter (Boolean);
70 const headerBlock = [ ... blocks]
71 . reverse ()
72 . find (( block ) => block. startsWith ( "HTTP/" ));
73
74 if ( ! headerBlock) {
75 throw new ManualFetchError ( "curl did not return HTTP response headers." );
76 }
77
78 const [ statusLine , ... lines ] = headerBlock. split ( " \n " );
79 const statusMatch = / ^ HTTP \/ \S + \s + ( \d {3} )/ . exec (statusLine);
80 if ( ! statusMatch) {
81 throw new ManualFetchError (
82 `Could not parse HTTP status from curl response: ${ statusLine }`
83 );
84 }
85
86 const headers = new Map ();
87 lines. forEach (( line ) => {
88 const separator = line. indexOf ( ":" );
89 if (separator === - 1 ) return ;
90 const name = line. slice ( 0 , separator). trim (). toLowerCase ();
91 const value = line. slice (separator + 1 ). trim ();
92 headers. set (name, value);
93 });
94
95 return {
96 headers,
97 status: Number (statusMatch[ 1 ]),
98 };
99 };
100
101 const tempFilePath = ( cacheDir , suffix ) =>
102 path. join (
103 cacheDir,
104 `.fetch-codex-manual-${ process . pid }-${ Date . now () }-${ Math . random ()
105 . toString ( 16 )
106 . slice ( 2 ) }${ suffix }`
107 );
108
109 const requestManualWithCurl = async ( url , { cacheDir , method , timeoutMs }) => {
110 const headerPath = tempFilePath (cacheDir, ".headers" );
111 const bodyPath = tempFilePath (cacheDir, ".body" );
112 const curlNames =
113 process.platform === "win32" ? [ "curl.exe" , "curl" ] : [ "curl" ];
114 const args = [
115 "--silent" ,
116 "--show-error" ,
117 "--location" ,
118 "--dump-header" ,
119 headerPath,
120 "--output" ,
121 bodyPath,
122 "--user-agent" ,
123 USER_AGENT ,
124 "--max-time" ,
125 String (Math. max ( 1 , Math. ceil (timeoutMs / 1000 ))),
126 ];
127
128 if (method === "HEAD" ) {
129 args. push ( "--head" );
130 } else {
131 args. push ( "--request" , method);
132 }
133 args. push (url);
134
135 let lastError;
136 for ( const curlName of curlNames) {
137 try {
138 await execFileAsync (curlName, args, { windowsHide: true });
139 const [ rawHeaders , body ] = await Promise . all ([
140 readFile (headerPath, "utf8" ),
141 readFile (bodyPath, "utf8" ),
142 ]);
143 const { headers , status } = parseCurlHeaders (rawHeaders);
144 return makeResponse ({ body, headers, status });
145 } catch (error) {
146 lastError = error;
147 if (error?.code !== "ENOENT" ) break ;
148 } finally {
149 await Promise . all ([
150 rm (headerPath, { force: true }),
151 rm (bodyPath, { force: true }),
152 ]);
153 }
154 }
155
156 if (lastError?.code === "ENOENT" ) {
157 throw new ManualFetchError ( "curl is unavailable in this environment." , {
158 cause: lastError,
159 });
160 }
161 throw new ManualFetchError ( `${ method } ${ url } could not be fetched.` , {
162 cause: lastError,
163 });
164 };
165
166 const requestManualWithFetch = async ( url , { method , timeoutMs }) => {
167 if ( typeof fetch !== "function" ) {
168 throw new ManualFetchError (
169 "Native fetch is unavailable in this Node runtime."
170 );
171 }
172
173 return withTimeout (
174 ( signal ) =>
175 fetch (url, {
176 method,
177 headers: { "User-Agent" : USER_AGENT },
178 signal,
179 }),
180 timeoutMs
181 );
182 };
183
184 const requestManual = async ( url , { cacheDir , method , timeoutMs }) => {
185 const preferCurl = Boolean ( proxyConfigured ()) || typeof fetch !== "function" ;
186 const transports = preferCurl
187 ? [
188 () => requestManualWithCurl (url, { cacheDir, method, timeoutMs }),
189 () => requestManualWithFetch (url, { method, timeoutMs }),
190 ]
191 : [
192 () => requestManualWithFetch (url, { method, timeoutMs }),
193 () => requestManualWithCurl (url, { cacheDir, method, timeoutMs }),
194 ];
195
196 let lastError;
197 for ( const transport of transports) {
198 try {
199 const response = await transport ();
200 if ( ! response.ok) {
201 throw new ManualFetchError (
202 `${ method } ${ url } failed with HTTP ${ response . status }.`
203 );
204 }
205 return response;
206 } catch (error) {
207 lastError = error;
208 }
209 }
210
211 throw new ManualFetchError ( `${ method } ${ url } could not be fetched.` , {
212 cause: lastError,
213 });
214 };
215
216 const readHeaderSha = ( response ) => {
217 const value = response.headers. get ( HASH_HEADER );
218 if ( ! value || ! / ^ [a-f0-9] {64}$ / i . test (value)) {
219 throw new ManualFetchError ( `Manual response is missing ${ HASH_HEADER }.` );
220 }
221 return value. toLowerCase ();
222 };
223
224 const nearestExistingParent = async ( target ) => {
225 let current = target;
226 while ( true ) {
227 try {
228 const info = await stat (current);
229 return info. isDirectory () ? current : null ;
230 } catch (error) {
231 if (error?.code !== "ENOENT" ) return null ;
232 }
233
234 const parent = path. dirname (current);
235 if (parent === current) return null ;
236 current = parent;
237 }
238 };
239
240 const usableCacheDir = async ( cacheDir ) => {
241 if ( ! cacheDir) return null ;
242 const resolved = path. resolve (cacheDir);
243
244 try {
245 const info = await stat (resolved);
246 if ( ! info. isDirectory ()) return null ;
247 } catch (error) {
248 if (error?.code !== "ENOENT" ) return null ;
249 }
250
251 const parent = await nearestExistingParent (resolved);
252 if ( ! parent) return null ;
253
254 try {
255 await access (parent, fsConstants. W_OK | fsConstants. X_OK );
256 } catch {
257 return null ;
258 }
259
260 return resolved;
261 };
262
263 const defaultCacheDirCandidates = () => {
264 const candidates = [];
265 const seen = new Set ();
266 const pushCandidate = ( candidate ) => {
267 if ( ! candidate || seen. has (candidate)) return ;
268 seen. add (candidate);
269 candidates. push (candidate);
270 };
271
272 [process.env. TMPDIR , process.env. TEMP , process.env. TMP ]. forEach (( baseDir ) => {
273 if (baseDir) {
274 pushCandidate (path. join (baseDir, DEFAULT_CACHE_DIR_NAME ));
275 }
276 });
277
278 if (process.platform !== "win32" ) {
279 pushCandidate ( `/private/tmp/${ DEFAULT_CACHE_DIR_NAME }` );
280 pushCandidate ( `/tmp/${ DEFAULT_CACHE_DIR_NAME }` );
281 }
282
283 return candidates;
284 };
285
286 const resolveCacheDir = async ( cacheDir ) => {
287 if (cacheDir) {
288 return usableCacheDir (cacheDir);
289 }
290
291 for ( const candidate of defaultCacheDirCandidates ()) {
292 const usable = await usableCacheDir (candidate);
293 if (usable) return usable;
294 }
295
296 return null ;
297 };
298
299 const cacheFilePath = ( cacheDir ) => path. join (cacheDir, CACHE_FILE_NAME );
300
301 const outlineFilePath = ( cacheDir ) => path. join (cacheDir, OUTLINE_FILE_NAME );
302
303 const manualLines = ( manual ) => {
304 const lines = manual. replace ( / \r\n / g , " \n " ). split ( " \n " );
305 if (lines[lines. length - 1 ] === "" ) lines. pop ();
306 return lines;
307 };
308
309 const sectionTitle = ( rawTitle ) =>
310 rawTitle. replace ( / \s + # + \s *$ / , "" ). replace ( / \s + / g , " " ). trim ();
311
312 const buildOutline = ( manual ) => {
313 const lines = manualLines (manual);
314 const headings = [];
315 let inFence = false ;
316
317 lines. forEach (( line , index ) => {
318 if ( / ^ \s * (``` | ~~~)/ . test (line)) {
319 inFence = ! inFence;
320 return ;
321 }
322 if (inFence) return ;
323
324 const match = / ^ (# {1,6} ) \s + ( . +? ) \s *$ / . exec (line);
325 if ( ! match) return ;
326
327 const level = match[ 1 ]. length ;
328 if (level < 2 || level > 3 ) return ;
329
330 headings. push ({
331 level,
332 title: sectionTitle (match[ 2 ]),
333 startLine: index + 1 ,
334 endLine: lines. length ,
335 });
336 });
337
338 for ( let index = 0 ; index < headings. length ; index += 1 ) {
339 const heading = headings[index];
340 const nextPeer = headings
341 . slice (index + 1 )
342 . find (( candidate ) => candidate.level <= heading.level);
343 if (nextPeer) {
344 heading.endLine = nextPeer.startLine - 1 ;
345 }
346 }
347
348 if (headings. length === 0 ) {
349 return {
350 headingCount: 0 ,
351 lineCount: lines. length ,
352 text: "No markdown headings found." ,
353 };
354 }
355
356 const minLevel = Math. min ( ... headings. map (( heading ) => heading.level));
357 return {
358 headingCount: headings. length ,
359 lineCount: lines. length ,
360 text: headings
361 . map (( heading ) => {
362 const indent = " " . repeat (heading.level - minLevel);
363 return `${ indent }- ${ heading . title } (lines ${ heading . startLine }-${ heading . endLine })` ;
364 })
365 . join ( " \n " ),
366 };
367 };
368
369 const outlineMarkdown = ( outline ) => `# Codex Manual Outline \n\n ${ outline . text } \n ` ;
370
371 const manualStatusLine = ( status ) =>
372 status.cacheStatus === "hit"
373 ? "Manual status: local manual was already current."
374 : "Manual status: local manual was updated." ;
375
376 const formatResult = ({ status , outlineText }) =>
377 [
378 `Manual path: ${ status . manualPath }` ,
379 `Outline path: ${ status . outlinePath }` ,
380 manualStatusLine (status),
381 "" ,
382 outlineText,
383 ]. join ( " \n " );
384
385 const readCachedManual = async ( cacheDir , expectedSha256 ) => {
386 try {
387 const manual = await readFile ( cacheFilePath (cacheDir), "utf8" );
388 return sha256 (manual) === expectedSha256 ? manual : null ;
389 } catch {
390 return null ;
391 }
392 };
393
394 const writeCachedManual = async ( cacheDir , manual ) => {
395 await mkdir (cacheDir, { recursive: true });
396 const tmpPath = tempFilePath (cacheDir, `.${ CACHE_FILE_NAME }.tmp` );
397 await writeFile (tmpPath, manual, "utf8" );
398 await rename (tmpPath, cacheFilePath (cacheDir));
399 };
400
401 const writeOutline = async ( cacheDir , outlineText ) => {
402 await mkdir (cacheDir, { recursive: true });
403 const tmpPath = tempFilePath (cacheDir, `.${ OUTLINE_FILE_NAME }.tmp` );
404 await writeFile (tmpPath, outlineText, "utf8" );
405 await rename (tmpPath, outlineFilePath (cacheDir));
406 };
407
408 const fetchCodexManual = async ({
409 manualUrl = DEFAULT_MANUAL_URL ,
410 cacheDir,
411 timeoutMs = 30000 ,
412 } = {}) => {
413 const resolvedCacheDir = await resolveCacheDir (cacheDir);
414 if ( ! resolvedCacheDir) {
415 throw new ManualFetchError (
416 "Manual cache directory is unavailable; pass --cache-dir to override or use OpenAI Docs MCP fallback."
417 );
418 }
419 await mkdir (resolvedCacheDir, { recursive: true });
420
421 const headResponse = await requestManual (manualUrl, {
422 cacheDir: resolvedCacheDir,
423 method: "HEAD" ,
424 timeoutMs,
425 });
426 const expectedSha256 = readHeaderSha (headResponse);
427 const manualPath = cacheFilePath (resolvedCacheDir);
428 const outlinePath = outlineFilePath (resolvedCacheDir);
429 const checkedAt = new Date (). toISOString ();
430
431 const cachedManual = await readCachedManual (resolvedCacheDir, expectedSha256);
432 if (cachedManual !== null ) {
433 const outline = buildOutline (cachedManual);
434 const outlineText = outlineMarkdown (outline);
435 await writeOutline (resolvedCacheDir, outlineText);
436
437 return {
438 outlineText,
439 status: {
440 manualUrl,
441 headerSha256: expectedSha256,
442 fetchedManualSha256: expectedSha256,
443 manualHashMatches: true ,
444 cacheStatus: "hit" ,
445 cacheDir: resolvedCacheDir,
446 manualPath,
447 outlinePath,
448 checkedAt,
449 lineCount: outline.lineCount,
450 headingCount: outline.headingCount,
451 },
452 };
453 }
454
455 const getResponse = await requestManual (manualUrl, {
456 cacheDir: resolvedCacheDir,
457 method: "GET" ,
458 timeoutMs,
459 });
460 const getHeaderSha256 = readHeaderSha (getResponse);
461 if (getHeaderSha256 !== expectedSha256) {
462 throw new ManualFetchError (
463 `${ HASH_HEADER } changed between HEAD and GET for ${ manualUrl }.`
464 );
465 }
466
467 const manualText = await getResponse. text ();
468 const actualSha256 = sha256 (manualText);
469 const manualHashMatches = actualSha256 === expectedSha256;
470 if ( ! manualHashMatches) {
471 throw new ManualFetchError (
472 `${ HASH_HEADER } did not match the fetched manual body for ${ manualUrl }.`
473 );
474 }
475
476 await writeCachedManual (resolvedCacheDir, manualText);
477 const outline = buildOutline (manualText);
478 const outlineText = outlineMarkdown (outline);
479 await writeOutline (resolvedCacheDir, outlineText);
480
481 return {
482 outlineText,
483 status: {
484 manualUrl,
485 headerSha256: expectedSha256,
486 fetchedManualSha256: actualSha256,
487 manualHashMatches,
488 cacheStatus: "updated" ,
489 cacheDir: resolvedCacheDir,
490 manualPath,
491 outlinePath,
492 checkedAt,
493 lineCount: outline.lineCount,
494 headingCount: outline.headingCount,
495 },
496 };
497 };
498
499 const parseArgs = ( argv ) => {
500 const args = {
501 manualUrl: DEFAULT_MANUAL_URL ,
502 cacheDir: undefined ,
503 timeoutMs: 30000 ,
504 statusJson: false ,
505 };
506
507 for ( let index = 0 ; index < argv. length ; index += 1 ) {
508 const arg = argv[index];
509 if (arg === "--manual-url" ) {
510 args.manualUrl = argv[ ++ index];
511 } else if (arg === "--cache-dir" ) {
512 args.cacheDir = argv[ ++ index];
513 } else if (arg === "--timeout-ms" ) {
514 args.timeoutMs = Number (argv[ ++ index]);
515 } else if (arg === "--status-json" ) {
516 args.statusJson = true ;
517 } else {
518 throw new ManualFetchError ( `Unknown argument: ${ arg }` );
519 }
520 }
521
522 if ( ! args.manualUrl) {
523 throw new ManualFetchError ( "--manual-url cannot be empty." );
524 }
525 if ( ! Number. isFinite (args.timeoutMs) || args.timeoutMs <= 0 ) {
526 throw new ManualFetchError ( "--timeout-ms must be a positive number." );
527 }
528
529 return args;
530 };
531
532 const main = async () => {
533 const args = parseArgs (process.argv. slice ( 2 ));
534 const { outlineText , status } = await fetchCodexManual (args);
535
536 process.stdout. write ( formatResult ({ status, outlineText }));
537
538 if (args.statusJson) {
539 console. error ( JSON . stringify (status));
540 }
541 };
542
543 const envProxyHint = () => {
544 if ( proxyConfigured ()) {
545 return "Hint: proxy env vars are present. This helper prefers `curl` in proxied sessions; if requests still fail, verify `curl` is installed and the proxy configuration is valid." ;
546 }
547 if ( typeof fetch !== "function" ) {
548 return "Hint: native fetch is unavailable in this Node runtime. Install `curl` or use a newer Node version to fetch the manual." ;
549 }
550 if (process.platform === "win32" ) {
551 return "Hint: on Windows, pass a cache dir under `%TEMP%` or `%TMP%`." ;
552 }
553 return null ;
554 };
555
556 const formatErrorDetails = ( error ) => {
557 const details = inspect (error, {
558 breakLength: 120 ,
559 colors: false ,
560 compact: false ,
561 depth: 8 ,
562 });
563 if ( ! error?.cause) {
564 return details;
565 }
566
567 return `${ details } \n\n Cause: \n ${ inspect ( error . cause , {
568 breakLength: 120 ,
569 colors: false ,
570 compact: false ,
571 depth: 8 ,
572 }) }` ;
573 };
574
575 const isCliEntrypoint = () => {
576 const entrypoint = process.argv[ 1 ];
577 if ( ! entrypoint) {
578 return false ;
579 }
580
581 return pathToFileURL (entrypoint).href === import . meta .url;
582 };
583
584 if ( isCliEntrypoint ()) {
585 main (). catch (( error ) => {
586 console. error ( `Error: ${ error . message }` );
587 const hint = envProxyHint ();
588 if (hint) {
589 console. error (hint);
590 }
591 console. error ( "" );
592 console. error ( "Details:" );
593 console. error ( formatErrorDetails (error));
594 process.exitCode = 1 ;
595 });
596 }
597
598 export { DEFAULT_MANUAL_URL, fetchCodexManual };