Media Fetch

1// Media downloads use public HTTP(S) URLs. Validate every redirect target;
2// a provider result must meet the same host policy as a direct ingest URL.
3// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support.
4// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted.
5
6import { BlockList, isIP } from "node:net";
7
8const blocked = new BlockList();
9for (const [network, prefix] of [
10  ["0.0.0.0", 8],
11  ["10.0.0.0", 8],
12  ["100.64.0.0", 10],
13  ["127.0.0.0", 8],
14  ["169.254.0.0", 16],
15  ["172.16.0.0", 12],
16  ["192.0.0.0", 24],
17  ["192.0.2.0", 24],
18  ["192.88.99.0", 24],
19  ["192.168.0.0", 16],
20  ["198.18.0.0", 15],
21  ["198.51.100.0", 24],
22  ["203.0.113.0", 24],
23  ["224.0.0.0", 4],
24  ["240.0.0.0", 4],
25])
26  blocked.addSubnet(network, prefix, "ipv4");
27for (const [network, prefix] of [
28  ["::", 128],
29  ["::1", 128],
30  ["fc00::", 7],
31  ["fe80::", 10],
32  ["fec0::", 10],
33  ["ff00::", 8],
34  ["2001:db8::", 32],
35])
36  blocked.addSubnet(network, prefix, "ipv6");
37
38export function isPublicMediaUrl(value) {
39  try {
40    const url = new URL(value);
41    if (url.protocol !== "http:" && url.protocol !== "https:") return false;
42    const host = url.hostname.replace(/\.$/, "");
43    if (
44      host === "localhost" ||
45      host.endsWith(".localhost") ||
46      host.endsWith(".local") ||
47      host.endsWith(".internal")
48    )
49      return false;
50    const address = host.replace(/^\[|\]$/g, "");
51    const family = isIP(address);
52    return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6");
53  } catch {
54    return false;
55  }
56}
57
58export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) {
59  let current = String(url);
60  for (let hop = 0; hop <= 5; hop++) {
61    if (!isPublicMediaUrl(current))
62      throw new Error("Media download blocked: URL is not public HTTP(S)");
63    const response = await fetchImpl(current, { method, signal, redirect: "manual" });
64    if (!(response.status >= 300 && response.status < 400)) return response;
65    const location = response.headers.get("location");
66    if (!location) return response;
67    await response.body?.cancel();
68    current = new URL(location, current).href;
69  }
70  throw new Error("Media download exceeded redirect limit");
71}
72
73// Stream and abort once the cap is crossed, so a lying or chunked body can't buffer past it.
74export async function readCappedBody(res, maxBytes, label) {
75  const declared = Number(res.headers.get("content-length"));
76  if (declared > maxBytes) throw new Error(`${label}: ${declared} bytes exceeds ${maxBytes} cap`);
77  const chunks = [];
78  let total = 0;
79  for await (const chunk of res.body) {
80    total += chunk.length;
81    if (total > maxBytes) throw new Error(`${label}: stream exceeds ${maxBytes} cap`);
82    chunks.push(chunk);
83  }
84  return Buffer.concat(chunks, total);
85}