Setting the file. One moment. File Cu Canary
— line 313
This file
- Number
- 10.44
- Position
- 44 of 77
- Type
- Python
- Size
- 14 KB
- Lines
- 353
helpers/document_assess.py
Python·353 lines·14 KB
subprocess
13import sys
14import threading
15
16PARSER_VERSION = "6.8.0"
17MAX_INPUT = 16 * 1024 * 1024
18MAX_PAGES = 200
19MAX_SELECTED = 8
20MAX_OUTPUT = 8192
21MAX_REQUEST = 16384
22WALL_SECONDS = 15
23FORMATS = {"auto", "pdf", "text", "markdown", "html", "json", "docx", "pptx",
24 "xlsx", "png", "jpeg"}
25
26
27class Blocked(ValueError):
28 """A public, content-free blocker code."""
29
30
31def blocked(code):
32 return {"schema_version": "1.0", "status": "blocked", "assessment": "not-assessed", "code": code,
33 "service_admission": "not-assessed", "facts": None}
34
35
36def validate_request(request):
37 if not isinstance(request, dict) or request.get("approved") is not True:
38 raise Blocked("inspection-not-approved")
39 if not {"approved", "path", "sha256"} <= set(request) or set(request) - {
40 "approved", "path", "sha256", "pages", "whole_document", "format"
41 }:
42 raise Blocked("invalid-request")
43 path, digest, pages = request["path"], request["sha256"], request.get("pages")
44 if isinstance(path, str) and path.lower().startswith(("http:", "https:", "abfs:", "abfss:")):
45 raise Blocked("remote-sample-access-not-supported")
46 if (not isinstance(path, str) or len(path) > 4096 or "\0" in path
47 or not Path(path).is_absolute()):
48 raise Blocked("invalid-source-path")
49 if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
50 raise Blocked("invalid-source-identity")
51 if (pages is not None and (not isinstance(pages, list) or not 1 <= len(pages) <= MAX_SELECTED
52 or any(type(page) is not int or not 1 <= page <= MAX_PAGES for page in pages)
53 or len(set(pages)) != len(pages))):
54 raise Blocked("invalid-page-selection")
55 if type(request.get("whole_document", False)) is not bool or (
56 (pages is not None) == request.get("whole_document", False)
57 ):
58 raise Blocked("explicit-document-scope-required")
59 if not isinstance(request.get("format", "auto"), str) or request.get("format", "auto") not in FORMATS:
60 raise Blocked("unsupported-format-hint")
61
62
63def read_source(request):
64 path = Path(request["path"])
65 if sys.platform == "win32":
66 import ctypes
67
68 # Reject device namespaces, UNC, ADS and mapped network drives.
69 if (str(path).startswith("\\\\") or ":" in str(path)[2:]
70 or ctypes.windll.kernel32.GetDriveTypeW(str(path.anchor)) not in (3, 6)):
71 raise Blocked("nonlocal-source")
72 if any(item.is_symlink() or (
73 getattr(item.lstat(), "st_file_attributes", 0) & 0x400
74 ) for item in (path, *path.parents)):
75 raise Blocked("source-link-unsupported")
76 canonical = path.resolve(strict=True)
77 with canonical.open("rb") as source:
78 before = os.fstat(source.fileno())
79 if not stat.S_ISREG(before.st_mode):
80 raise Blocked("nonregular-source")
81 if before.st_size > MAX_INPUT:
82 raise Blocked("input-limit")
83 data = source.read(MAX_INPUT + 1)
84 after = os.fstat(source.fileno())
85 if len(data) > MAX_INPUT:
86 raise Blocked("input-limit")
87 if (before.st_size, before.st_mtime_ns, before.st_ino) != (
88 after.st_size, after.st_mtime_ns, after.st_ino
89 ) or len(data) != before.st_size:
90 raise Blocked("source-changed")
91 if hashlib.sha256(data).hexdigest() != request["sha256"]:
92 raise Blocked("source-identity-mismatch")
93 binding = hashlib.sha256(os.path.normcase(str(canonical)).encode("utf-8")).hexdigest()
94 return data, binding
95
96
97def deny_document_reads(event, args):
98 if event in {"open", "os.listdir", "os.scandir"}:
99 raise PermissionError("Document external read denied")
100
101
102def assess_bytes(data, selected):
103 import codecs
104 import logging
105 import warnings
106
107 try:
108 import pypdf
109 except ImportError:
110 raise Blocked("parser-dependency-missing") from None
111 if pypdf.__version__ != PARSER_VERSION:
112 raise Blocked("parser-version-unsupported")
113 from pypdf.errors import PdfReadError, PdfStreamError
114 from pypdf.generic import ContentStream
115
116 class RejectWarning(logging.Handler):
117 def emit(self, record):
118 raise Blocked("pdf-parser-warning")
119
120 logger = logging.getLogger("pypdf")
121 logger.handlers = [RejectWarning()]
122 logger.propagate = False
123 logger.setLevel(logging.WARNING)
124 warnings.simplefilter("error")
125 for encoding in ("charmap", "utf-16-be", "utf-16-le", "utf-8", "latin-1", "ascii"):
126 codecs.lookup(encoding)
127 # Parser imports are trusted installation reads; once bytes are supplied,
128 # no PDF-directed file read (even local) is permitted.
129 sys.addaudithook(deny_document_reads)
130 if not data.startswith(b"%PDF-") or not data.rstrip().endswith(b"%%EOF"):
131 raise Blocked("invalid-pdf")
132 try:
133 reader = pypdf.PdfReader(io.BytesIO(data), strict=True)
134 if reader.is_encrypted:
135 raise Blocked("encrypted-pdf")
136 page_count = len(reader.pages)
137 if page_count > MAX_PAGES:
138 raise Blocked("page-limit")
139 if max(selected) > page_count:
140 raise Blocked("page-out-of-range")
141 pages = []
142 for number in selected:
143 page = reader.pages[number - 1]
144 # Direct stream operators, not object inventory or rendered entities.
145 content = page.get_contents()
146 operations = ContentStream(content, reader).operations if content is not None else []
147 images, drawings, forms = 0, 0, 0
148 for operands, operator in operations:
149 if operator == b"INLINE IMAGE":
150 images += 1
151 elif operator in {b"S", b"s", b"f", b"F", b"f*", b"B", b"B*", b"b", b"b*"}:
152 drawings += 1
153 elif operator == b"Do":
154 obj = page["/Resources"]["/XObject"][operands[0]].get_object()
155 subtype = obj["/Subtype"]
156 if subtype == "/Image":
157 images += 1
158 elif subtype == "/Form":
159 forms += 1
160 else:
161 raise Blocked("unsupported-paint-object")
162 # Forms can reference other pages/resources. Do not traverse them to
163 # estimate modality; report text unknown instead of widening scope.
164 characters = None if forms else len(page.extract_text())
165 pages.append({
166 "page": number,
167 "text_characters": characters,
168 "text_available": None if characters is None else characters > 0,
169 "text_status": "not-assessed-form-content" if forms else "assessed",
170 "direct_image_paints": images,
171 "direct_path_paints": drawings,
172 "form_invocations": forms,
173 "total_images": None,
174 "total_drawings": None,
175 "table_structure": "not-assessed",
176 "layout_relationships": "not-assessed",
177 })
178 return {"page_count": page_count, "pages": pages, "input_profile": "unknown",
179 "answerability": "not-assessed", "ocr_need": "not-assessed"}
180 except (PdfReadError, PdfStreamError, KeyError, IndexError, TypeError,
181 UnicodeError, NotImplementedError):
182 raise Blocked("pdf-parse-unsupported") from None
183 except Warning:
184 raise Blocked("pdf-parser-warning") from None
185 except PermissionError:
186 raise Blocked("pdf-external-read-blocked") from None
187
188
189def dispatch(data, request):
190 hint = request.get("format", "auto")
191 if data.startswith(b"%PDF-"):
192 if hint not in {"auto", "pdf"}:
193 raise Blocked("format-hint-mismatch")
194 if request.get("pages") is None:
195 raise Blocked("pdf-page-selection-required")
196 return "pdf", f"pypdf=={PARSER_VERSION}", assess_bytes(data, request["pages"])
197 if request.get("pages") is not None:
198 raise Blocked("non-pdf-requires-whole-document-scope")
199 from _document_adapters import AdapterBlocked, assess_document
200
201 try:
202 return assess_document(data, hint, deny_document_reads)
203 except AdapterBlocked as error:
204 raise Blocked(str(error)) from None
205
206
207def worker():
208 # Isolated mode excludes ambient PYTHONPATH; add only this packaged helper.
209 sys.path.insert(0, str(Path(__file__).resolve().parent))
210 from _document_limits import deny_side_effects, install_limits
211
212 try:
213 job_handle = install_limits()
214 except (OSError, ValueError):
215 return blocked("worker-limits-unavailable")
216 sys.addaudithook(deny_side_effects)
217 binding = {}
218 try:
219 raw = sys.stdin.buffer.read(MAX_REQUEST + 1)
220 if len(raw) > MAX_REQUEST:
221 raise Blocked("request-limit")
222 request = json.loads(raw)
223 validate_request(request)
224 data, path_digest = read_source(request)
225 binding = {"source_sha256": request["sha256"], "path_sha256": path_digest}
226 kind, parser, facts = dispatch(data, request)
227 return {"schema_version": "1.0", "status": "completed", "assessment": "assessed",
228 "service_admission": "not-assessed", "format": kind, "parser": parser,
229 **binding, "facts": facts}
230 except Blocked as error:
231 return dict(blocked(str(error)), **binding)
232 except MemoryError:
233 return dict(blocked("worker-memory-limit"), **binding)
234 except RecursionError:
235 return dict(blocked("document-recursion-limit"), **binding)
236 except (OSError, ValueError, OverflowError):
237 return dict(blocked("document-input-or-parse-failed"), **binding)
238 finally:
239 # This local deliberately remains live through parsing on Windows.
240 _ = job_handle
241
242
243def run_worker(command, request, timeout=WALL_SECONDS):
244 """Pipe-only IPC; bounded output reader, no sample or stderr artifacts."""
245 encoded = json.dumps(request, separators=(",", ":")).encode()
246 if len(encoded) > MAX_REQUEST:
247 return blocked("request-limit")
248 output = bytearray()
249 exceeded = threading.Event()
250 io_failed = threading.Event()
251 with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
252 stderr=subprocess.DEVNULL) as process:
253 def drain():
254 try:
255 chunk = process.stdout.read(MAX_OUTPUT + 1)
256 if len(chunk) > MAX_OUTPUT:
257 exceeded.set()
258 process.kill()
259 else:
260 output.extend(chunk)
261 except OSError:
262 io_failed.set()
263
264 def send():
265 try:
266 process.stdin.write(encoded)
267 process.stdin.close()
268 except OSError:
269 io_failed.set()
270
271 thread = threading.Thread(target=drain, daemon=True)
272 writer = threading.Thread(target=send, daemon=True)
273 thread.start()
274 writer.start()
275 try:
276 process.wait(timeout=timeout)
277 except subprocess.TimeoutExpired:
278 process.kill()
279 process.wait()
280 return blocked("worker-timeout")
281 finally:
282 if process.poll() is None:
283 process.kill()
284 process.wait()
285 thread.join()
286 writer.join()
287 if exceeded.is_set():
288 return blocked("worker-output-limit")
289 if process.returncode not in (0, 2):
290 return blocked("worker-failed-or-resource-limit")
291 if io_failed.is_set():
292 return blocked("worker-io-failed")
293 try:
294 result = json.loads(output)
295 except (ValueError, UnicodeError):
296 return blocked("worker-invalid-output")
297 if not isinstance(result, dict) or result.get("status") not in {"completed", "blocked"}:
298 return blocked("worker-invalid-output")
299 return result
300
301
302def assess(request):
303 try:
304 validate_request(request)
305 return run_worker([sys.executable, "-I", "-B", str(Path(__file__).resolve()),
306 "--worker"], request)
307 except Blocked as error:
308 return blocked(str(error))
309 except OSError:
310 return blocked("worker-start-failed")
311
312
313class SafeArgumentParser(argparse.ArgumentParser):
314 def error(self, message):
315 raise Blocked("invalid-cli-arguments")
316
317
318def main():
319 def private_failure(exc_type, exc_value, traceback):
320 print(json.dumps(blocked("worker-failed-or-resource-limit")))
321
322 # Unexpected parser/runtime failures must not print attacker-controlled
323 # exception strings or stack traces, including when --worker is invoked.
324 sys.excepthook = private_failure
325 if sys.argv[1:] == ["--worker"]:
326 result = worker()
327 else:
328 parser = SafeArgumentParser(description=__doc__)
329 parser.add_argument("--approve-inspection", action="store_true")
330 parser.add_argument("--file")
331 parser.add_argument("--sha256")
332 parser.add_argument("--pages", help="Explicit 1-based pages, e.g. 1,3; no ranges")
333 parser.add_argument("--whole-document", action="store_true",
334 help="Approve whole bounded non-PDF document, not rendered pages")
335 parser.add_argument("--format", choices=sorted(FORMATS), default="auto")
336 try:
337 args = parser.parse_args()
338 if not args.approve_inspection:
339 raise Blocked("inspection-not-approved")
340 pages = None if args.pages is None else [int(value) for value in args.pages.split(",")]
341 result = assess({"approved": True, "path": args.file,
342 "sha256": args.sha256, "pages": pages,
343 "whole_document": args.whole_document, "format": args.format})
344 except Blocked as error:
345 result = blocked(str(error))
346 except ValueError:
347 result = blocked("invalid-page-selection")
348 print(json.dumps(result, separators=(",", ":"), ensure_ascii=True))
349 return 0 if result["status"] == "completed" else 2
350
351
352if __name__ == "__main__":
353 raise SystemExit(main())