Setting the file. One moment. File Cu Canary
— line 170
This file
- Number
- 10.28
- Position
- 28 of 77
- Type
- Python
- Size
- 15 KB
- Lines
- 336
helpers/_document_adapters.py
Python·336 lines·15 KB
12
import
zipfile
13import zlib
14
15XML_VERSION = "0.7.1"
16MAX_TEXT = 2 * 1024 * 1024
17MAX_ENTRIES = 256
18MAX_INFLATED = 16 * 1024 * 1024
19MAX_RATIO = 100
20MAX_NODES = 100_000
21MAX_DEPTH = 64
22MAX_IMAGE_PIXELS = 100_000_000
23CT = "http://schemas.openxmlformats.org/package/2006/content-types"
24W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
25P = "http://schemas.openxmlformats.org/presentationml/2006/main"
26A = "http://schemas.openxmlformats.org/drawingml/2006/main"
27S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
28OFFICE = {
29 "docx": ("/word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"),
30 "pptx": ("/ppt/presentation.xml", "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"),
31 "xlsx": ("/xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"),
32}
33
34
35class AdapterBlocked(ValueError):
36 """Fixed content-free reason; callers never forward parser exceptions."""
37
38
39def facts(**values):
40 return {
41 "page_count": "not-applicable",
42 "text_characters": "not-assessed", "text_available": "not-assessed",
43 "total_images": "not-assessed", "total_drawings": "not-assessed",
44 "table_structure": "not-assessed", "layout_relationships": "not-assessed",
45 "input_profile": "unknown", "answerability": "not-assessed",
46 "ocr_need": "not-assessed", **values,
47 }
48
49
50def text_facts(count, basis, **values):
51 return facts(text_characters=count, text_available=count > 0, text_basis=basis, **values)
52
53
54class HTMLCounts(HTMLParser):
55 def __init__(self):
56 super().__init__(convert_charrefs=True)
57 self.characters = 0
58 self.tags = {"img": 0, "table": 0, "tr": 0, "td": 0, "th": 0}
59 self.suppressed = []
60 self.nodes = 0
61
62 def handle_starttag(self, tag, attrs):
63 self.nodes += 1
64 if self.nodes > MAX_NODES:
65 raise AdapterBlocked("document-node-limit")
66 if tag in {"script", "style"}:
67 self.suppressed.append(tag)
68 if not self.suppressed and tag in self.tags:
69 self.tags[tag] += 1
70
71 def handle_endtag(self, tag):
72 if self.suppressed and self.suppressed[-1] == tag:
73 self.suppressed.pop()
74
75 def handle_data(self, data):
76 if not self.suppressed:
77 self.characters += len(data)
78
79
80def assess_text(data, hint):
81 if len(data) > MAX_TEXT:
82 raise AdapterBlocked("text-input-limit")
83 try:
84 text = data.decode("utf-8-sig", errors="strict")
85 except UnicodeError:
86 raise AdapterBlocked("unsupported-text-encoding-or-format") from None
87 if any(ord(char) < 32 and char not in "\t\r\n" for char in text):
88 raise AdapterBlocked("unsupported-binary-format")
89 kind = "text" if hint == "auto" else hint
90 if kind not in {"text", "markdown", "html", "json"}:
91 raise AdapterBlocked("format-hint-mismatch")
92 if kind == "json":
93 def reject_constant(value):
94 raise AdapterBlocked("nonstandard-json")
95
96 def unique_object(pairs):
97 result = {}
98 for key, value in pairs:
99 if key in result:
100 raise AdapterBlocked("duplicate-json-key")
101 result[key] = value
102 return result
103
104 try:
105 value = json.loads(text, parse_constant=reject_constant, object_pairs_hook=unique_object)
106 except AdapterBlocked:
107 raise
108 except (ValueError, UnicodeError):
109 raise AdapterBlocked("invalid-json") from None
110 count = nodes = objects = arrays = keys = 0
111 pending = [(value, 1)]
112 while pending:
113 value, depth = pending.pop()
114 nodes += 1
115 if nodes > MAX_NODES or depth > MAX_DEPTH:
116 raise AdapterBlocked("document-node-or-depth-limit")
117 if isinstance(value, str):
118 count += len(value)
119 elif isinstance(value, dict):
120 objects += 1
121 keys += len(value)
122 pending.extend((item, depth + 1) for item in value.values())
123 elif isinstance(value, list):
124 arrays += 1
125 pending.extend((item, depth + 1) for item in value)
126 return kind, "stdlib-json", text_facts(count, "string-values-only",
127 objects=objects, arrays=arrays, keys=keys)
128 if kind == "html":
129 parser = HTMLCounts()
130 parser.feed(text)
131 parser.close()
132 return kind, "stdlib-HTMLParser", text_facts(
133 parser.characters, "data-events-excluding-script-style",
134 html_elements=parser.tags, validation="tolerant-syntax-not-rendering")
135 return kind, "stdlib-utf8", text_facts(
136 len(text), "decoded-characters-including-whitespace-and-markup",
137 markdown_structure="not-assessed" if kind == "markdown" else "not-applicable")
138
139
140def xml_parser():
141 try:
142 import defusedxml
143 from defusedxml.ElementTree import fromstring
144 from defusedxml.common import DefusedXmlException
145 from xml.etree.ElementTree import ParseError
146 except ImportError:
147 raise AdapterBlocked("office-parser-dependency-missing") from None
148 if defusedxml.__version__ != XML_VERSION:
149 raise AdapterBlocked("office-parser-version-unsupported")
150 return fromstring, (DefusedXmlException, ParseError)
151
152
153def bounded_xml(data, parse, errors):
154 if len(data) > MAX_TEXT:
155 raise AdapterBlocked("archive-entry-limit")
156 try:
157 root = parse(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)
158 except errors:
159 raise AdapterBlocked("unsafe-or-invalid-office-xml") from None
160 pending, nodes = [(root, 1)], 0
161 while pending:
162 element, depth = pending.pop()
163 nodes += 1
164 if nodes > MAX_NODES or depth > MAX_DEPTH:
165 raise AdapterBlocked("document-node-or-depth-limit")
166 pending.extend((child, depth + 1) for child in element)
167 return root, nodes
168
169
170def assess_office(data, hint, parse, errors):
171 try:
172 with zipfile.ZipFile(io.BytesIO(data)) as archive:
173 entries = archive.infolist()
174 if len(entries) > MAX_ENTRIES:
175 raise AdapterBlocked("archive-entry-count-limit")
176 names, total = set(), 0
177 for entry in entries:
178 path = PurePosixPath(entry.filename)
179 if (entry.filename in names or path.is_absolute() or ".." in path.parts
180 or "\\" in entry.filename or ":" in entry.filename):
181 raise AdapterBlocked("unsafe-or-duplicate-archive-entry")
182 names.add(entry.filename)
183 total += entry.file_size
184 if entry.flag_bits & 1:
185 raise AdapterBlocked("encrypted-office-archive")
186 if entry.compress_type not in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}:
187 raise AdapterBlocked("unsupported-archive-compression")
188 if (entry.file_size > MAX_TEXT or total > MAX_INFLATED
189 or entry.file_size > MAX_RATIO * max(entry.compress_size, 1)):
190 raise AdapterBlocked("archive-inflation-limit")
191 if entry.filename.lower().endswith("vbaproject.bin"):
192 raise AdapterBlocked("macro-package-not-assessed")
193
194 def read_xml(part):
195 if part not in names:
196 raise AdapterBlocked("incomplete-office-package")
197 with archive.open(part) as source:
198 content = source.read(MAX_TEXT + 1)
199 return bounded_xml(content, parse, errors)
200
201 if "[Content_Types].xml" not in names:
202 raise AdapterBlocked("unsupported-zip-format")
203 types, nodes = read_xml("[Content_Types].xml")
204 if types.tag != f"{{{CT}}}Types":
205 raise AdapterBlocked("unsupported-office-namespace")
206 declared = [(child.get("PartName"), child.get("ContentType")) for child in types]
207 if any("macroenabled" in (kind or "").lower() for _, kind in declared):
208 raise AdapterBlocked("macro-package-not-assessed")
209 kinds = [kind for kind, definition in OFFICE.items() if definition in declared]
210 if len(kinds) != 1:
211 raise AdapterBlocked("unsupported-office-profile")
212 kind = kinds[0]
213 if hint not in {"auto", kind}:
214 raise AdapterBlocked("format-hint-mismatch")
215 main = OFFICE[kind][0][1:]
216 root, used = read_xml(main)
217 nodes += used
218 expected = {"docx": f"{{{W}}}document", "pptx": f"{{{P}}}presentation",
219 "xlsx": f"{{{S}}}workbook"}[kind]
220 if root.tag != expected:
221 raise AdapterBlocked("unsupported-office-namespace")
222 patterns = {
223 "docx": r"word/document\.xml",
224 "pptx": r"ppt/slides/slide[0-9]+\.xml",
225 "xlsx": r"xl/(worksheets/sheet[0-9]+|sharedStrings|tables/table[0-9]+)\.xml",
226 }
227 parts = sorted(part for part in names if re.fullmatch(patterns[kind], part))
228 if not parts:
229 raise AdapterBlocked("office-content-parts-unavailable")
230 characters = tables = image_refs = paragraphs = formulas = cells = 0
231 for part in parts:
232 tree, used = (root, 0) if part == main else read_xml(part)
233 nodes += used
234 if nodes > MAX_NODES:
235 raise AdapterBlocked("document-node-limit")
236 allowed_roots = {
237 "docx": {f"{{{W}}}document"}, "pptx": {f"{{{P}}}sld"},
238 "xlsx": {f"{{{S}}}worksheet", f"{{{S}}}sst", f"{{{S}}}table"},
239 }[kind]
240 if tree.tag not in allowed_roots:
241 raise AdapterBlocked("unsupported-office-namespace")
242 for element in tree.iter():
243 if element.tag == {"docx": f"{{{W}}}t", "pptx": f"{{{A}}}t",
244 "xlsx": f"{{{S}}}t"}[kind]:
245 characters += len(element.text or "")
246 tables += element.tag in {f"{{{W}}}tbl", f"{{{A}}}tbl", f"{{{S}}}table"}
247 image_refs += element.tag == f"{{{A}}}blip"
248 paragraphs += element.tag in {f"{{{W}}}p", f"{{{A}}}p"}
249 formulas += element.tag == f"{{{S}}}f"
250 cells += element.tag == f"{{{S}}}c"
251 result = text_facts(
252 characters, "stored-text-elements-not-rendered-or-cell-occurrences",
253 page_count="not-assessed", inspected_xml_parts=len(parts),
254 table_elements=tables, image_reference_elements=image_refs,
255 paragraphs=paragraphs if kind != "xlsx" else "not-applicable",
256 formula_elements=formulas if kind == "xlsx" else "not-applicable",
257 cell_elements=cells if kind == "xlsx" else "not-applicable",
258 slide_parts=len(parts) if kind == "pptx" else "not-applicable",
259 declared_sheets=sum(e.tag == f"{{{S}}}sheet" for e in root.iter())
260 if kind == "xlsx" else "not-applicable",
261 coverage={"docx": "main-document-only", "pptx": "all-standard-slide-parts",
262 "xlsx": "standard-worksheet-shared-string-table-parts"}[kind],
263 relationships="not-resolved", rendering="not-assessed",
264 )
265 return kind, f"stdlib-zipfile+defusedxml=={XML_VERSION}", result
266 except (zipfile.BadZipFile, zipfile.LargeZipFile, EOFError, zlib.error):
267 raise AdapterBlocked("invalid-office-archive") from None
268
269
270def image_facts(kind, width, height, **values):
271 if not width or not height or width * height > MAX_IMAGE_PIXELS:
272 raise AdapterBlocked("image-dimension-limit")
273 return kind, "stdlib-struct-header", facts(
274 width=width, height=height, text_characters="not-assessed",
275 text_available="not-assessed", validation="header-only-pixels-not-assessed",
276 frame_count="not-assessed", **values)
277
278
279def assess_png(data):
280 if len(data) < 33 or data[8:16] != b"\0\0\0\rIHDR":
281 raise AdapterBlocked("invalid-png-header")
282 width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", data[16:29])
283 if (zlib.crc32(data[12:29]) != struct.unpack(">I", data[29:33])[0]
284 or compression or filtering or interlace not in (0, 1)
285 or depth not in {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8},
286 4: {8, 16}, 6: {8, 16}}.get(color, set())):
287 raise AdapterBlocked("invalid-png-header")
288 return image_facts("png", width, height, bit_depth=depth, color_type=color)
289
290
291def assess_jpeg(data):
292 position = 2
293 for _ in range(4096):
294 if position >= len(data) or data[position] != 255:
295 raise AdapterBlocked("invalid-jpeg-header")
296 while position < len(data) and data[position] == 255:
297 position += 1
298 if position >= len(data):
299 raise AdapterBlocked("invalid-jpeg-header")
300 marker = data[position]
301 position += 1
302 if marker in (0xDA, 0xD9) or position + 2 > len(data):
303 raise AdapterBlocked("jpeg-frame-header-unavailable")
304 length = int.from_bytes(data[position:position + 2], "big")
305 if length < 2 or position + length > len(data):
306 raise AdapterBlocked("invalid-jpeg-header")
307 if marker in (0xC0, 0xC1, 0xC2):
308 if length < 8:
309 raise AdapterBlocked("invalid-jpeg-header")
310 depth, height, width, components = struct.unpack(">BHHB", data[position + 2:position + 8])
311 if length != 8 + 3 * components or not 1 <= components <= 4 or depth not in (8, 12):
312 raise AdapterBlocked("invalid-jpeg-header")
313 return image_facts("jpeg", width, height, precision=depth, components=components)
314 position += length
315 raise AdapterBlocked("image-marker-limit")
316
317
318def assess_document(data, hint, deny_reads):
319 for encoding in ("utf-8-sig", "cp437", "ascii", "utf-8"):
320 codecs.lookup(encoding)
321 office = data.startswith((b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"))
322 parse, errors = xml_parser() if office else (None, ())
323 sys.addaudithook(deny_reads)
324 if office:
325 return assess_office(data, hint, parse, errors)
326 if data.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
327 raise AdapterBlocked("legacy-or-encrypted-office-not-assessed")
328 if data.startswith(b"\x89PNG\r\n\x1a\n"):
329 if hint not in {"auto", "png"}:
330 raise AdapterBlocked("format-hint-mismatch")
331 return assess_png(data)
332 if data.startswith(b"\xff\xd8"):
333 if hint not in {"auto", "jpeg"}:
334 raise AdapterBlocked("format-hint-mismatch")
335 return assess_jpeg(data)
336 return assess_text(data, hint)