Setting the file. One moment.
Generate Perf Report · Amazon DynamoDB · aws/agent-toolkit-for-aws · Skills Docs
Back to the top of the page Reference Architecture
93
Routing Traffic With Route53 And CloudFront
Resilience Program Design
Creating API Gateway Stage
def _driver_saturated
— line 975
This file
Number 65.12
Position 12 of 14
Type Python
Size 77 KB
Lines 1,665 scripts/ generate_perf_report.py
Python · 1,665 lines · 77 KB
16 --summary perf_summary.json \\
17 --output performance_report.md \\
18 --findings-out design_findings.json
19 """
20 from __future__ import annotations
21
22 import argparse
23 import json
24 import sys
25 from pathlib import Path
26 from typing import Optional
27
28 # Import the sibling calculator module for exact expected-number parity.
29 _THIS = Path( __file__ ).resolve().parent
30 sys.path.insert( 0 , str ( _THIS ))
31 import calculate_costs as cc # noqa: E402
32
33 TOLERANCE = 0.10 # 10% deviation triggers flagging
34 PAGE_CAP_BYTES = cc. PAGE_CAP_KB * 1024
35 # Above this NON-throttle error rate a pattern is treated as structurally broken
36 # (e.g. ValidationException from a bad index/attr, a duplicate-key batch, or
37 # AccessDenied). Its observed CU/latency are meaningless, so it becomes a
38 # high-severity correctness finding rather than a benign cost-delta. Below it,
39 # sporadic errors are surfaced as a low-severity note.
40 ERROR_RATE_HIGH = 0.5
41
42 # Error/cancellation codes that signal a BENCHMARK ARTIFACT rather than a design
43 # defect. These arise from the synthetic load shape — many concurrent writes
44 # contending on a small seeded key space — not from anything wrong with the
45 # schema or access-pattern JSON:
46 # TransactionConflict — two in-flight transactions touched the same item
47 # (the benchmark reuses ~seed_items_per_table keys;
48 # real unique IDs don't collide).
49 # ConditionalCheckFailed* — a conditional write's guard fired because the item
50 # already exists / changed — expected when the
51 # benchmark rewrites a bounded key pool.
52 # Everything else above the error threshold (ValidationException,
53 # ResourceNotFoundException, AccessDeniedException, ...) is treated as a genuine
54 # STRUCTURAL defect the design/JSON must fix.
55 #
56 # This list is intentionally small and conservative: a code NOT listed here is
57 # classified "structural" (the fail-safe direction — we'd rather over-flag a
58 # real defect than downplay one as a benign artifact). If AWS ever surfaces
59 # another pure-contention / condition cancellation code that a benchmark's small
60 # key space can provoke, add it here.
61 ARTIFACT_ERROR_CODES = frozenset (
62 {
63 "TransactionConflict" ,
64 "ConditionalCheckFailed" ,
65 "ConditionalCheckFailedException" ,
66 }
67 )
68
69
70 def _is_artifact_code (code: str ) -> bool :
71 return code in ARTIFACT_ERROR_CODES
72
73
74 def _classify_error_codes (error_codes: dict , cancellation_reason_codes: dict ):
75 """Return (kind, dominant_code, reason_histogram).
76
77 kind is "artifact" when the dominant failure is contention/condition (a
78 benchmark-key-space artifact) or "structural" otherwise. For a
79 TransactionCanceledException the per-item cancellation reasons (when the
80 Lambda captured them) are authoritative — they say WHY it cancelled — so
81 they drive the classification; otherwise the top-level error code does.
82 `reason_histogram` is the cancellation-reason map when present, else {}.
83 """
84 reasons = cancellation_reason_codes or {}
85 if reasons:
86 dominant = max (reasons, key =lambda k: reasons[k])
87 kind = "artifact" if all (_is_artifact_code(c) for c in reasons) else "structural"
88 return kind, dominant, dict (reasons)
89 codes = error_codes or {}
90 if not codes:
91 return "structural" , "unknown" , {}
92 dominant = max (codes, key =lambda k: codes[k])
93 # Treat as artifact only when EVERY observed code is an artifact code — a
94 # mix that includes a real ValidationException stays structural.
95 kind = "artifact" if all (_is_artifact_code(c) for c in codes) else "structural"
96 return kind, dominant, {}
97
98
99 def _load_json (path: Path) -> dict :
100 if not path.exists():
101 print ( f "ERROR: file not found: { path } " , file = sys.stderr)
102 sys.exit( 2 )
103 with path.open() as f:
104 return json.load(f)
105
106
107 def _fmt_delta (obs: float , exp: float ) -> str :
108 if exp == 0 :
109 return "—"
110 return f " { ((obs - exp) / exp) * 100 :+.1f} %"
111
112
113 def _fmt_money (v: float ) -> str :
114 if v is None :
115 return "—"
116 if v >= 100 :
117 return f "$ { v :,.0f} "
118 return f "$ { v :,.2f} "
119
120
121 def _fmt_ms (v) -> str :
122 if v is None :
123 return "—"
124 return f " { v :.1f} "
125
126
127 def _hot_cold_ratio (r: dict ):
128 """Hot-partition p99 ÷ cold-partition p99 for a row, or None if unknown.
129
130 Used to tell a genuine hot-partition effect (hot p99 ≫ cold p99) from
131 uniform capacity starvation (hot p99 ≈ cold p99). Returns None when the
132 distribution wasn't captured (uniform/quick/standard runs) or cold p99 is
133 zero/missing — callers treat None as "cannot confirm skew".
134 """
135 kd = r.get( "key_distribution" ) or {}
136 hot = kd.get( "hot_partition_p99_ms" )
137 cold = kd.get( "cold_partition_p99_ms" )
138 if hot is None or cold is None or cold <= 0 :
139 return None
140 return hot / cold
141
142
143 # ---------------------------------------------------------------------------
144 # Per-pattern merge — observed vs expected
145 # ---------------------------------------------------------------------------
146
147
148 def _merge_rows (model: dict , summary: dict ) -> list[ dict ]:
149 tables = model.get( "tables" , [])
150 table_map = {t[ "table_name" ]: t for t in tables}
151 entity_attr_sizes = cc._build_entity_attr_sizes(tables)
152 ap_map = {ap[ "pattern_id" ]: ap for ap in model.get( "access_patterns" , [])}
153
154 rows = []
155 for p in summary.get( "patterns" , []):
156 pid = p[ "pattern_id" ]
157 ap = ap_map.get(pid)
158 if not ap:
159 continue
160 td = table_map.get(ap.get( "table" , "" ))
161 pc = cc.pattern_monthly_cost(ap, td, entity_attr_sizes)
162 cap = pc[ "cap" ]
163
164 # Expected capacity per call = cap["rcus"] + cap["wcus"] (mutually exclusive).
165 expected_cu = cap[ "rcus" ] + cap[ "wcus" ]
166 observed_cu = p[ "steady_state" ][ "mean_observed_cu" ]
167
168 declared_rps = ap[ "peak_rps" ]
169 # Extrapolation uses pricing constants + declared peak.
170 unit_price = cc. WRU_PRICE if ap[ "operation" ] in cc. WRITE_OPS else cc. RRU_PRICE
171 extrapolated = observed_cu * declared_rps * cc. SECONDS_PER_MONTH * unit_price
172 expected_monthly = pc[ "total_cost" ]
173
174 # Vector capacity is metered in BYTES, so it contributes nothing to observed_cu.
175 # Without the block below a SearchVectors pattern extrapolates to exactly $0 and
176 # the design reads as though its search traffic were free.
177 #
178 # Extrapolating from these numbers is sound even though calculate_costs.py
179 # refuses to PREDICT search bytes. The two are different problems: prediction
180 # needs the fraction of the index ANN examines, which measured ~10x apart across
181 # configurations, whereas here the per-call bytes were observed on this design at
182 # its real dimensions and projection. Measure-then-multiply, not model-then-hope.
183 # This is the figure the cost report deliberately leaves blank.
184 ss = p[ "steady_state" ]
185 obs_search_bytes = ss.get( "mean_vector_search_bytes" ) or 0.0
186 obs_write_bytes_by_index = ss.get( "mean_vector_write_bytes_by_index" ) or {}
187 obs_write_bytes = sum (obs_write_bytes_by_index.values())
188 calls_per_month = declared_rps * cc. SECONDS_PER_MONTH
189 vector_monthly = (
190 obs_search_bytes / cc. BYTES_PER_GB * cc. VECTOR_SEARCH_PRICE_PER_GB
191 + obs_write_bytes / cc. BYTES_PER_GB * cc. VECTOR_WRITE_PRICE_PER_GB
192 ) * calls_per_month
193 extrapolated += vector_monthly
194
195 delta_pct = None
196 if expected_cu > 0 :
197 delta_pct = (observed_cu - expected_cu) / expected_cu
198 rows.append(
199 {
200 "pattern_id" : pid,
201 "op" : ap[ "operation" ],
202 "table" : ap[ "table" ],
203 "index" : ap.get( "index" ),
204 "table_index" : ( f " { ap[ 'table' ] } / { ap[ 'index' ] } " if ap.get( "index" ) else ap[ "table" ]),
205 "declared_peak_rps" : declared_rps,
206 "bench_rps" : p[ "bench_rps" ],
207 "observed_cu" : observed_cu,
208 "expected_cu" : expected_cu,
209 "delta_pct" : delta_pct,
210 "p50_ms" : p[ "steady_state" ][ "p50_ms" ],
211 "p95_ms" : p[ "steady_state" ][ "p95_ms" ],
212 "p99_ms" : p[ "steady_state" ][ "p99_ms" ],
213 "throttles" : p[ "steady_state" ][ "throttles" ],
214 "errors" : p[ "steady_state" ].get( "errors" , 0 ),
215 "error_rate" : p[ "steady_state" ].get( "error_rate" , 0.0 ),
216 "error_codes" : p[ "steady_state" ].get( "error_codes" ) or {},
217 "cancellation_reason_codes" : p[ "steady_state" ].get( "cancellation_reason_codes" )
218 or {},
219 "call_count" : p[ "steady_state" ][ "call_count" ],
220 "key_distribution" : p[ "steady_state" ].get( "key_distribution" ),
221 "extrapolated_monthly" : extrapolated,
222 "expected_monthly" : expected_monthly,
223 "amplification_ratio" : p[ "steady_state" ][ "amplification_ratio" ],
224 "gsi_cu_by_index" : p[ "steady_state" ][ "gsi_cu_by_index" ],
225 "observed_vector_search_bytes" : obs_search_bytes,
226 "observed_vector_write_bytes_by_index" : obs_write_bytes_by_index,
227 "observed_vector_write_bytes" : obs_write_bytes,
228 "vector_monthly" : vector_monthly,
229 "cold_start" : p[ "cold_start" ],
230 "measurement_tainted" : p.get( "measurement_tainted" , False ),
231 "consistency" : ap.get( "consistency" , "eventual" ),
232 "items_per_request" : ap.get( "items_per_request" , 1 ),
233 "estimated_item_size_bytes" : ap.get( "estimated_item_size_bytes" , 1024 ),
234 "attributes_written" : ap.get( "attributes_written" ) or [],
235 "conditional_fail_rate" : ap.get( "conditional_fail_rate" , 0.0 ),
236 "projection_type" : None ,
237 "ap" : ap,
238 "td" : td,
239 }
240 )
241
242 return rows
243
244
245 # ---------------------------------------------------------------------------
246 # Design signal extraction
247 # ---------------------------------------------------------------------------
248
249
250 def _extract_signals (rows: list[ dict ]) -> dict :
251 total_monthly = sum ((r[ "extrapolated_monthly" ] or 0.0 ) for r in rows)
252
253 dominant: list[ dict ] = []
254 if total_monthly > 0 :
255 for r in rows:
256 share = (r[ "extrapolated_monthly" ] or 0.0 ) / total_monthly
257 if share > 0.20 :
258 dominant.append(
259 {
260 "pattern_id" : r[ "pattern_id" ],
261 "monthly" : r[ "extrapolated_monthly" ],
262 "share" : share,
263 # op/consistency drive axiom selection in _classify_findings —
264 # a transactional write's cost driver (Mechanics #18 2×) is
265 # nothing like an analytical read's (move-off-DDB / projection).
266 "op" : r[ "op" ],
267 "consistency" : r.get( "consistency" , "eventual" ),
268 # Does any LIVE signal corroborate treating this as high-sev?
269 # Cost share alone is load-invariant (already in cost_report).
270 "throttles" : r.get( "throttles" , 0 ),
271 "delta_pct" : r.get( "delta_pct" ),
272 }
273 )
274 dominant.sort( key =lambda x: - x[ "share" ])
275
276 persistent_throttles = [
277 { "pattern_id" : r[ "pattern_id" ], "throttles" : r[ "throttles" ]}
278 for r in rows
279 if r[ "throttles" ] > 0 and not r[ "measurement_tainted" ]
280 ]
281
282 high_amp: list[ dict ] = []
283 for r in rows:
284 td = r[ "td" ] or {}
285 if r[ "op" ] not in cc. WRITE_OPS or not td.get( "gsis" ):
286 continue
287 obs_amp = r[ "amplification_ratio" ]
288 for g in td[ "gsis" ]:
289 proj = (g.get( "projection" ) or {}).get( "type" , "ALL" ).upper()
290 implied = { "ALL" : 1.0 , "INCLUDE" : 0.3 , "KEYS_ONLY" : 0.1 }.get(proj, 1.0 )
291 if obs_amp > implied * 1.15 and obs_amp > 0.01 :
292 high_amp.append(
293 {
294 "pattern_id" : r[ "pattern_id" ],
295 "gsi_name" : g[ "index_name" ],
296 "projection" : proj,
297 "observed_amp" : obs_amp,
298 "projection_implied_amp" : implied,
299 }
300 )
301
302 # Strong-read overhead: flag a strong-consistency read only if its share
303 # of the total monthly bill is meaningful AND at least one cheaper
304 # alternative exists (eventual read on the same aggregate). Without that
305 # comparison, a strong read isn't inherently a design flaw — it's only a
306 # finding when the added cost is non-trivial. Threshold: >10% of total
307 # monthly.
308 strong_reads: list[ dict ] = []
309 for r in rows:
310 if r[ "op" ] not in cc. READ_OPS or r[ "consistency" ] != "strong" :
311 continue
312 share = (r[ "extrapolated_monthly" ] or 0.0 ) / total_monthly if total_monthly > 0 else 0.0
313 if share > 0.10 :
314 strong_reads.append(
315 {
316 "pattern_id" : r[ "pattern_id" ],
317 "extrapolated_monthly" : r[ "extrapolated_monthly" ],
318 "share" : share,
319 }
320 )
321
322 page_cap_hits: list[ dict ] = []
323 for r in rows:
324 if r[ "op" ] == "Query" :
325 approx_bytes = r[ "items_per_request" ] * r[ "estimated_item_size_bytes" ]
326 if approx_bytes >= PAGE_CAP_BYTES * 0.9 :
327 page_cap_hits.append(
328 {
329 "pattern_id" : r[ "pattern_id" ],
330 "approx_page_bytes" : approx_bytes,
331 }
332 )
333
334 cold_elevated = [
335 {
336 "pattern_id" : r[ "pattern_id" ],
337 "warmup_p99" : r[ "cold_start" ][ "warmup_p99_ms" ],
338 "steady_p99" : r[ "p99_ms" ],
339 }
340 for r in rows
341 if r[ "cold_start" ].get( "cold_start_elevated" )
342 ]
343
344 # Key skew → hot-partition risk (Mechanics #3). Fires when a pattern's
345 # measured key distribution is materially uneven (stddev/mean > 0.5, the
346 # threshold documented in performance-report-format.md) AND the hot
347 # partition shows distress — EITHER throttles OR materially elevated tail
348 # latency relative to the design's other patterns. The latency arm matters
349 # because on **on-demand** tables (the skill's default) adaptive capacity
350 # isolates a single hot key and absorbs it as LATENCY rather than throttles:
351 # a throttle-only test would never warn on the most common configuration.
352 # Throttles remain the stronger signal (provisioned tables, or load beyond
353 # what adaptive capacity can split). Only representative/zipf runs carry
354 # key_distribution; uniform runs omit it so this never fires spuriously.
355 key_skew: list[ dict ] = []
356 for r in rows:
357 kd = r.get( "key_distribution" )
358 if not kd:
359 continue
360 if kd.get( "stddev_over_mean" , 0.0 ) <= 0.5 :
361 continue
362 # Baseline-free hot-partition latency check: the hottest partition's p99
363 # vs the cold partitions' p99, WITHIN this pattern. > 1.8× means the hot
364 # key is materially slower than its peers — a hot partition by
365 # definition, independent of any other pattern or absolute threshold.
366 hot_p99 = kd.get( "hot_partition_p99_ms" )
367 cold_p99 = kd.get( "cold_partition_p99_ms" )
368 elevated_latency = (
369 hot_p99 is not None
370 and cold_p99 is not None
371 and cold_p99 > 0
372 and hot_p99 > 1.8 * cold_p99
373 )
374 if r[ "throttles" ] > 0 or elevated_latency:
375 key_skew.append(
376 {
377 "pattern_id" : r[ "pattern_id" ],
378 "stddev_over_mean" : kd[ "stddev_over_mean" ],
379 "top_key_share" : kd.get( "top_key_share" ),
380 "throttles" : r[ "throttles" ],
381 "hot_partition_p99_ms" : hot_p99,
382 "cold_partition_p99_ms" : cold_p99,
383 "evidence_kind" : ( "throttles" if r[ "throttles" ] > 0 else "elevated_latency" ),
384 }
385 )
386
387 # High non-throttle error rate → the pattern is structurally broken, not
388 # mispriced. This MUST be detected before large_delta below, because a
389 # 100%-error pattern has observed_cu == 0 and would otherwise be misread as a
390 # benign "observed << expected" delta ("RPS/item-size off") — exactly the
391 # silent-failure mode the real-AWS run exposed (a Query on a missing GSI, a
392 # duplicate-key batch, an undefined GSI attr all exit 0 with errors recorded
393 # but never surfaced). error_rate/error_codes come from the Lambda's exact
394 # (uncapped) non-throttle tally via benchmark_model._aggregate.
395 high_error: list[ dict ] = []
396 error_pids: set = set ()
397 for r in rows:
398 if r.get( "error_rate" , 0.0 ) >= ERROR_RATE_HIGH and r[ "call_count" ] > 0 :
399 error_pids.add(r[ "pattern_id" ])
400 codes = r.get( "error_codes" ) or {}
401 reasons = r.get( "cancellation_reason_codes" ) or {}
402 top_code = max (codes, key =lambda k: codes[k]) if codes else "unknown"
403 # Distinguish a benchmark artifact (contention on the small seeded
404 # key space) from a genuine structural defect, using the per-item
405 # cancellation reasons when the Lambda captured them.
406 kind, dom_code, reason_hist = _classify_error_codes(codes, reasons)
407 high_error.append(
408 {
409 "pattern_id" : r[ "pattern_id" ],
410 "error_rate" : r[ "error_rate" ],
411 "errors" : r[ "errors" ],
412 "call_count" : r[ "call_count" ],
413 "top_error_code" : top_code,
414 "error_codes" : codes,
415 "error_kind" : kind, # "artifact" | "structural"
416 "dominant_code" : dom_code,
417 "cancellation_reason_codes" : reason_hist,
418 }
419 )
420
421 # Low-but-nonzero error rate (below the structural threshold): surface as a
422 # note so transient/partial failures aren't completely invisible either.
423 minor_error: list[ dict ] = []
424 for r in rows:
425 if r[ "pattern_id" ] not in error_pids and r.get( "errors" , 0 ) > 0 :
426 codes = r.get( "error_codes" ) or {}
427 top_code = max (codes, key =lambda k: codes[k]) if codes else "unknown"
428 minor_error.append(
429 {
430 "pattern_id" : r[ "pattern_id" ],
431 "error_rate" : r.get( "error_rate" , 0.0 ),
432 "errors" : r[ "errors" ],
433 "call_count" : r[ "call_count" ],
434 "top_error_code" : top_code,
435 }
436 )
437
438 large_delta: list[ dict ] = []
439 for r in rows:
440 if r[ "delta_pct" ] is None :
441 continue
442 # Skip patterns dominated by structural errors: their observed_cu is 0
443 # because the calls FAILED, not because the inputs are mispriced. Letting
444 # them fall through would emit a misleading "input-accuracy" delta finding
445 # and mask the real correctness problem.
446 if r[ "pattern_id" ] in error_pids:
447 continue
448 # Same for throttled patterns: a throttled write reports observed_cu well
449 # below expected because most calls were REJECTED, not because the item
450 # size was overstated. Labeling that "item_size_off" contradicts the
451 # (correct, loud) throttle finding the Load-risk section already raised.
452 # Suppress — the throttle is the story, not a cost deviation.
453 if r[ "throttles" ] > 0 :
454 continue
455 if abs (r[ "delta_pct" ]) > TOLERANCE :
456 likely = "other"
457 # Heuristics: a write with observed << expected is often an item-size
458 # or conditional_fail_rate misstatement; a read with observed >>
459 # expected often points at RPS being off or strong-vs-eventual drift.
460 if r[ "op" ] in cc. WRITE_OPS :
461 likely = (
462 "item_size_off" if r[ "delta_pct" ] < 0 else "conditional_fail_rate_or_projection"
463 )
464 else :
465 likely = "RPS_or_consistency_off"
466 large_delta.append(
467 {
468 "pattern_id" : r[ "pattern_id" ],
469 "delta_pct" : r[ "delta_pct" ],
470 "likely_cause" : likely,
471 }
472 )
473
474 return {
475 "dominant_cost_patterns" : dominant,
476 "persistent_throttles" : persistent_throttles,
477 "high_gsi_amplification" : high_amp,
478 "strong_read_overhead" : strong_reads,
479 "page_cap_hits" : page_cap_hits,
480 "cold_start_elevated_patterns" : cold_elevated,
481 "key_skew_patterns" : key_skew,
482 "high_error_rate_patterns" : high_error,
483 "minor_error_patterns" : minor_error,
484 "large_expected_observed_delta_patterns" : large_delta,
485 }
486
487
488 def _classify_findings (signals: dict ) -> list[ dict ]:
489 out = []
490 counter = 1
491
492 for d in signals[ "dominant_cost_patterns" ]:
493 op = d.get( "op" , "" )
494 consistency = d.get( "consistency" , "eventual" )
495 is_write = op in cc. WRITE_OPS
496 is_txn = op == "TransactWriteItems" or (is_write and consistency == "transactional" )
497 # Axioms must match WHY this pattern dominates the bill:
498 # - transactional write → the 2× transaction multiplier (Mechanics #18);
499 # aggregate tightness is the lever (Mechanics #1). NOT projection
500 # (Mechanics #7) and NOT "move off DynamoDB" (Integration #8) — those
501 # are analytical-read levers and are nonsensical for an OLTP write.
502 # - plain write → aggregate tightness + mutable-GSI-key amplification.
503 # - read/scan → the original analytical triple (projection / move-off-DDB).
504 if is_txn:
505 axioms = [ "Mechanics #18" , "Mechanics #1" ]
506 elif is_write:
507 axioms = [ "Mechanics #1" , "Mechanics #8" ]
508 else :
509 axioms = [ "Mechanics #1" , "Mechanics #7" , "Integration #8" ]
510 # Severity: cost SHARE alone is load-invariant (it's computed from declared
511 # peak, not anything the run stressed) and is already surfaced in
512 # cost_report.md. Only escalate to high when a LIVE signal corroborates a
513 # real problem — throttles, or a materially-off observed-vs-expected delta.
514 # Otherwise cap at medium so a clean unit-cost run doesn't manufacture a
515 # high-severity finding that flips no_significant_findings / triggers the
516 # iteration offer on its own.
517 live_corroborated = (d.get( "throttles" ) or 0 ) > 0 or (
518 d.get( "delta_pct" ) is not None and abs (d[ "delta_pct" ]) > TOLERANCE
519 )
520 severity = "high" if (d[ "share" ] > 0.4 and live_corroborated) else "medium"
521 out.append(
522 {
523 "id" : f "finding- { counter } " ,
524 "category" : "design" ,
525 "signal" : "dominant_cost_patterns" ,
526 "pattern_ids" : [d[ "pattern_id" ]],
527 "evidence" : { "monthly" : d[ "monthly" ], "share" : d[ "share" ]},
528 "axioms" : axioms,
529 "severity" : severity,
530 }
531 )
532 counter += 1
533
534 for t in signals[ "persistent_throttles" ]:
535 out.append(
536 {
537 "id" : f "finding- { counter } " ,
538 "category" : "design" ,
539 "signal" : "persistent_throttles" ,
540 "pattern_ids" : [t[ "pattern_id" ]],
541 "evidence" : { "throttles" : t[ "throttles" ]},
542 "axioms" : [ "Mechanics #3" ],
543 "severity" : "high" ,
544 }
545 )
546 counter += 1
547
548 for h in signals[ "high_gsi_amplification" ]:
549 out.append(
550 {
551 "id" : f "finding- { counter } " ,
552 "category" : "design" ,
553 "signal" : "high_gsi_amplification" ,
554 "pattern_ids" : [h[ "pattern_id" ]],
555 "evidence" : {
556 "observed_amp" : h[ "observed_amp" ],
557 "projection_implied_amp" : h[ "projection_implied_amp" ],
558 "projection" : h[ "projection" ],
559 "gsi_name" : h[ "gsi_name" ],
560 },
561 "axioms" : [ "Mechanics #6" , "Mechanics #7" , "Mechanics #8" ],
562 "severity" : (
563 "high" if h[ "observed_amp" ] > 2 * h[ "projection_implied_amp" ] else "medium"
564 ),
565 }
566 )
567 counter += 1
568
569 for p in signals[ "page_cap_hits" ]:
570 out.append(
571 {
572 "id" : f "finding- { counter } " ,
573 "category" : "design" ,
574 "signal" : "page_cap_hits" ,
575 "pattern_ids" : [p[ "pattern_id" ]],
576 "evidence" : { "approx_page_bytes" : p[ "approx_page_bytes" ]},
577 "axioms" : [ "Mechanics #17" ],
578 "severity" : "medium" ,
579 }
580 )
581 counter += 1
582
583 for s in signals[ "strong_read_overhead" ]:
584 out.append(
585 {
586 "id" : f "finding- { counter } " ,
587 "category" : "design" ,
588 "signal" : "strong_read_overhead" ,
589 "pattern_ids" : [s[ "pattern_id" ]],
590 "evidence" : {
591 "extrapolated_monthly" : s[ "extrapolated_monthly" ],
592 "share" : s[ "share" ],
593 },
594 "axioms" : [ "Mechanics #15" ],
595 "severity" : "medium" ,
596 }
597 )
598 counter += 1
599
600 for c in signals[ "cold_start_elevated_patterns" ]:
601 out.append(
602 {
603 "id" : f "finding- { counter } " ,
604 "category" : "design" ,
605 "signal" : "cold_start_elevated_patterns" ,
606 "pattern_ids" : [c[ "pattern_id" ]],
607 "evidence" : {
608 "warmup_p99" : c[ "warmup_p99" ],
609 "steady_p99" : c[ "steady_p99" ],
610 },
611 "axioms" : [ "Mechanics #19" ],
612 "severity" : "low" ,
613 }
614 )
615 counter += 1
616
617 for k in signals[ "key_skew_patterns" ]:
618 out.append(
619 {
620 "id" : f "finding- { counter } " ,
621 "category" : "design" ,
622 "signal" : "key_skew_patterns" ,
623 "pattern_ids" : [k[ "pattern_id" ]],
624 "evidence" : {
625 "stddev_over_mean" : k[ "stddev_over_mean" ],
626 "top_key_share" : k[ "top_key_share" ],
627 "throttles" : k[ "throttles" ],
628 "hot_partition_p99_ms" : k.get( "hot_partition_p99_ms" ),
629 "cold_partition_p99_ms" : k.get( "cold_partition_p99_ms" ),
630 "evidence_kind" : k.get( "evidence_kind" , "throttles" ),
631 },
632 "axioms" : [ "Mechanics #3" ],
633 # Throttles are the stronger signal; elevated latency under skew on
634 # an on-demand table is a warning, not a hard ceiling breach.
635 "severity" : "high" if k[ "throttles" ] > 0 else "medium" ,
636 }
637 )
638 counter += 1
639
640 # Structural errors first — highest priority. A pattern failing most/all of
641 # its calls is broken, not mispriced; this is a correctness finding the agent
642 # must act on (fix the index/attr/key shape) before any cost reasoning.
643 for e in signals.get( "high_error_rate_patterns" , []):
644 is_artifact = e.get( "error_kind" ) == "artifact"
645 if is_artifact:
646 reason = e.get( "dominant_code" , "TransactionConflict" )
647 interpretation = (
648 f "Most calls for this pattern were rejected with ` { reason } `. "
649 "This is almost certainly a BENCHMARK ARTIFACT, not a design "
650 "defect: the load generator drives many concurrent writes "
651 "against a small synthetic key pool (`seed_items_per_table`), so "
652 "the same items collide far more often than they would under real "
653 "traffic with unique IDs. The cost/latency numbers from the "
654 "calls that DID succeed are still representative of per-op cost. "
655 "To drive this pattern to a clean error rate, raise "
656 "`seed_items_per_table` well above the "
657 "write RPS, or lower the driven rate — do NOT change the design "
658 "on account of this error rate alone. Confirm against the "
659 "real cancellation-reason histogram in the Correctness section."
660 )
661 else :
662 interpretation = (
663 "Most/all calls for this pattern FAILED (non-throttle). "
664 "Observed capacity is 0 because the operation errored, not "
665 "because the design is cheap. Common causes: Query/GSI names "
666 "an index that does not exist on the table; an operation "
667 "references an attribute the table never defines; a batch/"
668 "transact request built duplicate or unseeded keys; or the "
669 "Lambda role lacks the action. Fix the structural cause and "
670 "re-run — the cost/latency numbers for this pattern are not "
671 "meaningful until it succeeds."
672 )
673 out.append(
674 {
675 "id" : f "finding- { counter } " ,
676 "category" : "correctness" ,
677 "signal" : (
678 "pattern_artifact_error_rate" if is_artifact else "pattern_high_error_rate"
679 ),
680 "pattern_ids" : [e[ "pattern_id" ]],
681 "evidence" : {
682 "error_rate" : e[ "error_rate" ],
683 "errors" : e[ "errors" ],
684 "call_count" : e[ "call_count" ],
685 "top_error_code" : e[ "top_error_code" ],
686 "error_codes" : e[ "error_codes" ],
687 "error_kind" : e.get( "error_kind" , "structural" ),
688 "cancellation_reason_codes" : e.get( "cancellation_reason_codes" ) or {},
689 "interpretation" : interpretation,
690 },
691 "axioms" : [ "Mechanics #2" ],
692 # An artifact is a measurement caveat, not a correctness defect, so
693 # it must not flip the run to "significant findings" / high severity.
694 "severity" : "medium" if is_artifact else "high" ,
695 }
696 )
697 counter += 1
698
699 for e in signals.get( "minor_error_patterns" , []):
700 out.append(
701 {
702 "id" : f "finding- { counter } " ,
703 "category" : "correctness" ,
704 "signal" : "pattern_minor_errors" ,
705 "pattern_ids" : [e[ "pattern_id" ]],
706 "evidence" : {
707 "error_rate" : e[ "error_rate" ],
708 "errors" : e[ "errors" ],
709 "call_count" : e[ "call_count" ],
710 "top_error_code" : e[ "top_error_code" ],
711 },
712 "axioms" : [ "Mechanics #2" ],
713 "severity" : "low" ,
714 }
715 )
716 counter += 1
717
718 for d in signals[ "large_expected_observed_delta_patterns" ]:
719 out.append(
720 {
721 "id" : f "finding- { counter } " ,
722 "category" : "input-accuracy" ,
723 "signal" : "large_expected_observed_delta_patterns" ,
724 "pattern_ids" : [d[ "pattern_id" ]],
725 "evidence" : {
726 "delta_pct" : d[ "delta_pct" ],
727 "likely_cause" : d[ "likely_cause" ],
728 },
729 "axioms" : [ "Mechanics #2" , "Mechanics #18" ],
730 "severity" : "medium" ,
731 }
732 )
733 counter += 1
734
735 return out
736
737
738 # ---------------------------------------------------------------------------
739 # Report assembly
740 # ---------------------------------------------------------------------------
741
742
743 def _render_report (
744 model: dict , summary: dict , rows: list[ dict ], signals: dict , findings: list[ dict ]
745 ) -> str :
746 mf = summary.get( "manifest" ) or {}
747 cfg = summary.get( "config" ) or {}
748 # benchmark_model.py surfaces the Lambda-echoed mode at the summary top-level
749 # (what actually ran); fall back to the config's mode for older summaries.
750 mode = summary.get( "mode" ) or cfg.get( "mode" ) or "standard"
751 is_representative = mode == "representative"
752 account = mf.get( "account" ) or "<unknown>"
753 region = mf.get( "region" ) or "<unknown>"
754 prefix = mf.get( "prefix" ) or "<unknown>"
755 run_id = mf.get( "run_id" ) or "<unknown>"
756 # Freshness markers (benchmark_model.py stamps these only after a run
757 # completes). Rendered in the Deployment block so the user/agent can
758 # confirm the report reflects the run they just launched, not a stale
759 # summary. Absent on older summaries → rendered as "<not recorded>".
760 completed_at = summary.get( "benchmark_completed_at" ) or "<not recorded>"
761 wall_seconds = summary.get( "benchmark_wall_seconds" )
762 total_rows = summary.get( "total_rows" )
763
764 total_extrapolated = sum ((r[ "extrapolated_monthly" ] or 0.0 ) for r in rows)
765 total_expected = sum ((r[ "expected_monthly" ] or 0.0 ) for r in rows)
766 delta = None
767 if total_expected > 0 :
768 delta = (total_extrapolated - total_expected) / total_expected * 100
769
770 # How much of total_extrapolated is measured vector SEARCH cost. The calculator
771 # deliberately omits search from total_expected, so this part of the delta is an
772 # expected gap rather than a model error, and the prose below says so. Vector WRITE
773 # cost is in both figures, so it does not need excluding.
774 vector_search_monthly = sum (
775 (r[ "observed_vector_search_bytes" ] or 0.0 )
776 / cc. BYTES_PER_GB
777 * cc. VECTOR_SEARCH_PRICE_PER_GB
778 * r[ "declared_peak_rps" ]
779 * cc. SECONDS_PER_MONTH
780 for r in rows
781 )
782
783 # Crude actual-bill estimate: mean_observed_cu × number of calls × unit price.
784 # Vector bytes are added on the same basis: they are real charges the run incurred,
785 # and omitting them understated the spend for any design with a vector index.
786 bench_cost = 0.0
787 for r in rows:
788 unit = cc. WRU_PRICE if r[ "op" ] in cc. WRITE_OPS else cc. RRU_PRICE
789 bench_cost += r[ "observed_cu" ] * r[ "call_count" ] * unit
790 bench_cost += (
791 (r[ "observed_vector_search_bytes" ] or 0.0 )
792 / cc. BYTES_PER_GB
793 * cc. VECTOR_SEARCH_PRICE_PER_GB
794 + (r[ "observed_vector_write_bytes" ] or 0.0 )
795 / cc. BYTES_PER_GB
796 * cc. VECTOR_WRITE_PRICE_PER_GB
797 ) * r[ "call_count" ]
798
799 # Effective driven throughput — describe what the run ACTUALLY drove, from
800 # the per-pattern bench_rps in the summary, not the nominal scale_factor
801 # (which the min_rps floor can override, as it did when a small-peak design
802 # got clamped to ~1 rps and "0 throttles" meant nothing). When every pattern
803 # ran at the min floor, this was a unit-cost sample, not a load test — say so.
804 _bench_rates = [ float (r[ "bench_rps" ]) for r in rows if r.get( "bench_rps" ) is not None ]
805 _min_rps = float (cfg.get( "min_rps_per_pattern" , 1 ))
806 _floor_bound = bool (_bench_rates) and all ( abs (b - _min_rps) < 1e-6 for b in _bench_rates)
807 if _bench_rates:
808 _lo, _hi = min (_bench_rates), max (_bench_rates)
809 _rate_str = (
810 f " { _lo :.2g} rps/pattern"
811 if abs (_hi - _lo) < 1e-6
812 else f " { _lo :.2g} – { _hi :.2g} rps/pattern"
813 )
814 else :
815 _rate_str = "unknown rate"
816 if _floor_bound:
817 _scale_clause = (
818 f "drove { _rate_str } — the `min_rps_per_pattern` floor, BELOW the "
819 f "configured scale_factor, so this is a UNIT-COST sample, not a load "
820 f "test: it validates per-op cost and unloaded latency only, NOT load, "
821 f "throttling, or behaviour at your declared peak"
822 )
823 else :
824 _scale_clause = (
825 f "drove { _rate_str } (a fraction of declared peak) for "
826 f " { cfg.get( 'duration_seconds' , 90 ) } s/pattern"
827 )
828
829 out: list[ str ] = []
830 a = out.append
831
832 a( "# DynamoDB Live Performance Report \n " )
833
834 # Coverage banner — belt-and-suspenders for a partial/zero-coverage summary
835 # (e.g. a hand-fed or interrupted run). benchmark_model already exits with a
836 # warning on incomplete coverage, but the report is sometimes generated from
837 # a summary directly, so surface it loudly at the very top too.
838 _cov = summary.get( "coverage" ) or {}
839 _all_zero = bool (rows) and all ((r.get( "call_count" ) or 0 ) == 0 for r in rows)
840 if _cov.get( "coverage_incomplete" ) or _cov.get( "missing_patterns" ) or _all_zero:
841 _miss = _cov.get( "missing_patterns" ) or []
842 a(
843 "> ⚠️ **INCOMPLETE COVERAGE — do not trust these numbers as a full "
844 "result.** "
845 + ( f "Patterns with no measurement: { ', ' .join(_miss) } . " if _miss else "" )
846 + ( "Every measured pattern recorded zero calls. " if _all_zero else "" )
847 + "The benchmark did not measure every declared pattern (a killed/"
848 "backgrounded run, a seed failure, or a hand-fed summary). Re-run to "
849 "completion before drawing conclusions. \n "
850 )
851
852 a(
853 "> **Disclaimer:** This report measures a scaled-down benchmark that "
854 f " { _scale_clause } , against real AWS resources deployed in "
855 f " { account } / { region } . Capacity numbers are live observations from "
856 "ReturnConsumedCapacity. Monthly-cost figures extrapolate linearly: "
857 "observed per-op capacity × declared peak RPS × on-demand unit price "
858 "(full public rate; rates vary by region — confirm against the AWS "
859 "DynamoDB pricing page for your region). This benchmark does NOT prove the design "
860 "sustains declared peak RPS — it validates per-op unit cost, latency, "
861 "and GSI amplification shape. Not measured: stream consumers, TTL "
862 "sweep, autoscaling, cross-region replication, long-tail bursts, any "
863 "non-DDB services in the design. \n "
864 )
865 if is_representative:
866 a(
867 "> **Representative mode:** this run drove ~"
868 f " { cfg.get( 'scale_factor' , 0.15 ) } × declared peak with **zipf hot-key "
869 "sampling** and **realistic item-collection cardinality** "
870 f "( { cfg.get( 'items_per_partition' , 40 ) } items/partition) to surface "
871 "SCALE risk — hot-partition throttling, throttle-under-load, GSI "
872 "amplification at volume, and Query-at-cardinality. **Throttle and "
873 "latency numbers here are load-risk signals collected under "
874 "deliberate skew at bounded scale — they scale NONLINEARLY with "
875 "skew and must NOT be linearly extrapolated to peak.** The cost "
876 "figures above remain valid: they extrapolate per-op capacity "
877 "(scale-invariant) against declared peak, not the driven bench RPS. "
878 "One in-region Lambda tops out near 1,500–2,000 RPS/pattern, so this "
879 "surfaces hot-partition risk at bounded cost; it does not prove "
880 "sustained-peak capacity. \n "
881 )
882
883 a(
884 f "**Extrapolated Monthly Cost (measurement-based): { _fmt_money(total_extrapolated) } ** *(from steady-state measurement only)*"
885 )
886 a( f "**Calculator Monthly Cost (expected): { _fmt_money(total_expected) } **" )
887 a(
888 f "**Delta: { f ' { delta :+.1f} %' if delta is not None else '—' } **"
889 )
890 if delta is not None and total_expected > 0 and vector_search_monthly / total_expected > 0.01 :
891 a(
892 f " \n *Of the measured figure, { _fmt_money(vector_search_monthly) } /month is vector "
893 f "**search** cost. The calculator deliberately does not price search, so that "
894 f "amount is missing from the expected column by design — it accounts for "
895 f " { vector_search_monthly / total_expected * 100 :+.1f} points of the delta above and "
896 f "is not a model error. See **Vector capacity (measured)** below.*"
897 )
898 a(
899 f "**This benchmark consumed: ~ { _fmt_money(bench_cost) } in actual AWS charges** *(seed + warmup + measurement)* \n "
900 )
901
902 a( "| Source | Measured Monthly | Calculator | Δ% |" )
903 a( "| ----------------------- | ---------------- | ---------- | -- |" )
904 a( "| Storage (not measured) | — | — | — |" )
905 a(
906 f "| Read/write requests | { _fmt_money(total_extrapolated) :<16} | { _fmt_money(total_expected) :<10} | { ( f ' { delta :+.1f} %' if delta is not None else '—' ) :<2} | \n "
907 )
908
909 # Deployment.
910 a( "## Deployment \n " )
911 a( f "Account: { account } Region: { region } Run ID: { run_id } " )
912 a( f "Resource prefix: ` { prefix } ` " )
913 _window_bits = [ f "completed { completed_at } UTC" ]
914 if wall_seconds is not None :
915 _window_bits.append( f " { wall_seconds } s wall" )
916 if total_rows is not None :
917 _window_bits.append( f " { total_rows } measured rows" )
918 a( f "Window: { ', ' .join(_window_bits) } . " )
919 a(
920 f "Resources: { len (model.get( 'tables' , [])) } tables, "
921 f " { sum ( len (t.get( 'gsis' ) or []) for t in model.get( 'tables' , [])) } GSIs. "
922 f "Manifest: created_resources.json. "
923 )
924 a( "Teardown: teardown.sh (run manually; the skill does NOT auto-delete). \n " )
925
926 # Storage.
927 a( "## Storage (not benchmarked) \n " )
928 a(
929 "Storage is not measured in a short-window benchmark. "
930 "See `cost_report.md` for the storage breakdown from the calculator. \n "
931 )
932
933 # Seed verification — how many items actually landed per table before the
934 # measurement phase. A shortfall means measurements ran against the wrong
935 # data shape (see the seed_shortfall correctness finding).
936 sv_map = summary.get( "seed_verification" ) or {}
937 if sv_map:
938 a( "## Seed verification \n " )
939 a( "| Table | Expected | Seeded (observed) | Status |" )
940 a( "| --- | --- | --- | --- |" )
941 for tname, sv in sv_map.items():
942 exp = sv.get( "expected" )
943 act = sv.get( "actual" )
944 if sv.get( "passed" , True ):
945 status = "ok (sampled to cap)" if sv.get( "sampled" ) else "ok"
946 else :
947 status = "SHORTFALL" if not sv.get( "sampled" ) else "below cap (sampled)"
948 act_cell = f " { act }{ '+' if sv.get( 'sampled' ) else '' } "
949 a(
950 f "| ` { tname } ` | { exp if exp is not None else '—' } | "
951 f " { act_cell if act is not None else '—' } | { status } |"
952 )
953 a( "" )
954
955 # Access Pattern Measurements.
956 a( "## Access Pattern Measurements \n " )
957 a( 'Steady-state only (warmup excluded). See "Cold start" below for warmup numbers. \n ' )
958 a(
959 "| Pattern | Operation | Table/Index | Peak RPS | Bench RPS | "
960 "Observed RCU/WCU | Expected RCU/WCU | Δ | p50 ms | p99 ms | Throttles | Errors | Extrapolated Monthly |"
961 )
962 a( "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" )
963 # Driver-saturation detection (Little's Law). The benchmark drives each
964 # pattern with `concurrency_per_pattern` threads sharing a connection pool.
965 # The mean number of requests in flight is bench_rps × mean_latency
966 # (p50, seconds). When that approaches/exceeds the thread count, requests
967 # queue INSIDE the driver waiting for a worker/connection, so the measured
968 # tail (p99) reflects client-side queueing, not DynamoDB. Flag those rows so
969 # a reader of this report alone isn't alarmed by an inflated p99 that the
970 # service didn't cause. (With the pool now sized to the concurrency this is
971 # rarer, but high bench_rps × non-trivial latency can still saturate the
972 # threads themselves.)
973 _concurrency = int (cfg.get( "concurrency_per_pattern" , 32 ) or 32 )
974
975 def _driver_saturated (r) -> bool :
976 p50 = r.get( "p50_ms" )
977 if not p50 or not r.get( "bench_rps" ):
978 return False
979 inflight = r[ "bench_rps" ] * (p50 / 1000.0 )
980 return inflight >= 0.9 * _concurrency
981
982 saturated = [r[ "pattern_id" ] for r in rows if _driver_saturated(r)]
983 for r in rows:
984 # Show errors as "N (rate%)" so a structurally broken pattern is visible
985 # in the headline table, not just buried in the findings JSON. A '*'
986 # marks an error rate high enough to make the cost/latency cells
987 # meaningless (the calls failed).
988 err_n = r.get( "errors" , 0 )
989 if err_n:
990 err_cell = f " { err_n } ( { r.get( 'error_rate' , 0.0 ) :.0%} )"
991 if r.get( "error_rate" , 0.0 ) >= ERROR_RATE_HIGH :
992 err_cell += " *"
993 else :
994 err_cell = "0"
995 # A '†' on p99 marks driver saturation: the latency tail is client-side
996 # queueing (threads/pool), not DynamoDB.
997 p99_cell = _fmt_ms(r[ "p99_ms" ]) + ( " †" if _driver_saturated(r) else "" )
998 a(
999 "| {pid} | {op} | {ti} | {pk} | {bk:.2f} | {oc:.3f} | {ec:.3f} | {d} | {p50} | {p99} | {thr} | {er} | {mo} |" .format(
1000 pid = r[ "pattern_id" ],
1001 op = r[ "op" ],
1002 ti = r[ "table_index" ],
1003 pk = r[ "declared_peak_rps" ],
1004 bk = r[ "bench_rps" ],
1005 oc = r[ "observed_cu" ],
1006 ec = r[ "expected_cu" ],
1007 d = _fmt_delta(r[ "observed_cu" ], r[ "expected_cu" ]),
1008 p50 = _fmt_ms(r[ "p50_ms" ]),
1009 p99 = p99_cell,
1010 thr = r[ "throttles" ],
1011 er = err_cell,
1012 mo = _fmt_money(r[ "extrapolated_monthly" ]),
1013 )
1014 )
1015 a( "" )
1016 if any (r.get( "error_rate" , 0.0 ) >= ERROR_RATE_HIGH for r in rows):
1017 a(
1018 "> `*` = this pattern errored on most/all calls (non-throttle). Its "
1019 "Observed RCU/WCU, latency, and Extrapolated Monthly are **not "
1020 "meaningful** — the operation failed. See **Correctness** below. \n "
1021 )
1022 if saturated:
1023 a(
1024 f "> `†` = **driver-saturated p99 — not a DynamoDB latency.** At the "
1025 f "driven rate, the in-flight request count (rps × p50) meets or "
1026 f "exceeds the { _concurrency } benchmark driver threads, so these "
1027 f "requests queue client-side waiting for a worker/connection and the "
1028 f "**p99 reflects the single-Lambda load generator, not the service**. "
1029 f "p50 is still representative; the true service tail is closer to the "
1030 f "p99 of the low-rate patterns. To measure an un-saturated tail at "
1031 f "this rate, raise `concurrency_per_pattern` or drive the pattern at a "
1032 f "lower rps. Affected: { ', ' .join(saturated) } . \n "
1033 )
1034
1035 # Correctness — operation errors. Placed high in the report (right after the
1036 # measurements) so a broken run cannot be mistaken for a clean one. A pattern
1037 # erroring on ~all calls is EITHER a design/JSON bug (structural) OR a
1038 # benchmark artifact (contention on the small synthetic key space) — the two
1039 # are split below so a contention artifact is never narrated as "your design
1040 # is broken".
1041 high_err = signals.get( "high_error_rate_patterns" , [])
1042 minor_err = signals.get( "minor_error_patterns" , [])
1043 struct_err = [e for e in high_err if e.get( "error_kind" ) != "artifact" ]
1044 artifact_err = [e for e in high_err if e.get( "error_kind" ) == "artifact" ]
1045 if high_err or minor_err:
1046 a( "## Correctness (operation errors) \n " )
1047
1048 def _err_table (entries):
1049 a( "| Pattern | Op | Error rate | Top reason | Errors / calls |" )
1050 a( "| --- | --- | --- | --- | --- |" )
1051 for e in entries:
1052 r = next ((x for x in rows if x[ "pattern_id" ] == e[ "pattern_id" ]), None )
1053 op = r[ "op" ] if r else "?"
1054 # Prefer the real cancellation reason over the generic
1055 # "TransactionCanceledException" wrapper when we captured it.
1056 shown = e.get( "dominant_code" ) or e[ "top_error_code" ]
1057 a(
1058 f "| { e[ 'pattern_id' ] } | { op } | { e[ 'error_rate' ] :.0%} | "
1059 f "` { shown } ` | { e[ 'errors' ] } / { e[ 'call_count' ] } |"
1060 )
1061 a( "" )
1062
1063 if struct_err:
1064 a(
1065 "**These patterns FAILED on most/all calls (non-throttle errors).** "
1066 "Their cost and latency numbers above are not meaningful until the "
1067 "operation succeeds. This is a structural problem in the design or "
1068 "the access-pattern JSON, not a capacity issue: \n "
1069 )
1070 _err_table(struct_err)
1071 a(
1072 "Likely causes by error code: `ValidationException` → a Query/GSI "
1073 "names an index that doesn't exist, an operation references an "
1074 "undefined attribute, or a batch/transact built duplicate/unseeded "
1075 "keys; `ResourceNotFoundException` → the pattern's table isn't in "
1076 "the design; `AccessDeniedException` → the benchmark role lacks the "
1077 "action. Fix the structural cause in `dynamodb_data_model.json` and "
1078 "re-run; do not interpret the cost numbers for these patterns until "
1079 "they succeed. \n "
1080 )
1081
1082 if artifact_err:
1083 a(
1084 "**Benchmark artifact — NOT a design defect.** These patterns showed "
1085 "a high error rate, but the failures are `TransactionConflict` / "
1086 "condition-check rejections, which come from the benchmark driving "
1087 "many concurrent writes against a small synthetic key pool "
1088 "(`seed_items_per_table`), not from anything wrong with the schema or the "
1089 "access-pattern JSON. Under real traffic with unique IDs these do "
1090 "not occur. The successful calls' per-op cost/latency are still "
1091 "representative — do **not** change the design on account of this "
1092 "error rate: \n "
1093 )
1094 _err_table(artifact_err)
1095 # Render the REAL cancellation-reason histogram when the Lambda
1096 # captured it — so the agent narrates from data, not a guess about
1097 # what cancelled.
1098 for e in artifact_err:
1099 hist = e.get( "cancellation_reason_codes" ) or {}
1100 if hist:
1101 parts = ", " .join(
1102 f "` { c } ` × { n } " for c, n in sorted (hist.items(), key =lambda kv: - kv[ 1 ])
1103 )
1104 a(
1105 f "- { e[ 'pattern_id' ] } cancellation reasons (per-item, "
1106 f "observed): { parts } ."
1107 )
1108 a(
1109 "To drive a clean error rate, raise `seed_items_per_table` well "
1110 "above the write RPS for these "
1111 "patterns, or lower the driven rate, then re-run. \n "
1112 )
1113
1114 if minor_err:
1115 a(
1116 "Patterns with a LOW but non-zero error rate (transient or partial "
1117 "— below the structural threshold, cost numbers still usable): "
1118 + ", " .join(
1119 f " { e[ 'pattern_id' ] } ( { e[ 'errors' ] } / { e[ 'call_count' ] } , "
1120 f "` { e[ 'top_error_code' ] } `)"
1121 for e in minor_err
1122 )
1123 + ". \n "
1124 )
1125
1126 # Load-risk signals — representative mode only. These are the numbers that
1127 # do NOT appear in a 1%-scale unit-cost run: hot-partition throttling and
1128 # the latency/amplification observed under deliberate skew.
1129 if is_representative:
1130 a( "## Load-risk signals (representative mode only) \n " )
1131 a(
1132 "Collected under zipf hot-key sampling at bounded scale. **These scale "
1133 "nonlinearly with key skew and must not be linearly extrapolated to "
1134 "peak** — they characterize hot-partition and throttle RISK, not "
1135 "sustained-peak capacity. \n "
1136 )
1137 a( "| Pattern | Throttles | p99 ms | Observed amp | Top-key share | Distinct keys |" )
1138 a( "| --- | --- | --- | --- | --- | --- |" )
1139 for r in rows:
1140 kd = r.get( "key_distribution" ) or {}
1141 tks = kd.get( "top_key_share" )
1142 ndk = kd.get( "n_distinct_keys" )
1143 a(
1144 "| {pid} | {thr} | {p99} | {amp:.2f} × | {tks} | {ndk} |" .format(
1145 pid = r[ "pattern_id" ],
1146 thr = r[ "throttles" ],
1147 p99 = _fmt_ms(r[ "p99_ms" ]),
1148 amp = r[ "amplification_ratio" ],
1149 tks = ( f " { tks :.0%} " if tks is not None else "—" ),
1150 ndk = (ndk if ndk is not None else "—" ),
1151 )
1152 )
1153 a( "" )
1154 skew_pids = {k[ "pattern_id" ] for k in signals.get( "key_skew_patterns" , [])}
1155 throttled = [r for r in rows if r[ "throttles" ] > 0 ]
1156 lat_only = [r for r in rows if r[ "pattern_id" ] in skew_pids and r[ "throttles" ] == 0 ]
1157 # Skew-vs-starvation split. A throttled pattern points at a HOT PARTITION
1158 # only if the hot partition's p99 is materially above the cold ones'
1159 # (the same within-pattern differential the key_skew signal uses). When
1160 # hot p99 ≈ cold p99, every partition throttled UNIFORMLY — that is
1161 # capacity starvation (the whole table/GSI is under-provisioned for the
1162 # driven load), NOT key skew, and must not be attributed to the
1163 # partition key. Missing/zero cold p99 → can't isolate → treat as
1164 # not-confirmed-skew (conservative).
1165 throttled_skew = [
1166 r for r in throttled if _hot_cold_ratio(r) is not None and _hot_cold_ratio(r) >= 1.3
1167 ]
1168 throttled_starved = [r for r in throttled if r not in throttled_skew]
1169 if throttled_skew:
1170 a(
1171 "Patterns with steady-state throttles AND a hot partition p99 "
1172 "materially above the cold partitions' — a hot partition is at the "
1173 "per-partition ceiling (Mechanics #3, ~1000 WCU / 3000 RCU). "
1174 "Write-shard the partition key (hash suffix) or re-aggregate: "
1175 + ", " .join(r[ "pattern_id" ] for r in throttled_skew)
1176 + ". \n "
1177 )
1178 if throttled_starved:
1179 a(
1180 "Patterns that throttled with hot-partition p99 ≈ cold-partition "
1181 "p99 — this is **uniform capacity starvation, NOT key skew**: the "
1182 "whole table/GSI was under-provisioned for the driven load, so "
1183 "every partition throttled equally. This run did **not** isolate a "
1184 "hot-partition effect for these — do not attribute it to the "
1185 "partition key. To test skew specifically, re-run with capacity set "
1186 "*above* uniform demand so only a genuinely hot partition throttles: "
1187 + ", " .join(r[ "pattern_id" ] for r in throttled_starved)
1188 + ". \n "
1189 )
1190 if lat_only:
1191 a(
1192 "Patterns whose hot partition shows **elevated tail latency** (no "
1193 "throttles) under skew — on an on-demand table this is adaptive "
1194 "capacity absorbing a hot key as latency rather than rejecting it. "
1195 "It signals the same partition-key concentration (Mechanics #3) and "
1196 "the same fix (write-shard / re-aggregate); on a PROVISIONED table "
1197 "the identical skew would throttle. Watch: "
1198 + ", " .join(r[ "pattern_id" ] for r in lat_only)
1199 + ". \n "
1200 )
1201 if not throttled and not lat_only:
1202 a(
1203 "No hot-partition distress (throttles or elevated tail latency) "
1204 "observed under skew at this scale. (Absence at bounded scale is "
1205 "not proof of headroom at peak; a provisioned table would throttle "
1206 "sooner than on-demand, which absorbs hot keys via adaptive "
1207 "capacity.) \n "
1208 )
1209
1210 # Cold start.
1211 a( "## Cold start \n " )
1212 a(
1213 f "Settle window: { cfg.get( 'table_settle_seconds' , 30 ) } s after CreateTable "
1214 "before the first measurement call. "
1215 f "Warmup window: { cfg.get( 'warmup_seconds' , 10 ) } s per pattern, excluded "
1216 "from percentiles above. \n "
1217 )
1218 a( "| Pattern | Warmup p50 ms | Warmup p99 ms | Steady p99 ms | Warmup throttles | Elevated? |" )
1219 a( "| --- | --- | --- | --- | --- | --- |" )
1220 for r in rows:
1221 cs = r[ "cold_start" ]
1222 a(
1223 f "| { r[ 'pattern_id' ] } | { _fmt_ms(cs[ 'warmup_p50_ms' ]) } | "
1224 f " { _fmt_ms(cs[ 'warmup_p99_ms' ]) } | { _fmt_ms(r[ 'p99_ms' ]) } | "
1225 f " { cs.get( 'warmup_throttles' , 0 ) } | "
1226 f " { 'yes' if cs.get( 'cold_start_elevated' ) else 'no' } |"
1227 )
1228 a( "" )
1229 a(
1230 'A pattern is flagged "Elevated" when warmup p99 > 2× steady-state p99 — '
1231 "a signal that callers hitting this pattern immediately after deploy will "
1232 "see materially worse latency than the steady-state numbers suggest. \n "
1233 )
1234
1235 # Supporting services.
1236 a( "## Supporting services (designed, not benchmarked) \n " )
1237 # Detect references to non-DDB services in the design JSON. The current
1238 # schema has no explicit "supporting_services" block; key off any
1239 # table-level "streams" flag as a proxy for "consumers exist but not
1240 # deployed."
1241 streams_tables = [
1242 t[ "table_name" ] for t in model.get( "tables" , []) if (t.get( "streams" ) or {}).get( "enabled" )
1243 ]
1244 if streams_tables:
1245 for tn in streams_tables:
1246 a(
1247 f "- Stream on table ` { tn } `: configured in design; consumers "
1248 "(Lambda / EventBridge Pipe / Kinesis) not deployed."
1249 )
1250 else :
1251 a( "The design references no non-DDB services that require mention here. \n " )
1252 a( "" )
1253
1254 # Axiom Findings.
1255 a( "## Axiom Findings \n " )
1256 a( "**Validated by this run** \n " )
1257 a(
1258 "- Mechanics #18: RCU/WCU formulas reproduced within tolerance for patterns "
1259 "where `|Δ| ≤ 10%`. See Access Pattern Measurements table."
1260 )
1261 # Mechanics #15 eventual/strong 2:1 check if we have both
1262 a(
1263 "- Mechanics #15 (eventual vs strong 2:1): observed where both modes "
1264 "are present in the design; see per-pattern deltas."
1265 )
1266 a( "" )
1267 a( "**Deviations** \n " )
1268 # Patterns dominated by structural errors are reported under Correctness, NOT
1269 # here: their observed_cu is 0 because the calls FAILED, so a "-100% vs
1270 # calculator" line with "Likely: other" would re-introduce the misleading
1271 # "looks mispriced" framing the error signal exists to kill. Exclude them so
1272 # a broken pattern isn't double-reported as a cost deviation to investigate.
1273 error_pids = {e[ "pattern_id" ] for e in signals.get( "high_error_rate_patterns" , [])}
1274 artifact_pids = {
1275 e[ "pattern_id" ]
1276 for e in signals.get( "high_error_rate_patterns" , [])
1277 if e.get( "error_kind" ) == "artifact"
1278 }
1279 structural_pids = error_pids - artifact_pids
1280 any_dev = False
1281 throttled_excluded = 0
1282 for r in rows:
1283 if r[ "pattern_id" ] in error_pids:
1284 continue
1285 # Throttled patterns are excluded for the same reason as errored ones:
1286 # observed_cu is low because calls were REJECTED, not mispriced. They're
1287 # reported in the Load-risk section; a "-100%, Likely: item_size_off"
1288 # line here would contradict that and re-introduce the "looks mispriced"
1289 # framing the throttle signal exists to kill.
1290 if r[ "throttles" ] > 0 :
1291 throttled_excluded += 1
1292 continue
1293 if r[ "delta_pct" ] is not None and abs (r[ "delta_pct" ]) > TOLERANCE :
1294 likely = next (
1295 (
1296 d[ "likely_cause" ]
1297 for d in signals[ "large_expected_observed_delta_patterns" ]
1298 if d[ "pattern_id" ] == r[ "pattern_id" ]
1299 ),
1300 "other" ,
1301 )
1302 a(
1303 f "- { r[ 'pattern_id' ] } : observed { r[ 'observed_cu' ] :.3f} vs "
1304 f "expected { r[ 'expected_cu' ] :.3f} "
1305 f "(Δ { _fmt_delta(r[ 'observed_cu' ], r[ 'expected_cu' ]) } ). Likely: { likely } ."
1306 )
1307 any_dev = True
1308 if structural_pids:
1309 a(
1310 f "- ( { len (structural_pids) } pattern(s) excluded here — they FAILED "
1311 "most/all calls and are reported under **Correctness** above, not as "
1312 "cost deviations.)"
1313 )
1314 if artifact_pids:
1315 a(
1316 f "- ( { len (artifact_pids) } pattern(s) excluded here — their high error "
1317 "rate is a benchmark artifact (contention on the synthetic key space), "
1318 "not a cost or design issue; see **Correctness** above.)"
1319 )
1320 if throttled_excluded:
1321 a(
1322 f "- ( { throttled_excluded } pattern(s) excluded here — they THROTTLED and "
1323 "are reported under **Load-risk signals** above, not as cost "
1324 "deviations.)"
1325 )
1326 if not any_dev:
1327 a( "- None: every pattern within ±10 % o f calculator prediction." )
1328 a( "" )
1329 a( "**Not validated by this run** \n " )
1330 if is_representative:
1331 a(
1332 "- Mechanics #3 (per-partition ceilings 1000 WCU / 3000 RCU): "
1333 "PROBED under zipf skew at bounded scale — see Load-risk signals. "
1334 "Throttles indicate a hot partition near its ceiling ONLY when the hot "
1335 "partition's p99 is materially above the cold partitions'; throttles "
1336 "with hot p99 ≈ cold p99 are uniform capacity starvation, not skew, and "
1337 "do not isolate a hot-partition effect. Absence of throttles is not "
1338 "proof of headroom at full peak."
1339 )
1340 else :
1341 a(
1342 "- Mechanics #3 (per-partition ceilings 1000 WCU / 3000 RCU): bench did not push to ceilings by design."
1343 )
1344 a( "- Mechanics #12 (TTL eventual delete): sweep cadence is hours; short window cannot observe." )
1345 a(
1346 "- Data Modeling #3, #5 (Streams/PITR/recovery granularity): configuration-level, not traffic-observable."
1347 )
1348 a( "- Data Modeling #13 (Global Tables LWW / MRSC): single-region deploy." )
1349 a( "- Patterns #1 (idempotency middleware): application layer, not DDB alone." )
1350 a( "- Integration #1 (consumer idempotency): consumers not deployed. \n " )
1351
1352 # Vector capacity — measured, because it cannot be soundly predicted.
1353 vector_rows = [
1354 r for r in rows if r[ "observed_vector_search_bytes" ] or r[ "observed_vector_write_bytes" ]
1355 ]
1356 if vector_rows:
1357 a( "## Vector capacity (measured) \n " )
1358 a(
1359 "Vector indexes are metered in **bytes**, not RCU/WCU, so none of this "
1360 "appears in the capacity columns above — a `SearchVectors` pattern legitimately "
1361 "consumes 0 CU. These are observed `VectorSearchRequestBytes` and "
1362 "`VectorWriteRequestBytes` per call, taken from `ConsumedCapacity` on this "
1363 "deployment. \n "
1364 )
1365 a(
1366 f "`calculate_costs.py` prices vector **storage and writes** but deliberately "
1367 f "declines to price **search**: the share of an index that approximate-nearest-"
1368 f "neighbour search examines varied by roughly 10x across configurations in "
1369 f "testing, so a predicted figure would be confidently wrong. The numbers below "
1370 f "close that gap by measurement — per-call bytes observed on this design, at its "
1371 f "real dimensions and projection, multiplied by declared peak. Search is billed at "
1372 f "$ { cc. VECTOR_SEARCH_PRICE_PER_GB } /GB and writes at "
1373 f "$ { cc. VECTOR_WRITE_PRICE_PER_GB } /GB. \n "
1374 )
1375 a(
1376 "| Pattern | Op | Index | Search B/call | Write B/call | Declared peak rps | "
1377 "Vector $/mo |"
1378 )
1379 a( "| --- | --- | --- | --- | --- | --- | --- |" )
1380 for r in vector_rows:
1381 wb = r[ "observed_vector_write_bytes_by_index" ]
1382 idx = r[ "index" ] or ( ", " .join( sorted (wb)) if wb else "—" )
1383 a(
1384 f "| { r[ 'pattern_id' ] } | { r[ 'op' ] } | { idx } | "
1385 f " { r[ 'observed_vector_search_bytes' ] :,.0f} | "
1386 f " { r[ 'observed_vector_write_bytes' ] :,.0f} | "
1387 f " { r[ 'declared_peak_rps' ] :,} | { _fmt_money(r[ 'vector_monthly' ]) } |"
1388 )
1389 a( "" )
1390 vector_total = sum (r[ "vector_monthly" ] for r in vector_rows)
1391 a(
1392 f "**Vector capacity total: { _fmt_money(vector_total) } /month** at declared peak, "
1393 "already included in the extrapolated monthly figures above. Vector index "
1394 "*storage* is not in this total — it rolls into table storage, which the cost "
1395 "report covers. \n "
1396 )
1397 min_b = cc. VECTOR_METERING_MIN_BYTES
1398 if any (
1399 0 < r[ "observed_vector_search_bytes" ] <= min_b
1400 or 0 < r[ "observed_vector_write_bytes" ] <= min_b
1401 for r in vector_rows
1402 ):
1403 a(
1404 f "At least one figure is at or below the { min_b :,} -byte metering minimum, so "
1405 "it reflects the floor rather than the vector's real size. Low-dimension "
1406 "indexes do not meter proportionally cheaper; scaling these numbers up by "
1407 "dimension count will overstate cost until the floor is cleared "
1408 f "(~ { min_b // 4 } dimensions). \n "
1409 )
1410
1411 # Cost-estimate validation.
1412 a( "## Cost-estimate validation \n " )
1413 if any_dev:
1414 a(
1415 "See **Deviations** above: at least one pattern diverges from the "
1416 "calculator by more than 10%. Investigate per-pattern as called out. \n "
1417 )
1418 elif structural_pids:
1419 a(
1420 "Cost validation is INCONCLUSIVE for "
1421 f " { len (structural_pids) } pattern(s) that errored on most/all calls — "
1422 "their observed capacity is 0 because the operation failed, not because "
1423 "the design is cheap (see **Correctness** above). Fix those patterns "
1424 "and re-run before trusting their cost numbers. The patterns that DID "
1425 "succeed reproduce the calculator within tolerance. \n "
1426 )
1427 elif artifact_pids:
1428 a(
1429 "Cost numbers are trustworthy. "
1430 f " { len (artifact_pids) } pattern(s) showed a high error rate, but it is a "
1431 "benchmark artifact (key-space contention, see **Correctness** above), "
1432 "not a design issue — the successful calls reproduce the calculator "
1433 "within tolerance. \n "
1434 )
1435 else :
1436 a(
1437 "Calculator unit-cost formulas reproduce live DynamoDB billing within "
1438 "tolerance for this design; any remaining cost risk lies in the RPS / "
1439 "item-size assumptions fed to the calculator. \n "
1440 )
1441
1442 # Cost-concentration caveat (P2.6): when one pattern dominates the bill, the
1443 # headline is only as accurate as that pattern's item-size/RPS inputs — carry
1444 # the fragility note from the calculator into the live report so the two
1445 # artifacts agree.
1446 dom = signals.get( "dominant_cost_patterns" ) or []
1447 if dom:
1448 top = dom[ 0 ]
1449 dom_row = next ((r for r in rows if r[ "pattern_id" ] == top[ "pattern_id" ]), None )
1450 delta_bit = ""
1451 if dom_row and dom_row.get( "delta_pct" ) is not None :
1452 delta_bit = f " (its observed vs expected Δ here is " f " { dom_row[ 'delta_pct' ] :+.0%} )"
1453 a(
1454 f "Cost concentration: ` { top[ 'pattern_id' ] } ` drives ~ { top[ 'share' ] :.0%} of "
1455 "the estimate, so the headline is dominated by that one pattern's "
1456 f "item-size and RPS inputs { delta_bit } — a misstatement there moves the "
1457 "whole number roughly in proportion. Validate those inputs before "
1458 "quoting the figure. \n "
1459 )
1460
1461 # Design reflection — scaffolded; agent authors the subsections.
1462 a( "## Design reflection \n " )
1463 a(
1464 f "Authored by the agent from `design_findings.json` ( { len (findings) } "
1465 "findings extracted) and the axioms. \n "
1466 )
1467 input_acc = [f for f in findings if f[ "category" ] == "input-accuracy" ]
1468 design_f = [f for f in findings if f[ "category" ] == "design" ]
1469 correctness_f = [f for f in findings if f[ "category" ] == "correctness" ]
1470
1471 a( "### Correctness findings (fix before reading cost numbers) \n " )
1472 if correctness_f:
1473 for f_ in correctness_f:
1474 ev = f_[ "evidence" ]
1475 if f_[ "signal" ] == "seed_shortfall" :
1476 a(
1477 f "- **seed_shortfall** on table ` { ev.get( 'table' ) } `: "
1478 f "severity= { f_[ 'severity' ] } . Seeded { ev.get( 'actual' ) } of "
1479 f " { ev.get( 'expected' ) } expected items. Measurements for "
1480 "patterns on this table ran against far less data than declared "
1481 "(hot-key skew and item-collection cardinality are both wrong) "
1482 "— fix the seed (or seed volume) and re-run before trusting any "
1483 "number on this table."
1484 )
1485 continue
1486 pid_list = ", " .join(f_[ "pattern_ids" ])
1487 a(
1488 f "- ** { f_[ 'signal' ] } ** on { pid_list } : severity= { f_[ 'severity' ] } . "
1489 f "Error rate { ev.get( 'error_rate' , 0.0 ) :.0%} , top code "
1490 f "` { ev.get( 'top_error_code' , 'unknown' ) } ` "
1491 f "( { ev.get( 'errors' , '?' ) } / { ev.get( 'call_count' , '?' ) } calls). "
1492 "Cost/latency for this pattern are meaningless until it succeeds — "
1493 "fix the structural cause in the design JSON and re-run."
1494 )
1495 else :
1496 a( "- None — every pattern completed its calls without structural errors." )
1497 a( "" )
1498
1499 a( "### Input-accuracy findings (update the inputs, not the design) \n " )
1500 if input_acc:
1501 for f_ in input_acc:
1502 pid = f_[ "pattern_ids" ][ 0 ]
1503 match_row: Optional[ dict ] = next ((x for x in rows if x[ "pattern_id" ] == pid), None )
1504 if not match_row:
1505 continue
1506 a(
1507 f "- { pid } : observed { match_row[ 'observed_cu' ] :.3f} , expected "
1508 f " { match_row[ 'expected_cu' ] :.3f} . Likely cause: "
1509 f " { f_[ 'evidence' ].get( 'likely_cause' , 'unknown' ) } . Proposed: "
1510 "update JSON and re-run calculator only."
1511 )
1512 else :
1513 a( "- None — declared inputs reproduce observed capacity within tolerance." )
1514 a( "" )
1515
1516 a( "### Design findings (the structure itself is the source) \n " )
1517 if design_f:
1518 for f_ in design_f:
1519 pid_list = ", " .join(f_[ "pattern_ids" ])
1520 a(
1521 f "- ** { f_[ 'signal' ] } ** on { pid_list } : severity= { f_[ 'severity' ] } . "
1522 f "Axioms: { ', ' .join(f_[ 'axioms' ]) } . Evidence: "
1523 f " { json.dumps(f_[ 'evidence' ], default = str ) } . "
1524 "See SKILL.md `## Live validation` for the axiom-indexed "
1525 "alternative-design menu and how to express it as a JSON diff."
1526 )
1527 else :
1528 a( "Measurements support the current design. No alternative is argued " "for by this run." )
1529 a( "" )
1530
1531 # Iteration offer — when at least one design OR correctness finding exists
1532 # (a structurally broken pattern needs a JSON fix + re-run just as much as a
1533 # design finding does).
1534 if design_f or correctness_f:
1535 a( "## Iteration offer \n " )
1536 a(
1537 "- **Calculator-only re-eval** — update the JSON with the proposed "
1538 "changes from the Design findings section above and re-run "
1539 "`scripts/calculate_costs.py`. No AWS calls."
1540 )
1541 a(
1542 "- **Full re-eval** — calculator + a second live validation against "
1543 "the revised design. Requires re-consenting to AWS deployment and "
1544 "running the prior `teardown.sh` first unless the change is "
1545 "additive-only (e.g. adding a GSI)."
1546 )
1547 a(
1548 "- **No changes** — record the decision as a deviation per Artifact "
1549 "#5 with your stated reason. \n "
1550 )
1551
1552 return " \n " .join(out)
1553
1554
1555 def main ():
1556 p = argparse.ArgumentParser( description = __doc__ .splitlines()[ 0 ])
1557 p.add_argument( "--model" , required = True , help = "path to dynamodb_data_model.json (the design)" )
1558 p.add_argument(
1559 "--summary" , required = True , help = "path to perf_summary.json written by benchmark_model.py"
1560 )
1561 p.add_argument(
1562 "--output" , required = True , help = "output path for the human-facing performance_report.md"
1563 )
1564 p.add_argument(
1565 "--findings-out" ,
1566 default = "design_findings.json" ,
1567 help = "output path for the compact machine-readable findings the "
1568 "agent reads (default: design_findings.json)" ,
1569 )
1570 args = p.parse_args()
1571
1572 model = _load_json(Path(args.model))
1573 summary = _load_json(Path(args.summary))
1574
1575 rows = _merge_rows(model, summary)
1576 signals = _extract_signals(rows)
1577 findings = _classify_findings(signals)
1578
1579 # Seed-shortfall finding (P1.5). verify_seed now reports a bounded-pagination
1580 # actual count plus a `sampled` flag. A real shortfall (passed:false AND
1581 # sampled:false — we counted to exhaustion, not just to the cap) means the
1582 # table seeded far fewer items than declared, so its measurements ran against
1583 # the wrong data shape: high-severity correctness. A `sampled` non-pass is
1584 # only "we stopped counting at the cap" and is NOT a defect.
1585 for tname, sv in (summary.get( "seed_verification" ) or {}).items():
1586 if not sv.get( "passed" , True ) and not sv.get( "sampled" , False ):
1587 findings.append(
1588 {
1589 "id" : f "seed- { tname } " ,
1590 "category" : "correctness" ,
1591 "signal" : "seed_shortfall" ,
1592 "pattern_ids" : [],
1593 "evidence" : {
1594 "table" : tname,
1595 "expected" : sv.get( "expected" ),
1596 "actual" : sv.get( "actual" ),
1597 "seed_shortfall_ratio" : sv.get( "seed_shortfall_ratio" ),
1598 },
1599 "axioms" : [ "Mechanics #2" ],
1600 "severity" : "high" ,
1601 }
1602 )
1603
1604 top = sorted (rows, key =lambda r: - (r[ "extrapolated_monthly" ] or 0.0 ))[: 5 ]
1605 total = sum ((r[ "extrapolated_monthly" ] or 0.0 ) for r in rows)
1606 top_drivers = [
1607 {
1608 "pattern_id" : r[ "pattern_id" ],
1609 "monthly" : r[ "extrapolated_monthly" ],
1610 "share" : (r[ "extrapolated_monthly" ] or 0.0 ) / total if total else 0.0 ,
1611 }
1612 for r in top
1613 ]
1614
1615 # "Significant" = something the agent must ACT on: any design finding, or a
1616 # STRUCTURAL correctness finding (a pattern failing most/all calls — the
1617 # high-severity pattern_high_error_rate). A 100%-error pattern is NOT a clean
1618 # run even with no design findings — surfacing that is the whole point of the
1619 # error signal added after the real-AWS run found silent 100%-error patterns.
1620 # A LOW-severity minor-error note (e.g. 1 transient error in 200 calls) is
1621 # informational AWS noise, not an action item: it stays visible in the
1622 # findings + report but does NOT flip the clean-run bit, so realistic
1623 # transient blips don't raise a false "significant finding".
1624 # A dominant_cost finding that did NOT reach high severity is load-invariant
1625 # cost concentration with no live corroboration — it's already shown in
1626 # cost_report.md and must not, on its own, flip the clean-run bit or trigger
1627 # the iteration offer on an otherwise-clean unit-cost run. Every other design
1628 # finding (throttles, skew, amplification, page-cap, strong-read, cold-start)
1629 # still counts, as does a high-severity correctness finding.
1630 def _is_significant (f: dict ) -> bool :
1631 if f[ "signal" ] == "dominant_cost_patterns" :
1632 return f[ "severity" ] == "high"
1633 if f[ "category" ] == "design" :
1634 return True
1635 return f[ "category" ] == "correctness" and f[ "severity" ] == "high"
1636
1637 significant = [f for f in findings if _is_significant(f)]
1638 # Incomplete coverage (missing patterns, or a partial/zero-call run) is itself
1639 # a reason the run is not a clean result — surface it as a flag the agent
1640 # reads and let it flip the clean-run bit so a half-finished benchmark never
1641 # reports "no significant findings".
1642 cov = summary.get( "coverage" ) or {}
1643 coverage_incomplete = bool (cov.get( "coverage_incomplete" ) or cov.get( "missing_patterns" ))
1644 no_significant = ( not significant) and ( not coverage_incomplete)
1645
1646 findings_payload = {
1647 "classified_findings" : findings,
1648 "top_cost_drivers" : top_drivers,
1649 "no_significant_findings" : no_significant,
1650 "coverage_incomplete" : coverage_incomplete,
1651 }
1652
1653 report_md = _render_report(model, summary, rows, signals, findings)
1654 Path(args.output).write_text(report_md)
1655 Path(args.findings_out).write_text(json.dumps(findings_payload, indent = 2 , default = str ))
1656 print ( f "Report: { args.output } " )
1657 print ( f "Findings: { args.findings_out } " )
1658 print (
1659 f "Analyzed { len (rows) } patterns, extracted { len (findings) } findings "
1660 f "( { 'clean' if no_significant else 'has design findings' } )."
1661 )
1662
1663
1664 if __name__ == "__main__" :
1665 main()