Setting the file. One moment.
Calculate Costs · Amazon DynamoDB · aws/agent-toolkit-for-aws · Skills Docs
Back to the top of the page Reference Architecture
93
Routing Traffic With Route53 And CloudFront
Resilience Program Design
Creating API Gateway Stage
435
def _fmt
— line 435
This file
Number 65.9
Position 9 of 14
Type Python
Size 85 KB
Lines 1,933 scripts/ calculate_costs.py
Python · 1,933 lines · 85 KB
16 - Filters and projections DO NOT reduce capacity.
17 - BatchGetItem / BatchWriteItem: rounded per-item, then summed.
18 - PutItem / UpdateItem: sized on LARGER of before and after.
19 - Transactional writes: 2× multiplier applies to BASE TABLE ONLY.
20 GSI and LSI writes remain 1×. (Verified.)
21 - Transactional reads: 2× multiplier.
22 - Conditional write failure still consumes capacity based on the existing
23 item size (or new item size for PutItem on a non-existent key).
24 - GSI write amplification:
25 * Only fires if the item has the GSI's partition-key attribute.
26 * Only fires if a projected attribute changed (UpdateItem).
27 * GSI write is sized by the PROJECTED attributes, not the base item.
28 - Storage: (raw_bytes + 100 bytes per item) / 1_000_000_000 (decimal GB),
29 billed at the full public Standard rate. Global tables add +48 bytes/item.
30 The 25 GB free tier is intentionally NOT applied (account-wide, often
31 already consumed) — see _storage_cost.
32
33 Usage:
34 python3 calculate_costs.py --model artifacts/{app}/dynamodb_data_model.json
35 python3 calculate_costs.py --model artifacts/{app}/dynamodb_data_model.json \
36 --requirements artifacts/{app}/{app}-requirements.json
37 python3 calculate_costs.py --model artifacts/{app}/dynamodb_data_model.json \
38 --output cost_report.md
39 """
40 from __future__ import annotations
41
42 import argparse
43 import json
44 import math
45 import sys
46 from pathlib import Path
47
48 # -----------------------------------------------------------------------------
49 # Pricing constants — DynamoDB Standard table class, on-demand public rates.
50 # Rates vary by region; confirm against the AWS DynamoDB pricing page.
51 # -----------------------------------------------------------------------------
52 SECONDS_PER_MONTH = 2_592_000
53 SECONDS_PER_DAY = 86_400
54 DAYS_PER_MONTH = 30
55
56 # Per-request prices (dollars per unit, converted from "$X per million").
57 WRU_PRICE = 0.625 / 1_000_000 # Standard on-demand write
58 RRU_PRICE = 0.125 / 1_000_000 # Strongly-consistent on-demand read
59 # (Eventually-consistent and transactional rates derive from these via multiplier.)
60
61 # Storage.
62 STORAGE_PRICE_PER_GB_MONTH = 0.25 # Standard class, full public rate
63 # NOTE : the 25 GB Standard storage free tier is intentionally NOT modeled — it is
64 # account-wide and often already consumed, so we price all storage at full rate.
65 BYTES_PER_GB = 1_000_000_000 # Decimal GB per AWS billing convention
66
67 # Capacity-unit sizing (bytes).
68 RCU_SIZE_BYTES = 4 * 1024 # 4 KB
69 WCU_SIZE_BYTES = 1 * 1024 # 1 KB
70
71 # Per-item storage overhead (AWS docs: "100 bytes per item for indexing").
72 PER_ITEM_STORAGE_OVERHEAD = 100
73 GLOBAL_TABLES_OVERHEAD = 48 # Additional bytes when global tables enabled
74
75 # Query/Scan pagination cap. DynamoDB returns at most 1 MB per page; we use a
76 # conservative 900 KB to leave headroom for metadata.
77 PAGE_CAP_KB = 900
78
79 # -----------------------------------------------------------------------------
80 # Vector index pricing.
81 #
82 # Vector index capacity is metered in BYTES, not request units, across three
83 # dimensions. Rates below are us-east-1 Standard, taken from the AWS Pricing API
84 # (usage types VectorWriteRequest / VectorSearch) rather than from a worked example.
85 # Standard-IA is 125% on both request dimensions (IA-VectorWriteRequest $0.65,
86 # IA-VectorSearch $0.0025) and 40% on storage.
87 #
88 # There is NO separate vector-storage usage type: index storage rolls into ordinary
89 # table storage at STORAGE_PRICE_PER_GB_MONTH.
90 #
91 # All three dimensions are modelled, but on different footings, and the report says
92 # which is which:
93 # storage, writes — derived from the design (dimensions, projection, item counts)
94 # searches — CALIBRATED against live measurements, because the metered bytes
95 # depend on index traversal rather than anything in the design
96 # Live validation supersedes the search figure with the observed
97 # VectorSearchRequestBytes. See references/vector-search.md.
98 # -----------------------------------------------------------------------------
99 VECTOR_WRITE_PRICE_PER_GB = 0.52
100 VECTOR_SEARCH_PRICE_PER_GB = 0.002
101 IA_REQUEST_MULTIPLIER = 1.25 # applies to both vector request dimensions
102 BYTES_PER_F32 = 4 # vectors are stored at 32-bit float precision
103
104 # 1 KB minimum. Applied ONCE PER REQUEST, per the documentation.
105 #
106 # We follow the docs here, and a measurement caveat is worth recording because it is easy
107 # to misread. `ConsumedCapacity` floors each index's REPORTED value at 1 KB independently
108 # of the request total: in a probe where one PutItem touched four indexes and the request
109 # total was ~26 KB (far above the floor), a 128-dimension index whose real content is 512 B
110 # still reported 1024. So the per-index reported numbers are a display rounding and cannot
111 # be summed to infer the billed total, and they cannot distinguish per-request from
112 # per-index billing. Settling that would need Cost Explorer / CUR data, not
113 # ConsumedCapacity.
114 #
115 # It barely matters in practice: the floor only binds below 256 dimensions (256 x 4 =
116 # 1024 B), and real embedding models are 256-3072.
117 VECTOR_METERING_MIN_BYTES = 1024
118
119 # ---- Search metering: measured, and deliberately NOT priced -------------------
120 # Measured us-west-2 2026-08-19 (see _research/verified-vector-facts.md). What is solid:
121 # * Metered bytes are EXACTLY linear in TopK within a configuration (0.0% midpoint error).
122 # * The PROJECTION is a ~170x multiplier on the per-result term: ~89 B for KEYS_ONLY,
123 # ~98 B for a narrow INCLUDE, and the whole projected item for ALL. A TopK=100 search
124 # on an ALL index measured 1.52 MB.
125 # * At a fixed index configuration, cost is linear in dimensions (~31 B per dimension
126 # measured at 256 / 1536 / 3072).
127 #
128 # What is NOT modellable, and why we refuse to emit a dollar figure: the fraction of the
129 # index a search examines varies by an ORDER OF MAGNITUDE with index configuration —
130 # 12.8% of all vector bytes on a 60-item unpartitioned index, but 1.3% on a 200-item
131 # partitioned one. An earlier attempt to calibrate a traversal term linearly across two
132 # probes was 51-69% low when tested at 256 / 1536 / 3072 dimensions, i.e. wrong at exactly
133 # the dimensionalities real embedding models use (Titan Text Embeddings V2, OpenAI
134 # text-embedding-3-large). A confidently wrong cost figure is worse than none, so the
135 # report gives the drivers and tells the user to measure. Live validation supplies the
136 # observed VectorSearchRequestBytes.
137 # Key/search-schema overhead a KEYS_ONLY vector index carries on top of the vector.
138 # Measured: a minimal single-vector item gave 4,105 B on a 1024-dim KEYS_ONLY index
139 # (4,096 + 9); with a SearchSchema HASH + INLINE_FILTER it gave 4,130 (+34). Use the
140 # larger, which is the conservative direction.
141 VECTOR_KEY_OVERHEAD_BYTES = 35
142 # A vector attribute copied into an index by ALL projection is carried in its base-table
143 # representation, not as f32. Measured: an unindexed 1024-dim vector added 5,633 B to an
144 # ALL index, i.e. ~5.5 B per number rather than 4.
145 BYTES_PER_BASE_TABLE_NUMBER = 5.5
146
147 VECTOR_RETURN_BYTES_KEYS_ONLY = 89
148 VECTOR_RETURN_BYTES_INCLUDE_BASE = 89
149
150 # Traversal slope used ONLY by the pre-spend gate in benchmark_model.py, never for a
151 # reported cost. Two measured points: 128 dims -> 2,647 B (20.7 B/dim) and 1,024 dims ->
152 # 10,588 B (10.3 B/dim) — traversal grows SUBLINEARLY in dimensions. Taking the steeper
153 # low-dimension slope therefore overshoots at higher dimensions, which is the correct
154 # direction for a guard whose job is to refuse a bill. See
155 # vector_search_bytes_spend_gate_upper().
156 VECTOR_TRAVERSAL_BYTES_PER_DIM_UPPER = 20.7
157
158 # Safety factor on the spend-gate bound. Not decoration — unfactored, the bound had
159 # essentially NO margin on two of the sixteen measured search points: 128-dim KEYS_ONLY at
160 # TopK=1 came out 2,739 B against 2,736 B measured (1.001x), and 1024-dim ALL at TopK=100
161 # came out 1,739,097 B against 1,524,351 B (1.14x) — i.e. thinnest exactly where the bill
162 # is largest. The exposure is structural: for an ALL projection the per-result term IS the
163 # declared item size, so the bound inherits the user's input error, and under-declaring
164 # item size is a common mistake. 1.5x absorbs a ~33% input shortfall.
165 # Applies ONLY to the gate, never to a reported cost.
166 VECTOR_SPEND_GATE_SAFETY = 1.5
167
168 # Service limits worth failing loudly on rather than silently mispricing.
169 MAX_VECTOR_INDEXES_PER_TABLE = 5
170 MAX_VECTOR_DIMENSIONS = 4096
171 MAX_VECTOR_TOP_K = 100
172
173 # -----------------------------------------------------------------------------
174 # Operation sets.
175 # -----------------------------------------------------------------------------
176 READ_OPS = { "GetItem" , "Query" , "Scan" , "BatchGetItem" , "TransactGetItems" }
177 WRITE_OPS = { "PutItem" , "UpdateItem" , "DeleteItem" , "BatchWriteItem" , "TransactWriteItems" }
178 TRANSACTIONAL_OPS = { "TransactGetItems" , "TransactWriteItems" }
179 MULTI_ITEM_READ = { "Query" , "Scan" , "BatchGetItem" , "TransactGetItems" }
180 # SearchVectors is deliberately NOT in READ_OPS. It consumes no RCU — it is billed on
181 # bytes examined — so routing it through the RCU path would silently misprice it.
182 VECTOR_SEARCH_OP = "SearchVectors"
183
184 # Default write_action when requirements aren't provided.
185 DEFAULT_WRITE_ACTION = {
186 "PutItem" : "create" ,
187 "UpdateItem" : "update" ,
188 "DeleteItem" : "delete" ,
189 "BatchWriteItem" : "mixed" ,
190 "TransactWriteItems" : "mixed" ,
191 }
192
193 # Fraction of writes that create new items, by write_action.
194 CREATE_RATIO = {
195 "create" : 1.0 ,
196 "update" : 0.0 ,
197 "delete" : 0.0 ,
198 "mixed" : 0.3 ,
199 }
200
201 DEFAULT_RETENTION_DAYS = 30
202
203 # Fraction of effective item bytes projected, by GSI projection type.
204 # Used to size the GSI write that fires as amplification.
205 PROJECTION_WRITE_RATIO = {
206 "ALL" : 1.0 ,
207 "INCLUDE" : 0.3 , # conservative default when include list is unknown
208 "KEYS_ONLY" : 0.1 , # keys + optional sort key, typically ≤ 1 KB
209 }
210
211 # Storage ratio relative to base table, by GSI projection type.
212 PROJECTION_STORAGE_RATIO = {
213 "ALL" : 1.0 ,
214 "INCLUDE" : 0.3 ,
215 "KEYS_ONLY" : 0.1 ,
216 }
217
218 DISCLAIMER = (
219 "> **Disclaimer:** This estimate covers **read/write request costs** and "
220 "**storage costs** only, at DynamoDB Standard table class on-demand **full "
221 "public rates** (the 25 GB storage free tier is NOT applied — it is "
222 "account-wide and often already used). **Rates vary by region** — confirm "
223 "the figure against the AWS DynamoDB pricing page (or `aws pricing "
224 "get-products --service-code AmazonDynamoDB`) for the region you will "
225 "deploy in before quoting it. The headline figure "
226 "is **peak-sustained** — it assumes every pattern runs at its declared "
227 "`peak_rps` continuously, 24/7 for a month (the worst case). Real on-demand "
228 "spend tracks actual request volume; set `avg_rps` on patterns to also get "
229 "an expected average-volume figure. For up-to-date pricing, refer to "
230 "the [Amazon DynamoDB Pricing](https://aws.amazon.com/dynamodb/pricing/) page."
231 )
232
233 GSI_FOOTNOTE = (
234 "¹ **GSI additional writes** — When a table write changes attributes "
235 "projected into a GSI, DynamoDB performs an additional write to that index. "
236 "The additional write is sized by the projected attributes (not the base "
237 "item), which is why KEYS_ONLY / INCLUDE projections are cheaper. "
238 "[Learn more](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html"
239 "#GSI.ThroughputConsiderations.Writes)"
240 )
241
242 # What a SearchVectors row shows in the Monthly Cost column. Never a dollar amount:
243 # search consumes no RCU/WCU and its byte cost is deliberately unpriced, so any figure
244 # there would be 0.00 — and "$0.00" in the table a reader scans to find expensive
245 # patterns asserts that vector search is free. It is not; it is unmeasured.
246 VECTOR_SEARCH_CELL = "not priced²"
247
248 VECTOR_SEARCH_FOOTNOTE = (
249 "² **Vector search is not priced here, and is NOT $0** — `SearchVectors` consumes no "
250 "RCU/WCU; it bills on vector bytes examined plus bytes returned, at "
251 f "$ { VECTOR_SEARCH_PRICE_PER_GB :.3f} /GB. The examined fraction is not derivable from a "
252 "design (measured to vary by an order of magnitude across index configurations), so no "
253 "figure is emitted rather than a wrong one. **This means the headline total above "
254 "excludes vector search cost.** See *Vector Index Capacity* below for the measured "
255 "drivers and how to obtain the real number."
256 )
257
258
259 # =============================================================================
260 # Core capacity-consumption formulas (verified against live DynamoDB).
261 # =============================================================================
262 def wru_for_item (item_size_bytes: int ) -> int :
263 """WRUs consumed for a standard write of a given item size.
264
265 Empirically verified: ceil(size / 1024), with a minimum of 1 WRU.
266 """
267 if item_size_bytes <= 0 :
268 return 1
269 return math.ceil(item_size_bytes / WCU_SIZE_BYTES )
270
271
272 def rru_for_item (item_size_bytes: int , strong: bool = False ) -> float :
273 """RRUs consumed reading a single item.
274
275 Strong: ceil(size / 4096), min 1. Eventual: half of strong, min 0.5.
276 Non-existent items still consume the minimum (verified).
277 """
278 strong_rru = max ( 1 , math.ceil( max ( 1 , item_size_bytes) / RCU_SIZE_BYTES ))
279 return float (strong_rru) if strong else strong_rru / 2.0
280
281
282 def rru_for_query (total_bytes_read: int , strong: bool = False ) -> float :
283 """RRUs for a Query/Scan that reads `total_bytes_read` from storage.
284
285 Query/Scan aggregates all items, rounds the TOTAL to next 4 KB, then
286 halves for eventual consistency. (Verified: 5×1KB items = 1 RCU eventual,
287 10×512B items = 1 RCU eventual, etc.)
288 """
289 strong_rru = max ( 1 , math.ceil( max ( 1 , total_bytes_read) / RCU_SIZE_BYTES ))
290 return float (strong_rru) if strong else strong_rru / 2.0
291
292
293 def rru_for_batch_read (item_sizes: list[ int ], strong: bool = False ) -> float :
294 """BatchGetItem rounds each item individually, then sums. Verified."""
295 total = 0.0
296 for sz in item_sizes:
297 total += rru_for_item(sz, strong = strong)
298 return total
299
300
301 def wru_for_batch_write (item_sizes: list[ int ]) -> int :
302 """BatchWriteItem rounds each item individually, then sums. Verified."""
303 return sum (wru_for_item(sz) for sz in item_sizes)
304
305
306 # =============================================================================
307 # GSI amplification (verified empirically).
308 # =============================================================================
309 def gsi_write_wru (
310 base_item_size_bytes: int , projection_type: str , projection_ratio_override: float | None = None
311 ) -> int :
312 """WRU consumed for the GSI write that amplifies from a base-table write.
313
314 The GSI item size is driven by PROJECTED attributes only. Empirically:
315 - ALL projection on a 4 KB base: GSI consumed = 4 WCU (full item)
316 - KEYS_ONLY / INCLUDE on a 4 KB base: GSI consumed = 1 WCU (tiny projected item)
317 """
318 ratio = (
319 projection_ratio_override
320 if projection_ratio_override is not None
321 else PROJECTION_WRITE_RATIO .get(projection_type, 1.0 )
322 )
323 projected_size = max ( 1 , int (base_item_size_bytes * ratio))
324 return wru_for_item(projected_size)
325
326
327 # =============================================================================
328 # Model loading and helpers.
329 # =============================================================================
330 def load_model (path: str ) -> dict :
331 with open (path) as f:
332 return json.load(f)
333
334
335 def _build_entity_attr_sizes (tables: list[ dict ]) -> dict :
336 """Build {entity_name: {attr_name: size_bytes}} from entity definitions."""
337 # Heuristic sizes used by modeling instructions.
338 type_sizes = {
339 "S" : 100 ,
340 "N" : 8 ,
341 "BOOL" : 1 ,
342 "B" : 256 ,
343 "L" : 200 ,
344 "M" : 200 ,
345 "SS" : 200 ,
346 "NS" : 200 ,
347 "BS" : 200 ,
348 "NULL" : 1 ,
349 }
350 # Attribute-name overhead. Average name ~10 bytes.
351 name_overhead = 10
352 result = {}
353 for t in tables:
354 for ent in t.get( "entities" , []):
355 attr_sizes = {}
356 for attr in ent.get( "attributes" , []):
357 attr_sizes[attr[ "name" ]] = (
358 type_sizes.get(attr.get( "type" , "S" ), 100 ) + name_overhead
359 )
360 result[ent[ "entity_name" ]] = attr_sizes
361 return result
362
363
364 def _cap_items_per_request (ap: dict , entity_sizes: dict ) -> int :
365 """Cap items_per_request at the 1 MB page limit using projected attribute sizes."""
366 items = ap.get( "items_per_request" , 1 )
367 if ap[ "operation" ] not in ( "Query" , "Scan" ):
368 return items
369
370 projection = ap.get( "projection" )
371 if not projection:
372 return items
373
374 projected_bytes = 0
375 for attr in projection:
376 found = False
377 for ent_key, attr_sizes in entity_sizes.items():
378 if attr in attr_sizes:
379 projected_bytes += attr_sizes[attr]
380 found = True
381 break
382 if not found:
383 projected_bytes += 100 # default fallback per attribute
384
385 if projected_bytes <= 0 :
386 return items
387
388 page_cap_bytes = PAGE_CAP_KB * 1024
389 items_per_page = max ( 1 , page_cap_bytes // projected_bytes)
390 return min (items, items_per_page)
391
392
393 def _resolve_write_action (ap: dict , requirements: dict | None ) -> str :
394 wa = ap.get( "write_action" )
395 if wa:
396 return wa
397 if requirements:
398 source_req = ap.get( "source_requirement" , "" )
399 updates = requirements.get( "updates" , {})
400 if source_req in updates:
401 wa = updates[source_req].get( "write_action" )
402 if wa:
403 return wa
404 return DEFAULT_WRITE_ACTION .get(ap[ "operation" ], "mixed" )
405
406
407 def _resolve_retention_days (ap: dict , tables: list[ dict ], requirements: dict | None ) -> int :
408 # A per-pattern retention_days (documented in cost-model-schema.md) is the
409 # most specific signal and wins over the requirements lookup and the default.
410 # Without this, a write pattern declaring e.g. retention_days: 365 was
411 # silently priced at the 30-day default, understating storage ~12×.
412 if "retention_days" in ap:
413 return int (ap[ "retention_days" ])
414 if not requirements:
415 return DEFAULT_RETENTION_DAYS
416 entity_name = None
417 table_name = ap.get( "table" , "" )
418 for t in tables:
419 if t[ "table_name" ] == table_name:
420 entities = t.get( "entities" , [])
421 if entities:
422 entity_name = entities[ 0 ].get( "entity_name" )
423 break
424 if entity_name:
425 ent_def = requirements.get( "entities" , {}).get(entity_name, {})
426 ret = ent_def.get( "retention_days" )
427 if ret is not None :
428 return ret
429 ret = requirements.get( "metadata" , {}).get( "retention_days_default" )
430 if ret is not None :
431 return ret
432 return DEFAULT_RETENTION_DAYS
433
434
435 def _fmt (cost: float ) -> str :
436 return f "$ { cost :.2f} "
437
438
439 def _padded_table (headers: list[ str ], rows: list[list[ str ]]) -> str :
440 widths = [ len (h) for h in headers]
441 for row in rows:
442 for i, cell in enumerate (row):
443 if i < len (widths):
444 widths[i] = max (widths[i], len (cell))
445 lines = [
446 "| " + " | " .join(h.ljust(widths[i]) for i, h in enumerate (headers)) + " |" ,
447 "| " + " | " .join( "-" * w for w in widths) + " |" ,
448 ]
449 for row in rows:
450 lines.append(
451 "| "
452 + " | " .join(
453 row[i].ljust(widths[i]) if i < len (row) else "" .ljust(widths[i])
454 for i in range ( len (headers))
455 )
456 + " |"
457 )
458 return " \n " .join(lines)
459
460
461 # =============================================================================
462 # Per-access-pattern capacity and cost.
463 # =============================================================================
464 def calc_pattern_capacity (ap: dict ) -> dict :
465 """Compute {rcus, wcus, notes} for one access pattern.
466
467 Returns a dict with:
468 op, rcus, wcus, strong, transactional, notes
469 Only read OR write is non-zero for a given pattern.
470 """
471 op = ap[ "operation" ]
472 size = ap.get( "estimated_item_size_bytes" , 1024 )
473 items = ap.get( "items_per_request" , 1 )
474 strong = ap.get( "consistency" , "eventual" ) == "strong"
475 transactional = op in TRANSACTIONAL_OPS
476 notes = []
477
478 rcus = 0.0
479 wcus = 0.0
480
481 if op == "GetItem" :
482 rcus = rru_for_item(size, strong = strong)
483 if ap.get( "non_existent_rate" ):
484 notes.append( "Non-existent items still consume the minimum RRU." )
485
486 elif op == "Query" :
487 total_bytes = size * items
488 rcus = rru_for_query(total_bytes, strong = strong)
489 if ap.get( "filter" ):
490 notes.append( "FilterExpression does not reduce cost." )
491 if ap.get( "projection" ):
492 notes.append( "ProjectionExpression does not reduce cost." )
493
494 elif op == "Scan" :
495 total_bytes = size * items
496 rcus = rru_for_query(total_bytes, strong = strong)
497 notes.append( "Scan reads every evaluated item regardless of filter." )
498
499 elif op == "BatchGetItem" :
500 item_sizes = ap.get( "item_sizes" ) or [size] * items
501 rcus = rru_for_batch_read(item_sizes, strong = strong)
502
503 elif op == "TransactGetItems" :
504 item_sizes = ap.get( "item_sizes" ) or [size] * items
505 rcus = rru_for_batch_read(item_sizes, strong = True ) * 2.0
506 notes.append( "Transactional reads consume 2× RRUs." )
507
508 elif op == "PutItem" :
509 # Replacement costs the LARGER of before/after.
510 before = ap.get( "previous_item_size_bytes" , 0 )
511 wcus = wru_for_item( max (size, before))
512
513 elif op == "UpdateItem" :
514 before = ap.get( "previous_item_size_bytes" , size)
515 wcus = wru_for_item( max (size, before))
516
517 elif op == "DeleteItem" :
518 wcus = wru_for_item(size)
519
520 elif op == "BatchWriteItem" :
521 item_sizes = ap.get( "item_sizes" ) or [size] * items
522 wcus = float (wru_for_batch_write(item_sizes))
523
524 elif op == "TransactWriteItems" :
525 item_sizes = ap.get( "item_sizes" ) or [size] * items
526 # Txn multiplier applies to base table only. GSI amplification is
527 # billed elsewhere (standard 1× per GSI write).
528 wcus = float (wru_for_batch_write(item_sizes)) * 2.0
529 notes.append(
530 "Transactional writes: 2× multiplier on base table; "
531 "GSI amplification remains 1× per GSI write."
532 )
533
534 # Conditional-failure handling
535 cond_fail_rate = ap.get( "conditional_fail_rate" , 0.0 )
536 if cond_fail_rate and op in WRITE_OPS :
537 # Failed writes still consume capacity sized by existing/new item.
538 # We assume the rate is a fraction of total calls; those calls still
539 # compute WRUs the same way.
540 notes.append(
541 f "Conditional failures (rate= { cond_fail_rate } ) still " f "consume the same WRUs."
542 )
543
544 return {
545 "op" : op,
546 "rcus" : rcus,
547 "wcus" : wcus,
548 "strong" : strong,
549 "transactional" : transactional,
550 "notes" : notes,
551 }
552
553
554 def _entity_attr_names (table_def: dict ) -> dict :
555 """{entity_name: set(attribute names)} for one table.
556
557 A DynamoDB item only appears in (and only amplifies a write to) a GSI whose
558 key attributes it actually CARRIES. In a single-table design with
559 heterogeneous entities, each GSI is keyed on an attribute only some entities
560 have — so this per-entity attribute-name set is what decides GSI membership.
561 """
562 out = {}
563 for ent in table_def.get( "entities" , []) or []:
564 name = ent.get( "entity_name" )
565 if not name:
566 continue
567 out[name] = {a[ "name" ] for a in ent.get( "attributes" , []) or [] if a.get( "name" )}
568 return out
569
570
571 def _gsi_key_attrs (gsi: dict ) -> tuple :
572 """(pk_attr, sk_attr|None) for a GSI, tolerating key-name spellings."""
573 pk = gsi.get( "partition_key" ) or gsi.get( "hash_key" )
574 sk = gsi.get( "sort_key" ) or gsi.get( "range_key" )
575 return pk, sk
576
577
578 def _table_key_attrs (table_def: dict ) -> set :
579 """The table's own PK/SK attribute names — every item carries these."""
580 ks = table_def.get( "key_schema" ) or {}
581 return {a for a in (ks.get( "partition_key" ), ks.get( "sort_key" )) if a}
582
583
584 def _item_carries_gsi_key (attr_names: set , gsi: dict , table_key_attrs: set | None = None ) -> bool :
585 """True iff an item with these attributes is INDEXED by this GSI.
586
587 Membership rule, verified live (ReturnConsumedCapacity=INDEXES): an item is
588 written to a GSI only if it carries the GSI partition key AND, for a
589 composite GSI, the GSI sort key. An item carrying the PK but missing the SK
590 of a composite GSI is NOT indexed (live: a g2-without-g2sk put fired ZERO
591 GSI writes). A GSI key that is also a TABLE key is carried by every item.
592 """
593 tk = table_key_attrs or set ()
594 pk, sk = _gsi_key_attrs(gsi)
595 if not pk or (pk not in attr_names and pk not in tk):
596 return False
597 if sk and sk not in attr_names and sk not in tk:
598 return False
599 return True
600
601
602 def _gsi_membership_determinable (table_def: dict , gsi: dict ) -> bool :
603 """Can we reason about which items belong to this GSI from the model?
604
605 Determinable only when the GSI's partition key is positively declared
606 somewhere we can attribute to items: by at least one entity's attribute
607 list, or as a table key (every item carries table keys). A GSI key declared
608 nowhere — or only at table level (`attribute_definitions`, which can't say
609 WHICH entity carries it) — is INDETERMINATE: we must not infer absence and
610 silently drop the GSI's cost. Callers fall back to the conservative
611 charge-everything bound for such a GSI.
612 """
613 pk, sk = _gsi_key_attrs(gsi)
614 if not pk:
615 return False
616 table_keys = _table_key_attrs(table_def)
617 declared = set (table_keys)
618 for ent in table_def.get( "entities" ) or []:
619 declared |= {a[ "name" ] for a in (ent.get( "attributes" ) or []) if a.get( "name" )}
620 # Both the GSI PK and (for a composite GSI) the GSI SK must be declared
621 # somewhere we can attribute to items, or we cannot reason about which items
622 # are members and must fall back to the conservative full-size charge.
623 if pk not in declared:
624 return False
625 if sk and sk not in declared:
626 return False
627 return True
628
629
630 def _projection_type (gsi: dict ) -> str :
631 proj = gsi.get( "projection" , {}) or {}
632 ptype = proj.get( "type" , "ALL" )
633 return ptype.upper() if isinstance (ptype, str ) else "ALL"
634
635
636 def _resolve_written_items (ap: dict , table_def: dict ) -> tuple[ list , bool ]:
637 """Resolve the per-item entity attribute-name sets a single call writes.
638
639 Returns (item_attr_sets, resolved). Each element of item_attr_sets is the
640 attribute-name set of ONE written item, so a transaction creating 1 order +
641 3 line items yields 4 sets. `resolved=False` means we could not identify the
642 written entities and the caller must fall back to a conservative upper bound.
643
644 Resolution order:
645 1. explicit `entities_written` on the access pattern — a list of entity
646 names (repeats allowed) or `[{"entity": "X", "count": n}]`.
647 2. exactly one entity declared on the table — the unambiguous single-entity
648 / classic-multi-table case; each of the items_per_request written items
649 is that entity.
650 3. otherwise unresolved (multi-entity single-table design with no
651 entities_written): conservative upper bound.
652 """
653 ent_attrs = _entity_attr_names(table_def)
654 items_per = int (ap.get( "items_per_request" , 1 ) or 1 )
655
656 ew = ap.get( "entities_written" )
657 if ew:
658 flat = []
659 for e in ew:
660 if isinstance (e, dict ):
661 name = e.get( "entity" ) or e.get( "entity_name" )
662 cnt = int (e.get( "count" , 1 ) or 1 )
663 else :
664 name, cnt = e, 1
665 # A named entity we can't find contributes items that carry no known
666 # key — i.e. amplify nowhere. That's the honest reading of a typo'd
667 # or undeclared entity name; it surfaces as a visibly low number
668 # rather than silently inheriting the full fan-out.
669 attrs = ent_attrs.get(name, set ())
670 flat.extend([attrs] * max ( 1 , cnt))
671 return flat, True
672
673 if len (ent_attrs) == 1 :
674 only = next ( iter (ent_attrs.values()))
675 return [only] * items_per, True
676
677 return [], False
678
679
680 def _gsi_index_writes_for_item (
681 item_attrs: set , gsi: dict , op: str , attrs_written, table_key_attrs: set | None = None
682 ) -> int :
683 """How many index writes ONE item's write costs against this GSI: 0, 1, or 2.
684
685 Membership (carries the GSI key) is necessary for every op. Verified live
686 against ReturnConsumedCapacity=INDEXES:
687 - Put / create / Delete, or an UpdateItem with no attrs_written info:
688 1 index write per member item (the projected-size write).
689 - UpdateItem that CHANGES the GSI's own key attribute (PK or SK): **2**
690 index writes — DynamoDB deletes the old index entry and inserts a new
691 one (a "ByStatus" KEYS_ONLY index cost 2 WCU when `status` changed).
692 - UpdateItem that changes a NON-key but projected attribute:
693 ALL → 1 (the whole item is re-projected);
694 INCLUDE → 1 if a projected attr changed, else 0;
695 KEYS_ONLY → 0 (nothing projected but the keys, which didn't move).
696 - UpdateItem touching nothing the GSI projects and no key it indexes: 0.
697 Each returned write is later sized by gsi_write_wru (ALL=full item,
698 INCLUDE/KEYS_ONLY=small).
699 """
700 if not _item_carries_gsi_key(item_attrs, gsi, table_key_attrs):
701 return 0
702 if op == "UpdateItem" and attrs_written is not None :
703 ptype = _projection_type(gsi)
704 pk, sk = _gsi_key_attrs(gsi)
705 keys = {k for k in (pk, sk) if k}
706 key_moved = bool ( set (attrs_written) & keys)
707 if key_moved:
708 # Old entry deleted + new entry inserted on this index.
709 return 2
710 if ptype == "ALL" :
711 return 1
712 if ptype == "INCLUDE" :
713 proj = gsi.get( "projection" , {}) or {}
714 inc = set (
715 proj.get( "attributes" )
716 or proj.get( "non_key_attributes" )
717 or proj.get( "NonKeyAttributes" )
718 or []
719 )
720 return 1 if ( set (attrs_written) & inc) else 0
721 if ptype == "KEYS_ONLY" :
722 return 0 # keys didn't move (handled above) and nothing else projected
723 return 1
724
725
726 def calc_gsi_amplification_wru (ap: dict , table_def: dict ) -> tuple[ float , list[ str ]]:
727 """Compute per-request WRU consumed by GSI amplification for this write.
728
729 Returns (total_wru_amp, details_strings).
730
731 A write amplifies to a GSI only when the written item CARRIES that GSI's key
732 attributes (PK, plus SK if the GSI is composite) — verified live against
733 ReturnConsumedCapacity=INDEXES. In a single-table design with heterogeneous
734 entities, each GSI is keyed on an attribute only some entities have, so a
735 given write touches only the few GSIs its entity is a member of — not all of
736 them. The per-GSI write is sized by the PROJECTED attributes (ALL = full
737 item; INCLUDE / KEYS_ONLY = small), which the live run also confirmed.
738 """
739 if ap[ "operation" ] not in WRITE_OPS :
740 return 0.0 , []
741 gsis = table_def.get( "gsis" , []) or []
742 if not gsis:
743 return 0.0 , []
744
745 op = ap[ "operation" ]
746 size = ap.get( "estimated_item_size_bytes" , 1024 )
747 attrs_written = ap.get( "attributes_written" ) # optional: list of attr names
748 items_per = int (ap.get( "items_per_request" , 1 ) or 1 )
749
750 written_items, resolved = _resolve_written_items(ap, table_def)
751 table_keys = _table_key_attrs(table_def)
752 total = 0.0
753 details: list[ str ] = []
754
755 if resolved:
756 # Accurate path: each written item amplifies only to the GSIs whose key
757 # it carries. A GSI whose key is declared NOWHERE attributable to an item
758 # (not on any entity, not a table key) is INDETERMINATE — we can't infer
759 # absence, so charge it conservatively (fires for every item, honoring
760 # the UpdateItem projection gate) rather than silently dropping its cost.
761 per_gsi_writes: dict = {} # index_name -> total index writes (1 or 2 each)
762 indeterminate: list[ str ] = []
763 member_but_gated = [] # member items whose UpdateItem projection gate excluded them
764 for gsi in gsis:
765 name = gsi.get( "index_name" , "?" )
766 determinable = _gsi_membership_determinable(table_def, gsi)
767 gsi_gated_member = False
768 for item_attrs in written_items:
769 if determinable:
770 is_member = _item_carries_gsi_key(item_attrs, gsi, table_keys)
771 writes = _gsi_index_writes_for_item(
772 item_attrs, gsi, op, attrs_written, table_keys
773 )
774 if is_member and writes == 0 :
775 gsi_gated_member = True
776 else :
777 # Indeterminate membership → conservative: count it unless the
778 # UpdateItem projection gate provably excludes it.
779 pk_a, sk_a = _gsi_key_attrs(gsi)
780 synthetic = {pk_a} | ({sk_a} if sk_a else set ())
781 writes = _gsi_index_writes_for_item(
782 synthetic, gsi, op, attrs_written, table_keys
783 )
784 if writes:
785 total += gsi_write_wru(size, _projection_type(gsi)) * writes
786 per_gsi_writes[name] = per_gsi_writes.get(name, 0 ) + writes
787 if not determinable and name in per_gsi_writes:
788 indeterminate.append(name)
789 if gsi_gated_member and name not in per_gsi_writes:
790 member_but_gated.append(name)
791 if per_gsi_writes:
792 details = [
793 f " { name } (× { n } index write { 's' if n != 1 else '' } "
794 + (
795 ", membership unverified — GSI key not declared on any entity"
796 if name in indeterminate
797 else ""
798 )
799 + ")"
800 for name, n in per_gsi_writes.items()
801 ]
802 elif member_but_gated:
803 details = [
804 f "no GSI amplification — { op } touched no attribute "
805 f "projected into { ', ' .join(member_but_gated) } "
806 f "(projection gate)"
807 ]
808 else :
809 details = [ "no GSI amplification — written item(s) carry no GSI key" ]
810 return total, details
811
812 # Unresolved: multi-entity single-table design with no `entities_written`.
813 # We cannot know which entity each written item is, so we keep the
814 # conservative UPPER BOUND (every item is assumed a member of every GSI) and
815 # say so loudly — declaring `entities_written` on the pattern refines it. A
816 # synthetic full-key item drives the same write-count logic (incl. the
817 # UpdateItem projection gate and the key-move 2× factor) as the accurate path.
818 for gsi in gsis:
819 pk_a, sk_a = _gsi_key_attrs(gsi)
820 synthetic = {a for a in (pk_a, sk_a) if a}
821 writes = _gsi_index_writes_for_item(synthetic, gsi, op, attrs_written, table_keys)
822 total += gsi_write_wru(size, _projection_type(gsi)) * writes
823 if op in ( "BatchWriteItem" , "TransactWriteItems" ):
824 total *= items_per
825 details.append(
826 f "UPPER BOUND — ' { ap.get( 'table' , '?' ) } ' has multiple entities and this "
827 f "pattern has no `entities_written`, so every written item is charged "
828 f "against every GSI. Declare `entities_written` to bill only the GSIs "
829 f "each written entity is a member of."
830 )
831 return total, details
832
833
834 # =============================================================================
835 # Vector index costs.
836 #
837 # Modelled honestly: storage and writes are derivable from the design, searches are
838 # not. Every function here labels which side of that line it is on, and the report
839 # carries the caveat so a user never mistakes the search figure for a measurement.
840 # =============================================================================
841 def _vector_indexes (table_def: dict | None ) -> list :
842 return (table_def or {}).get( "vector_indexes" , []) or []
843
844
845 def _ia_multiplier (table_def: dict | None ) -> float :
846 """Standard-IA charges 125% on both vector request dimensions.
847
848 Matching on "IA" alone was wrong, and wrong in the silent direction. DynamoDB's actual
849 table-class enum is `STANDARD_INFREQUENT_ACCESS`, which does NOT contain the substring
850 "IA" -- so a model written with the real AWS value was priced at 1.0x and under-reported
851 Standard-IA vector requests by 25% with every other number looking correct. "INFREQUENT"
852 is the reliable token; "IA" stays for the shorthand spellings an author might reasonably
853 write by hand.
854 """
855 tclass = ((table_def or {}).get( "table_class" ) or "STANDARD" ).upper()
856 return IA_REQUEST_MULTIPLIER if "INFREQUENT" in tclass or "IA" in tclass else 1.0
857
858
859 def _vector_projection (vi: dict ) -> dict :
860 """The index's projection block, accepting the bare-string shorthand.
861
862 `"projection": "KEYS_ONLY"` is the natural thing to write — it is how the API-level value
863 reads — and treating it as a dict raised a bare `AttributeError: 'str' object has no
864 attribute 'get'` that said nothing about the model. Normalised here rather than at each
865 call site, and it mirrors the `attribute_definitions` `{"name","type"}` shorthand these
866 scripts already accept.
867 """
868 proj = vi.get( "projection" , {}) or {}
869 return { "type" : proj} if isinstance (proj, str ) else proj
870
871
872 def _vector_item_bytes (vi: dict , table_def: dict , entity_attr_sizes: dict | None ) -> int :
873 """Bytes one item contributes to one vector index. Measured semantics, not a ratio.
874
875 A controlled probe (two indexes on the SAME attribute and dimensions, differing only in
876 projection) established:
877
878 KEYS_ONLY flat. Insensitive to item content: 4,105 / 4,104 / 4,104 B on a 1024-dim
879 index across a minimal item, a +10 KB item, and an item with a second
880 vector. So it is dimensions x 4 plus a small key overhead — NOT a fraction
881 of item size, which is what an earlier ratio-based model wrongly assumed.
882 ALL dimensions x 4 plus essentially the whole rest of the item. Adding a
883 10,240 B payload added 10,246 B. It also copies UNINDEXED vector
884 attributes, in their base-table representation (an unindexed 1024-dim
885 vector added 5,633 B, ~5.5 B/number rather than 4).
886 INCLUDE dimensions x 4 plus the named attributes.
887
888 Consequence worth stating plainly: on a minimal item ALL costs the SAME as KEYS_ONLY.
889 The cost of ALL is entirely the cost of the rest of the item.
890
891 CONVENTION: `estimated_item_size_bytes` EXCLUDES the vector attributes. The ALL branch
892 adds the declared item size to this index's own `dimensions x 4`, so a declared size
893 that already counted the embedding bills it twice and overstates an ALL index by
894 roughly 2x. This is the direction the probe above was measured in, and it is the more
895 accurate one -- `dimensions` is exact where an item-size estimate is not. Documented in
896 references/cost-model-schema.md against both the field and the vector section; the
897 alternative (subtracting a vector footprint from the declared size) would silently
898 under-count every model that already follows the documented convention.
899 """
900 dims = int (vi.get( "dimensions" , 0 ) or 0 )
901 vector_bytes = dims * BYTES_PER_F32
902 proj = _vector_projection(vi)
903 ptype = (proj.get( "type" ) or "ALL" ).upper()
904
905 if ptype == "KEYS_ONLY" :
906 return int (vector_bytes + VECTOR_KEY_OVERHEAD_BYTES )
907
908 if ptype == "INCLUDE" :
909 named = proj.get( "attributes" , []) or []
910 extra, _ = _named_attr_bytes(table_def, named)
911 return int (vector_bytes + VECTOR_KEY_OVERHEAD_BYTES + extra)
912
913 # ALL: the whole rest of the item rides along on every vector write.
914 entities = table_def.get( "entities" , []) or []
915 if entities:
916 non_vector = sum (e.get( "estimated_item_size_bytes" , 1024 ) for e in entities) / len (entities)
917 else :
918 non_vector = 1024.0
919 # Other vector attributes declared on this table also get copied, in base-table form.
920 own_attr = vi.get( "vector_attribute" )
921 others = 0.0
922 for other in table_def.get( "vector_indexes" , []) or []:
923 if other.get( "vector_attribute" ) and other.get( "vector_attribute" ) != own_attr:
924 others += int (other.get( "dimensions" , 0 ) or 0 ) * BYTES_PER_BASE_TABLE_NUMBER
925 return int (vector_bytes + VECTOR_KEY_OVERHEAD_BYTES + non_vector + others)
926
927
928 def _vector_indexed_item_count (vi: dict , table_def: dict ) -> tuple[ int , bool ]:
929 """(item count in this vector index, was it declared explicitly?).
930
931 Only items carrying the vector attribute — and the SearchSchema partition key, if
932 one is defined — are replicated into the index. The model can declare that directly
933 with `estimated_indexed_items`; otherwise we fall back to the table's entity counts,
934 which is a conservative upper bound, and say so in the report.
935 """
936 declared = vi.get( "estimated_indexed_items" )
937 if declared is not None :
938 return int (declared), True
939 total = sum (
940 e.get( "estimated_item_count" , 100_000 ) for e in (table_def.get( "entities" , []) or [])
941 )
942 return int (total or 100_000 ), False
943
944
945 def vector_write_bytes_per_call (ap: dict , table_def: dict ) -> tuple[ float , list[ str ]]:
946 """Bytes replicated into a table's vector indexes by one write call.
947
948 Mirrors the GSI amplification gate: a write only pays for an index whose vector
949 attribute it actually touched. When `attributes_written` is absent we cannot tell,
950 so we stay at the conservative upper bound (assume it did) and flag it — the same
951 convention the GSI path uses.
952
953 The 1 KB minimum is applied ONCE across all of the table's vector indexes, because
954 the floor is per request, not per index.
955 """
956 vis = _vector_indexes(table_def)
957 if not vis:
958 return 0.0 , []
959
960 written = ap.get( "attributes_written" )
961 details: list[ str ] = []
962 raw_bytes = 0.0
963 for vi in vis:
964 attr = vi.get( "vector_attribute" ) or ""
965 touched = True
966 if written is not None :
967 touched = attr in set (written)
968 if not touched:
969 details.append( f " { vi.get( 'index_name' ) } : not touched by this write" )
970 continue
971 b = _vector_item_bytes(vi, table_def, None )
972 raw_bytes += b
973 note = "" if written is not None else " (upper bound: attributes_written absent)"
974 details.append( f " { vi.get( 'index_name' ) } : { b :,} B { note } " )
975
976 if raw_bytes <= 0 :
977 return 0.0 , details
978 return max ( float (raw_bytes), float ( VECTOR_METERING_MIN_BYTES )), details
979
980
981 def _named_attr_bytes (table_def: dict , names: list ) -> tuple[ float , bool ]:
982 """(summed size of the named attributes, were they all actually declared?).
983
984 Uses the sizes declared in entities[].attributes[] where available. Falls back to the
985 generic S=100 heuristic for anything undeclared, which is coarse — measurement showed
986 a 6-character Title contributes ~9 B per result, not 100 — so an undeclared attribute
987 list overstates INCLUDE cost. The report flags when the fallback was used.
988 """
989 type_sizes = {
990 "S" : 100 ,
991 "N" : 8 ,
992 "BOOL" : 1 ,
993 "B" : 256 ,
994 "L" : 200 ,
995 "M" : 200 ,
996 "SS" : 200 ,
997 "NS" : 200 ,
998 "BS" : 200 ,
999 "NULL" : 1 ,
1000 }
1001 declared = {}
1002 for ent in table_def.get( "entities" , []) or []:
1003 for a in ent.get( "attributes" , []) or []:
1004 if a.get( "name" ):
1005 declared[a[ "name" ]] = type_sizes.get(a.get( "type" , "S" ), 100 )
1006 total, all_declared = 0.0 , True
1007 for n in names:
1008 if n in declared:
1009 total += declared[n]
1010 else :
1011 total += 100.0
1012 all_declared = False
1013 return total, all_declared
1014
1015
1016 def _vector_return_bytes_per_result (vi: dict , table_def: dict ) -> tuple[ float , str ]:
1017 """Bytes returned per search result, driven by the index projection.
1018
1019 Measured per result: ~89 B for KEYS_ONLY, ~98 B for a narrow INCLUDE, and the whole
1020 projected item for ALL (15,243 B on the probe item — roughly 170x KEYS_ONLY). This is
1021 the dominant term for ALL and negligible for KEYS_ONLY, which is why projection choice
1022 drives search cost far more than index size does.
1023 """
1024 proj = _vector_projection(vi)
1025 ptype = (proj.get( "type" ) or "ALL" ).upper()
1026 if ptype == "KEYS_ONLY" :
1027 return float ( VECTOR_RETURN_BYTES_KEYS_ONLY ), "KEYS_ONLY, keys only (~89 B measured)"
1028 if ptype == "INCLUDE" :
1029 named = proj.get( "attributes" , []) or []
1030 extra, all_declared = _named_attr_bytes(table_def, named)
1031 note = "" if all_declared else " (some sizes defaulted to 100 B — declare them to tighten)"
1032 return (
1033 VECTOR_RETURN_BYTES_INCLUDE_BASE + extra,
1034 f "INCLUDE, keys + { len (named) } attribute(s) { note } " ,
1035 )
1036 return (
1037 float (_vector_item_bytes(vi, table_def, None )),
1038 "ALL, the whole projected item is returned per result" ,
1039 )
1040
1041
1042 def vector_search_bytes_per_call (ap: dict , vi: dict , table_def: dict ) -> tuple[ float , str ]:
1043 """Bytes RETURNED by one SearchVectors call — the part that is soundly measurable.
1044
1045 This is deliberately NOT the full metered figure. Metering also includes the vector
1046 data the search examines during traversal, and measurement showed that term varies by
1047 an order of magnitude with index configuration (see the constants block). So this
1048 returns only the returned-data component, which is exactly linear in TopK and driven by
1049 the projection, and the caller reports it as a DRIVER rather than a cost.
1050 """
1051 top_k = int (ap.get( "top_k" , 10 ) or 10 )
1052 per_result, proj_basis = _vector_return_bytes_per_result(vi, table_def or {})
1053 return top_k * per_result, f "TopK { top_k } x { per_result :,.0f} B/result — { proj_basis } "
1054
1055
1056 def vector_search_bytes_spend_gate_upper (ap: dict , vi: dict , table_def: dict ) -> float :
1057 """Deliberately HIGH byte estimate for one SearchVectors call — pre-spend gate ONLY.
1058
1059 Do not use this for a reported cost. vector_search_bytes_per_call() is the reported
1060 driver and is a LOWER bound: returned data only, no traversal term. A spend gate that
1061 under-estimates fails in the expensive direction, so this adds a traversal term at the
1062 steeper of the two measured dimension slopes, then a safety factor.
1063
1064 Measured against the search points from the live probe (declared item size set to the
1065 13,048 B the ALL index actually copies — non-vector payload plus the other vector
1066 attributes in base-table form — so the estimate is not fed the measured answer):
1067
1068 projection dims TopK bound measured ratio
1069 ALL 1024 100 2,608,645 1,524,351 1.71x
1070 ALL 1024 1 57,564 15,276 3.77x
1071 KEYS_ONLY 1024 100 45,145 19,488 2.32x
1072 KEYS_ONLY 1024 1 31,929 10,677 2.99x
1073 KEYS_ONLY 128 1 4,108 2,736 1.50x
1074 INCLUDE 1024 100 60,145 20,437 2.94x
1075
1076 Loosest where the absolute cost is trivial, tightest where the money is — the same
1077 shape the cost model has. See VECTOR_SPEND_GATE_SAFETY for why the factor is load-
1078 bearing rather than padding. The gate is allowed to be loose. It is not allowed to be
1079 low.
1080 """
1081 returned, _ = vector_search_bytes_per_call(ap, vi, table_def)
1082 dims = int (vi.get( "dimensions" ) or 0 )
1083 traversal = dims * VECTOR_TRAVERSAL_BYTES_PER_DIM_UPPER
1084 bound = (returned + traversal) * VECTOR_SPEND_GATE_SAFETY
1085 return max ( float ( VECTOR_METERING_MIN_BYTES ), bound)
1086
1087
1088 def _as_int (value: object ) -> int | None :
1089 """``int(value)`` or None. Never raises.
1090
1091 Everything this function's callers inspect is user-authored JSON, so a non-numeric
1092 value has to become an actionable error string rather than a traceback. A bare
1093 ``not dims`` guard does not achieve that: ``"dimensions": "1k"`` is truthy, so it
1094 short-circuits past the guard and reaches ``int("1k")``, crashing the one function
1095 whose whole contract is to fail loudly instead of pricing something impossible.
1096
1097 ``bool`` is rejected on purpose. ``int(True)`` is 1, so ``"dimensions": true`` would
1098 otherwise validate as a legal 1-dimension index.
1099 """
1100 if isinstance (value, bool ):
1101 return None
1102 try :
1103 return int (value) # type: ignore[call-overload]
1104 except ( TypeError , ValueError ):
1105 return None
1106
1107
1108 def validate_vector_model (tables: list , access_patterns: list ) -> list[ str ]:
1109 """Hard validation. These are service constraints, so a violation means the design
1110 cannot be deployed — better to fail loudly than to price something impossible."""
1111 errors: list[ str ] = []
1112 for t in tables:
1113 vis = _vector_indexes(t)
1114 if not vis:
1115 continue
1116 tname = t.get( "table_name" , "<unnamed>" )
1117
1118 if len (vis) > MAX_VECTOR_INDEXES_PER_TABLE :
1119 errors.append(
1120 f " { tname } : { len (vis) } vector indexes exceeds the per-table "
1121 f "limit of { MAX_VECTOR_INDEXES_PER_TABLE } "
1122 )
1123 if t.get( "provisioned_capacity" ):
1124 errors.append(
1125 f " { tname } : vector indexes require on-demand capacity; this "
1126 f "table declares provisioned_capacity"
1127 )
1128
1129 by_attr: dict[ str , set ] = {}
1130 for vi in vis:
1131 name = vi.get( "index_name" , "<unnamed>" )
1132 # `index_name` is what a SearchVectors pattern's `index` resolves against, so
1133 # a missing one is unpriceable rather than merely untidy. Called out on its own
1134 # because the plausible wrong spelling is a bare `name`, which leaves every
1135 # other field looking correct — observed in a real run.
1136 if not vi.get( "index_name" ):
1137 errors.append(
1138 f " { tname } : a vector index is missing `index_name` "
1139 f "(found keys: { sorted (vi) } ) — the field is `index_name`, not `name`"
1140 )
1141 dims = vi.get( "dimensions" )
1142 dims_int = _as_int(dims)
1143 if dims_int is None or not ( 1 <= dims_int <= MAX_VECTOR_DIMENSIONS ):
1144 errors.append(
1145 f " { tname } . { name } : dimensions must be an integer 1-"
1146 f " { MAX_VECTOR_DIMENSIONS } , got { dims !r} "
1147 )
1148 fn = (vi.get( "distance_function" ) or "" ).upper()
1149 if fn not in { "COSINE" , "EUCLIDEAN" , "DOT_PRODUCT" }:
1150 errors.append(
1151 f " { tname } . { name } : distance_function must be COSINE, "
1152 f "EUCLIDEAN or DOT_PRODUCT, got { vi.get( 'distance_function' ) !r} "
1153 )
1154 if not vi.get( "vector_attribute" ):
1155 errors.append( f " { tname } . { name } : vector_attribute is required" )
1156 by_attr.setdefault(vi.get( "vector_attribute" ) or "" , set ()).add(dims_int or 0 )
1157
1158 for attr, dimset in by_attr.items():
1159 if len (dimset) > 1 :
1160 errors.append(
1161 f " { tname } : indexes on attribute { attr !r} declare differing "
1162 f "dimensions { sorted (dimset) } — DynamoDB rejects this "
1163 f "('Attributes cannot be redefined')"
1164 )
1165
1166 # Only NAMED indexes are resolvable targets. Including unnamed ones would put
1167 # (table, None) in this set, and a pattern that omits `index` also looks up
1168 # (table, None) — so a missing target would silently MATCH a missing name and the
1169 # error below would never fire. Two bugs cancelling out is not a passing design.
1170 index_names = {
1171 (t.get( "table_name" ), vi.get( "index_name" ))
1172 for t in tables
1173 for vi in _vector_indexes(t)
1174 if vi.get( "index_name" )
1175 }
1176 for ap in access_patterns:
1177 if ap.get( "operation" ) != VECTOR_SEARCH_OP :
1178 continue
1179 pid = ap.get( "pattern_id" , "<unnamed>" )
1180 top_k = ap.get( "top_k" , 10 )
1181 top_k_int = _as_int(top_k)
1182 if top_k_int is None or not ( 1 <= top_k_int <= MAX_VECTOR_TOP_K ):
1183 errors.append( f " { pid } : top_k must be an integer 1- { MAX_VECTOR_TOP_K } , got { top_k !r} " )
1184 if not ap.get( "table" ):
1185 errors.append(
1186 f " { pid } : operation SearchVectors has no `table` (found keys: "
1187 f " { sorted (ap) } ) — a vector index belongs to a table, so the pattern must "
1188 f "name it just as every other access pattern does"
1189 )
1190 continue
1191 if not ap.get( "index" ):
1192 errors.append(
1193 f " { pid } : operation SearchVectors has no `index` "
1194 f "(found keys: { sorted (ap) } ) — the field is `index`, not `vector_index`; "
1195 f "it must name an entry in the table's vector_indexes"
1196 )
1197 continue
1198 key = (ap.get( "table" ), ap.get( "index" ))
1199 if key not in index_names:
1200 errors.append(
1201 f " { pid } : operation SearchVectors targets "
1202 f " { ap.get( 'table' ) } . { ap.get( 'index' ) } , which is not declared "
1203 f "in that table's vector_indexes"
1204 )
1205 return errors
1206
1207
1208 def pattern_monthly_cost (
1209 ap: dict ,
1210 table_def: dict | None ,
1211 entity_attr_sizes: dict | None = None ,
1212 ) -> dict :
1213 """Return per-pattern monthly cost and capacity detail for one access pattern.
1214
1215 Inputs:
1216 ap — one access-pattern dict (see cost-model-schema.md).
1217 table_def — the table-def dict (used for GSI amplification); may be None.
1218 entity_attr_sizes — output of _build_entity_attr_sizes(tables); needed to cap
1219 Query/Scan items_per_request at the 900 KB page limit.
1220
1221 Returns:
1222 {
1223 "ap": the effective ap dict (post-cap),
1224 "cap": output of calc_pattern_capacity,
1225 "base_cost": monthly dollars for base-table RCU/WCU,
1226 "gsi_amp_wru": observed-equivalent GSI amp WRU per call,
1227 "gsi_amp_details": list[str] of "IndexName (PROJ): +X WRU",
1228 "gsi_amp_cost": monthly dollars for GSI amplification,
1229 "total_cost": base_cost + gsi_amp_cost,
1230 }
1231
1232 No AWS calls; no side effects. The same formulas the CLI uses.
1233 """
1234 # SearchVectors consumes no RCU/WCU — it is billed on bytes examined — so it takes
1235 # its own path rather than going through calc_pattern_capacity.
1236 if ap[ "operation" ] == VECTOR_SEARCH_OP :
1237 return _search_vectors_monthly_cost(ap, table_def)
1238
1239 if ap[ "operation" ] in ( "Query" , "Scan" ) and entity_attr_sizes is not None :
1240 capped = _cap_items_per_request(ap, entity_attr_sizes)
1241 ap = dict (ap, items_per_request = capped)
1242
1243 cap = calc_pattern_capacity(ap)
1244 rps = ap.get( "peak_rps" , 0 )
1245 # Optional expected/average-volume scenario. `avg_rps` defaults to peak_rps,
1246 # so a model without it produces byte-identical numbers to before. When set,
1247 # `expected_*` is the same per-op CU (scale-invariant) driven at the lower
1248 # average rate — the realistic monthly figure, computed by the calculator
1249 # instead of hand-derived.
1250 avg_rps = ap.get( "avg_rps" , rps)
1251
1252 per_op_cu_cost = cap[ "rcus" ] * RRU_PRICE + cap[ "wcus" ] * WRU_PRICE
1253 base_cost = per_op_cu_cost * rps * SECONDS_PER_MONTH
1254 expected_base_cost = per_op_cu_cost * avg_rps * SECONDS_PER_MONTH
1255
1256 gsi_amp_wru = 0.0
1257 gsi_amp_details: list[ str ] = []
1258 if ap[ "operation" ] in WRITE_OPS and table_def:
1259 gsi_amp_wru, gsi_amp_details = calc_gsi_amplification_wru(ap, table_def)
1260 gsi_amp_cost = gsi_amp_wru * WRU_PRICE * rps * SECONDS_PER_MONTH
1261 expected_gsi_amp_cost = gsi_amp_wru * WRU_PRICE * avg_rps * SECONDS_PER_MONTH
1262
1263 cond_fail_rate = ap.get( "conditional_fail_rate" , 0.0 )
1264 if cond_fail_rate and ap[ "operation" ] in WRITE_OPS :
1265 fail_multiplier = cond_fail_rate
1266 base_cost *= 1.0 + fail_multiplier
1267 gsi_amp_cost *= 1.0 + fail_multiplier
1268 expected_base_cost *= 1.0 + fail_multiplier
1269 expected_gsi_amp_cost *= 1.0 + fail_multiplier
1270
1271 # Vector write capacity, if this write touches a vector-indexed attribute. Billed in
1272 # bytes at a different rate from WCU, so it is a separate line rather than folded in.
1273 vec_write_bytes = 0.0
1274 vec_write_details: list[ str ] = []
1275 vec_write_cost = expected_vec_write_cost = 0.0
1276 if ap[ "operation" ] in WRITE_OPS and table_def and _vector_indexes(table_def):
1277 vec_write_bytes, vec_write_details = vector_write_bytes_per_call(ap, table_def)
1278 rate = ( VECTOR_WRITE_PRICE_PER_GB / BYTES_PER_GB ) * _ia_multiplier(table_def)
1279 vec_write_cost = vec_write_bytes * rate * rps * SECONDS_PER_MONTH
1280 expected_vec_write_cost = vec_write_bytes * rate * avg_rps * SECONDS_PER_MONTH
1281 if cond_fail_rate:
1282 vec_write_cost *= 1.0 + cond_fail_rate
1283 expected_vec_write_cost *= 1.0 + cond_fail_rate
1284
1285 return {
1286 "ap" : ap,
1287 "cap" : cap,
1288 "base_cost" : base_cost,
1289 "gsi_amp_wru" : gsi_amp_wru,
1290 "gsi_amp_details" : gsi_amp_details,
1291 "gsi_amp_cost" : gsi_amp_cost,
1292 "vector_write_bytes" : vec_write_bytes,
1293 "vector_write_details" : vec_write_details,
1294 "vector_write_cost" : vec_write_cost,
1295 "total_cost" : base_cost + gsi_amp_cost + vec_write_cost,
1296 "expected_cost" : expected_base_cost + expected_gsi_amp_cost + expected_vec_write_cost,
1297 }
1298
1299
1300 def _vector_search_cap (ap: dict , notes: list[ str ]) -> dict :
1301 """A zero-capacity `cap` matching calc_pattern_capacity's shape.
1302
1303 SearchVectors consumes no RCU/WCU, but the report renders every pattern through
1304 the same columns, so the shape must match or rendering breaks.
1305 """
1306 return {
1307 "op" : ap.get( "operation" , VECTOR_SEARCH_OP ),
1308 "rcus" : 0.0 ,
1309 "wcus" : 0.0 ,
1310 "strong" : False ,
1311 "transactional" : False ,
1312 "notes" : notes,
1313 }
1314
1315
1316 def _search_vectors_monthly_cost (ap: dict , table_def: dict | None ) -> dict :
1317 """Monthly cost for one SearchVectors pattern.
1318
1319 The dollar figure here is an UPPER BOUND, not a model — see
1320 vector_search_bytes_per_call. It is surfaced with its basis string so the report can
1321 say plainly how it was derived and that live validation supersedes it.
1322 """
1323 # No rate needed: search contributes no priced line, only reported drivers.
1324 vi = next (
1325 (v for v in _vector_indexes(table_def) if v.get( "index_name" ) == ap.get( "index" )), None
1326 )
1327 if vi is None :
1328 return {
1329 "ap" : ap,
1330 "cap" : _vector_search_cap(
1331 ap, [ "SearchVectors target index not found in the model — cost not estimated" ]
1332 ),
1333 "base_cost" : 0.0 ,
1334 "gsi_amp_wru" : 0.0 ,
1335 "gsi_amp_details" : [],
1336 "gsi_amp_cost" : 0.0 ,
1337 "vector_write_bytes" : 0.0 ,
1338 "vector_write_details" : [],
1339 "vector_write_cost" : 0.0 ,
1340 "vector_search_bytes" : 0.0 ,
1341 "vector_search_basis" : "target index not declared" ,
1342 "vector_search_cost" : 0.0 ,
1343 "is_vector_search" : True ,
1344 "total_cost" : 0.0 ,
1345 "expected_cost" : 0.0 ,
1346 }
1347
1348 per_call, basis = vector_search_bytes_per_call(ap, vi, table_def or {})
1349 # No dollar figure: the examined-data term is not modellable from a design.
1350 cost = expected = 0.0
1351 return {
1352 "ap" : ap,
1353 "cap" : _vector_search_cap(
1354 ap, [ "SearchVectors consumes no RCU/WCU — billed on vector bytes examined" ]
1355 ),
1356 "base_cost" : 0.0 ,
1357 "gsi_amp_wru" : 0.0 ,
1358 "gsi_amp_details" : [],
1359 "gsi_amp_cost" : 0.0 ,
1360 "vector_write_bytes" : 0.0 ,
1361 "vector_write_details" : [],
1362 "vector_write_cost" : 0.0 ,
1363 "vector_search_bytes" : per_call,
1364 "vector_search_basis" : basis,
1365 "vector_search_cost" : cost,
1366 "is_vector_search" : True ,
1367 "total_cost" : cost,
1368 "expected_cost" : expected,
1369 }
1370
1371
1372 def _gsi_membership_byte_fraction (table_def: dict , gsi: dict ) -> float :
1373 """Fraction of the base table's BYTES that this GSI actually indexes.
1374
1375 A GSI holds only the items that carry its key (PK + SK if composite), so its
1376 storage is base_storage × (member-entity bytes / all-entity bytes), NOT the
1377 full base table. Computed from entity declarations; an ALL-projection GSI on
1378 a single-entity table yields 1.0 (unchanged from legacy behavior).
1379
1380 Guard: the refinement only applies when the GSI's partition key is declared
1381 by at least one entity's attribute list — proof that attributes are
1382 specified meaningfully. If no entity declares the GSI PK at all (attributes
1383 underspecified, or a key that lives only in `attribute_definitions`), we
1384 cannot reason about membership and return 1.0 (conservative — never silently
1385 zeroes a GSI's storage on a thin model).
1386 """
1387 ents = table_def.get( "entities" ) or []
1388 if not ents:
1389 return 1.0
1390 # Indeterminate membership (GSI key declared on no entity and not a table
1391 # key) → conservative full size; never silently shrink a GSI's storage on a
1392 # thin model. A GSI key that IS a table key is carried by every item → 1.0.
1393 if not _gsi_membership_determinable(table_def, gsi):
1394 return 1.0
1395 table_keys = _table_key_attrs(table_def)
1396
1397 total = 0.0
1398 member = 0.0
1399 for ent in ents:
1400 cnt = ent.get( "estimated_item_count" , 100_000 )
1401 sz = ent.get( "estimated_item_size_bytes" , 1024 )
1402 b = cnt * sz
1403 total += b
1404 attrs = {a[ "name" ] for a in (ent.get( "attributes" ) or []) if a.get( "name" )}
1405 if _item_carries_gsi_key(attrs, gsi, table_keys):
1406 member += b
1407 if total <= 0 :
1408 return 1.0
1409 return member / total
1410
1411
1412 def _storage_cost (tables: list , by_table: dict ) -> tuple[ list , float ]:
1413 """Storage rows + total dollars for a given per-table byte map.
1414
1415 Factored out so the peak and the expected (average-volume) scenarios run
1416 through identical projection-ratio logic — no drift between the two
1417 headlines. `by_table` maps table_name → steady-state bytes (write-driven);
1418 tables with no write traffic fall back to entity-count storage.
1419
1420 All storage is billed at the **full public rate** — the 25 GB Standard free
1421 tier is deliberately NOT applied. The free tier is account-wide and is often
1422 already consumed by other tables in the same account, so assuming it here
1423 understates the bill and produces a "$0.00, storage is free" claim a user
1424 cannot safely repeat. Pricing at full rate is the honest, account-agnostic
1425 default.
1426 """
1427 rows = []
1428 total = 0.0
1429 for t in tables:
1430 tname = t[ "table_name" ]
1431 total_bytes = by_table.get(tname, 0 )
1432 # Fallback: if no write patterns hit this table, use entity counts.
1433 if total_bytes == 0 :
1434 for ent in t.get( "entities" , []):
1435 item_size = ent.get( "estimated_item_size_bytes" , 1024 )
1436 item_count = ent.get( "estimated_item_count" , 100_000 )
1437 overhead = PER_ITEM_STORAGE_OVERHEAD
1438 if t.get( "global_tables" ):
1439 overhead += GLOBAL_TABLES_OVERHEAD
1440 total_bytes += item_count * (item_size + overhead)
1441
1442 gb = total_bytes / BYTES_PER_GB
1443 sc = gb * STORAGE_PRICE_PER_GB_MONTH
1444 rows.append([tname, "Table" , f " { gb :.2f} " , _fmt(sc)])
1445 total += sc
1446
1447 for g in t.get( "gsis" , []):
1448 proj = g.get( "projection" , {}) or {}
1449 ptype = proj.get( "type" , "ALL" )
1450 ptype = ptype.upper() if isinstance (ptype, str ) else "ALL"
1451 ratio = PROJECTION_STORAGE_RATIO .get(ptype, 1.0 )
1452 # A GSI stores only the items that carry its key (PK + SK if
1453 # composite), not the whole base table. Scale by the fraction of
1454 # base BYTES whose entity is a member of this index. On a
1455 # single-entity table (or an underspecified model) this is 1.0, so
1456 # legacy single-table-single-entity output is unchanged.
1457 membership = _gsi_membership_byte_fraction(t, g)
1458 ggb = gb * ratio * membership
1459 gsc = ggb * STORAGE_PRICE_PER_GB_MONTH
1460 rows.append([g[ "index_name" ], "GSI" , f " { ggb :.2f} " , _fmt(gsc)])
1461 total += gsc
1462
1463 # Vector index storage. Sized from item COUNT rather than base-table bytes,
1464 # because the vector portion is a fixed dimensions × 4 regardless of how large
1465 # the rest of the item is. Priced at the same per-GB rate as table storage —
1466 # there is no separate vector-storage usage type.
1467 for vi in _vector_indexes(t):
1468 n_items, declared = _vector_indexed_item_count(vi, t)
1469 per_item = _vector_item_bytes(vi, t, None )
1470 vgb = (n_items * per_item) / BYTES_PER_GB
1471 vsc = vgb * STORAGE_PRICE_PER_GB_MONTH
1472 label = "Vector index" if declared else "Vector index*"
1473 rows.append([vi.get( "index_name" , "<unnamed>" ), label, f " { vgb :.2f} " , _fmt(vsc)])
1474 total += vsc
1475
1476 return rows, total
1477
1478
1479 # =============================================================================
1480 # Main reporting.
1481 # =============================================================================
1482 def calculate_and_report (model: dict , requirements: dict | None = None ) -> str :
1483 tables = model.get( "tables" , [])
1484 access_patterns = model.get( "access_patterns" , [])
1485
1486 # Lookups.
1487 table_map = {t[ "table_name" ]: t for t in tables}
1488 entity_attr_sizes = _build_entity_attr_sizes(tables)
1489
1490 # Per-pattern costs.
1491 results = []
1492 for ap in access_patterns:
1493 table_def = table_map.get(ap.get( "table" , "" ))
1494 pc = pattern_monthly_cost(ap, table_def, entity_attr_sizes)
1495 eff_ap = pc[ "ap" ]
1496 cap = pc[ "cap" ]
1497
1498 results.append(
1499 {
1500 "pattern_id" : eff_ap[ "pattern_id" ],
1501 "description" : eff_ap.get( "description" , "" ),
1502 "op" : eff_ap[ "operation" ],
1503 "table" : eff_ap.get( "table" , "" ),
1504 "index" : eff_ap.get( "index" ),
1505 "rps" : eff_ap.get( "peak_rps" , 0 ),
1506 "rcus" : cap[ "rcus" ],
1507 "wcus" : cap[ "wcus" ],
1508 "strong" : cap[ "strong" ],
1509 "transactional" : cap[ "transactional" ],
1510 "base_cost" : pc[ "base_cost" ],
1511 "gsi_amp_cost" : pc[ "gsi_amp_cost" ],
1512 "gsi_amp_details" : pc[ "gsi_amp_details" ],
1513 "total_cost" : pc[ "total_cost" ],
1514 "expected_cost" : pc[ "expected_cost" ],
1515 "notes" : cap[ "notes" ],
1516 # Vector capacity. Writes and storage are priced into the headline;
1517 # search is carried through as a diagnostic ceiling only.
1518 "vector_write_bytes" : pc.get( "vector_write_bytes" , 0.0 ),
1519 "vector_write_cost" : pc.get( "vector_write_cost" , 0.0 ),
1520 "vector_write_details" : pc.get( "vector_write_details" , []),
1521 "is_vector_search" : pc.get( "is_vector_search" , False ),
1522 "vector_search_bytes" : pc.get( "vector_search_bytes" , 0.0 ),
1523 "vector_search_basis" : pc.get( "vector_search_basis" , "" ),
1524 "vector_search_cost" : pc.get( "vector_search_cost" , 0.0 ),
1525 }
1526 )
1527
1528 # -------------------------------------------------------------------------
1529 # Storage estimates.
1530 # -------------------------------------------------------------------------
1531 storage_by_table: dict = {}
1532 # Parallel accumulation at avg_rps so the expected (average-volume) headline
1533 # uses average-rate storage growth, not peak — otherwise storage (often
1534 # ~10-15% of the bill) would keep the "expected" number peak-inflated.
1535 expected_storage_by_table: dict = {}
1536 for ap in access_patterns:
1537 if ap[ "operation" ] not in WRITE_OPS :
1538 continue
1539 write_action = _resolve_write_action(ap, requirements)
1540 create_ratio = CREATE_RATIO .get(write_action, 0.3 )
1541 if create_ratio <= 0 :
1542 continue
1543 rps = ap.get( "peak_rps" , 0 )
1544 if rps <= 0 :
1545 continue
1546 avg_rps = ap.get( "avg_rps" , rps)
1547
1548 raw_item_size = ap.get( "estimated_item_size_bytes" , 1024 )
1549 overhead = PER_ITEM_STORAGE_OVERHEAD
1550 # Optional: add global tables overhead if configured on the table.
1551 table_name = ap.get( "table" , "" )
1552 table_def = table_map.get(table_name, {})
1553 if table_def.get( "global_tables" ):
1554 overhead += GLOBAL_TABLES_OVERHEAD
1555 item_size = raw_item_size + overhead
1556
1557 retention_days = _resolve_retention_days(ap, tables, requirements)
1558 create_rate = rps * create_ratio
1559 steady_state_bytes = item_size * create_rate * SECONDS_PER_DAY * retention_days
1560 storage_by_table[table_name] = storage_by_table.get(table_name, 0 ) + steady_state_bytes
1561 expected_bytes = item_size * (avg_rps * create_ratio) * SECONDS_PER_DAY * retention_days
1562 expected_storage_by_table[table_name] = (
1563 expected_storage_by_table.get(table_name, 0 ) + expected_bytes
1564 )
1565
1566 storage_rows, storage_total = _storage_cost(tables, storage_by_table)
1567 # Expected (average-volume) storage uses the same logic on avg-rate bytes.
1568 # Only computed/rendered when avg_rps is in play; otherwise identical to peak.
1569 _, expected_storage_total = _storage_cost(tables, expected_storage_by_table)
1570
1571 rw_total = sum (r[ "total_cost" ] for r in results)
1572 total = storage_total + rw_total
1573 # Expected (average-volume) headline. `avg_rps` defaults to peak_rps per
1574 # pattern, so when no pattern sets it, expected_total == total exactly and
1575 # the two-headline block is suppressed (legacy byte-identical output).
1576 any_avg_rps = any ( "avg_rps" in ap for ap in access_patterns)
1577 expected_rw_total = sum (r[ "expected_cost" ] for r in results)
1578 expected_total = expected_storage_total + expected_rw_total
1579
1580 # Sort results by cost descending.
1581 results.sort( key =lambda r: r[ "total_cost" ], reverse = True )
1582
1583 # Cost concentration note: if a single pattern drives >30% of the RW
1584 # total, item-size drift on that pattern could move the headline number
1585 # materially. Surface it so the user knows where to scrutinise inputs.
1586 concentration_notes = []
1587 if rw_total > 0 :
1588 for r in results:
1589 share = r[ "total_cost" ] / rw_total
1590 if share > 0.30 :
1591 concentration_notes.append(
1592 f "- ` { r[ 'pattern_id' ] } ` contributes { share * 100 :.0f} % of "
1593 f "read/write cost. A 50% error in `estimated_item_size_bytes` "
1594 f "on this pattern would move the headline number by "
1595 f "~ { share * 50 :.0f} %. Revisit its attribute walkthrough if "
1596 f "you haven't already, or rely on live validation to "
1597 f "confirm the number."
1598 )
1599
1600 # Build report.
1601 lines = [
1602 "# DynamoDB Cost Report" ,
1603 "" ,
1604 DISCLAIMER ,
1605 "" ,
1606 ]
1607 if any_avg_rps:
1608 # Two labeled headlines: the peak-sustained worst case and the
1609 # expected figure at the stated average volume. This is the calculator
1610 # producing the "realistic" number so the agent never hand-derives it.
1611 lines += [
1612 f "**Peak-Sustained Monthly Cost: { _fmt(total) } ** *(every pattern at "
1613 "its declared `peak_rps`, sustained 24/7 — the worst case)*" ,
1614 "" ,
1615 f "**Expected Monthly Cost (your stated average volume): "
1616 f " { _fmt(expected_total) } ** *(patterns driven at `avg_rps`; on-demand "
1617 "bills per request, so this is the realistic figure)*" ,
1618 "" ,
1619 _padded_table(
1620 [ "Source" , "Peak-Sustained" , "Expected" ],
1621 [
1622 [ "Storage" , _fmt(storage_total), _fmt(expected_storage_total)],
1623 [ "Read and write requests" , _fmt(rw_total), _fmt(expected_rw_total)],
1624 ],
1625 ),
1626 ]
1627 else :
1628 lines += [
1629 f "**Peak-Sustained Monthly Cost: { _fmt(total) } ** *(every pattern at "
1630 "its declared `peak_rps`, sustained 24/7; set `avg_rps` on patterns "
1631 "for a realistic average-volume figure)*" ,
1632 "" ,
1633 _padded_table(
1634 [ "Source" , "Monthly Cost" ],
1635 [
1636 [ "Storage" , _fmt(storage_total)],
1637 [ "Read and write requests" , _fmt(rw_total)],
1638 ],
1639 ),
1640 ]
1641 if concentration_notes:
1642 lines += [
1643 "" ,
1644 "> **Cost concentration — item-size sensitivity.** One or more "
1645 "patterns drive a disproportionate share of the estimate. The "
1646 "calculator formulas are empirically verified; the remaining risk "
1647 "is in the inputs — particularly `estimated_item_size_bytes`. If "
1648 "any of these patterns' item sizes were guessed without an "
1649 "attribute walkthrough, the headline number could be off by "
1650 "tens of percent." ,
1651 "" ,
1652 * concentration_notes,
1653 ]
1654 lines += [
1655 "" ,
1656 "## Storage Costs" ,
1657 "" ,
1658 f "**Monthly Cost:** { _fmt(storage_total) } " ,
1659 "" ,
1660 "*Priced at the full public Standard rate ($0.25/GB-month). The 25 GB "
1661 "free tier is not applied — it is account-wide and often already "
1662 "consumed by other tables.*" ,
1663 "" ,
1664 _padded_table([ "Resource" , "Type" , "Storage (GB)" , "Monthly Cost" ], storage_rows),
1665 "" ,
1666 "## Access Pattern Costs" ,
1667 "" ,
1668 f "**Monthly Cost:** { _fmt(rw_total) } " ,
1669 "" ,
1670 ]
1671
1672 has_gsi_footnote = False
1673 has_vector_search_footnote = False
1674 detail_headers = [ "Pattern" , "Operation" , "Table/Index" , "Peak RPS" , "RRU/WRU" , "Monthly Cost" ]
1675 detail_rows = []
1676 for r in results:
1677 target = r[ "table" ]
1678 if r[ "index" ]:
1679 target += f " / { r[ 'index' ] } "
1680 ru = r[ "wcus" ] if r[ "wcus" ] > 0 else r[ "rcus" ]
1681 flag = ""
1682 if r[ "transactional" ]:
1683 flag += " (txn)"
1684 if r[ "strong" ] and r[ "wcus" ] == 0 :
1685 flag += " (strong)"
1686 detail_rows.append(
1687 [
1688 r[ "pattern_id" ],
1689 r[ "op" ] + flag,
1690 target,
1691 f " { r[ 'rps' ] :.1f} " ,
1692 f " { ru :.2f} " ,
1693 # SearchVectors consumes no RCU/WCU AND its capacity is deliberately
1694 # not priced, so BOTH base_cost and vector_search_cost are 0. Rendering
1695 # either as "$0.00" states that search is free — in the one table a
1696 # reader scans to find the expensive patterns. Render the carve-out
1697 # instead and footnote it. This is the whole point of the section below:
1698 # do not let an unpriced dimension read as a free one.
1699 ( VECTOR_SEARCH_CELL if r.get( "is_vector_search" ) else _fmt(r[ "base_cost" ])),
1700 ]
1701 )
1702 if r.get( "is_vector_search" ):
1703 has_vector_search_footnote = True
1704 if r[ "gsi_amp_cost" ] > 0 :
1705 detail_rows.append(
1706 [
1707 f " { r[ 'pattern_id' ] } ¹" ,
1708 "GSI writes" ,
1709 r[ "table" ],
1710 f " { r[ 'rps' ] :.1f} " ,
1711 "-" ,
1712 _fmt(r[ "gsi_amp_cost" ]),
1713 ]
1714 )
1715 has_gsi_footnote = True
1716
1717 lines.append(_padded_table(detail_headers, detail_rows))
1718
1719 if has_gsi_footnote:
1720 lines += [ "" , GSI_FOOTNOTE ]
1721
1722 if has_vector_search_footnote:
1723 lines += [ "" , VECTOR_SEARCH_FOOTNOTE ]
1724
1725 lines += _vector_report_section(results)
1726
1727 return " \n " .join(lines)
1728
1729
1730 def _vector_report_section (results: list ) -> list[ str ]:
1731 """The vector-capacity section: what was priced, and what deliberately was not.
1732
1733 Vector index storage and vector WRITE capacity are modelled and already included in
1734 the headline. Vector SEARCH capacity is not, and this section says so explicitly
1735 rather than leaving a $0.00 row unexplained. It reports a byte ceiling as a
1736 diagnostic and tells the user how to obtain the real figure.
1737 """
1738 searches = [r for r in results if r.get( "is_vector_search" )]
1739 writes = [
1740 r for r in results if r.get( "vector_write_bytes" , 0 ) and not r.get( "is_vector_search" )
1741 ]
1742 if not searches and not writes:
1743 return []
1744
1745 out = [ "" , "## Vector Index Capacity" , "" ]
1746
1747 if writes:
1748 out += [
1749 "Vector **write** capacity is metered in bytes at "
1750 f "$ { VECTOR_WRITE_PRICE_PER_GB :.2f} /GB, separately from base-table WCU, and "
1751 "**is included** in the headline above. A 1 KB minimum applies per request "
1752 "(not per index), so low-dimension vectors do not meter proportionally lower." ,
1753 "" ,
1754 ]
1755 rows = [
1756 [
1757 r.get( "pattern_id" , "?" ),
1758 f " { r[ 'vector_write_bytes' ] :,.0f} " ,
1759 _fmt(r[ "vector_write_cost" ]),
1760 "; " .join(r.get( "vector_write_details" ) or []) or "-" ,
1761 ]
1762 for r in writes
1763 ]
1764 out.append(
1765 _padded_table([ "Pattern" , "Bytes/write" , "Monthly Cost" , "Per-index detail" ], rows)
1766 )
1767 out.append( "" )
1768
1769 if searches:
1770 out += [
1771 "Vector **search** capacity is **not priced here**, and that is deliberate. It "
1772 "is metered on the vector data a search examines inside the index plus the data "
1773 "returned, and measurement showed the examined fraction varies by an order of "
1774 "magnitude with index configuration — 12.8 % o f all vector bytes on a 60-item "
1775 "unpartitioned index against 1.3 % o n a 200-item partitioned one. A calibration "
1776 "fitted across configurations came out 51-69 % lo w when tested at 256 / 1536 / "
1777 "3072 dimensions, which is exactly where real embedding models sit. A "
1778 "confidently wrong figure is worse than none." ,
1779 "" ,
1780 "What IS measured, and what you can act on:" ,
1781 "" ,
1782 "- **`TopK` is exactly linear.** Halving `TopK` halves the returned-data term." ,
1783 "- **The projection is a ~170x multiplier** on bytes returned per result: ~89 B "
1784 "for `KEYS_ONLY`, ~98 B for a narrow `INCLUDE`, and the whole projected item "
1785 "for `ALL`. A `TopK=100` search on an `ALL` index measured **1.52 MB**; the same "
1786 "search on `KEYS_ONLY` measured **19 KB**." ,
1787 "- **Dimensions scale it linearly** at a fixed index configuration (~31 B per "
1788 "dimension measured at 256 / 1536 / 3072)." ,
1789 "- **Index size is not a linear driver.** ANN prunes: within one index, going "
1790 "from 10 to 200 vectors in the searched partition barely moved the figure. "
1791 'That is not the same as "population never matters" — an EMPTY 1024-dim '
1792 "index measured ~2x a populated one, so do not extrapolate this shape down "
1793 "to a sparse or freshly-created index." ,
1794 "" ,
1795 "The returned-data component below is the part that follows soundly from your "
1796 "design. Treat it as a **lower bound on bytes**, not a cost." ,
1797 "" ,
1798 ]
1799 rows = [
1800 [
1801 r.get( "pattern_id" , "?" ),
1802 f " { r[ 'vector_search_bytes' ] / 1_000 :,.1f} KB" ,
1803 r.get( "vector_search_basis" , "-" ),
1804 ]
1805 for r in searches
1806 ]
1807 out.append(_padded_table([ "Pattern" , "Returned bytes/search" , "Basis" ], rows))
1808 out += [
1809 "" ,
1810 f "**Get the real number before you quote one.** Vector search bills at "
1811 f "$ { VECTOR_SEARCH_PRICE_PER_GB :.3f} /GB. Set `ReturnConsumedCapacity` on your "
1812 "`SearchVectors` calls and read `VectorSearchRequestBytes`, or chart the "
1813 "`VectorSearchRequestBytes` CloudWatch metric (dimensioned by `TableName` and "
1814 "`VectorIndexName`). A live validation run measures it against your own data "
1815 "and reports the observed value." ,
1816 "" ,
1817 "**The cheapest lever is the projection.** If search cost turns out to matter, "
1818 "narrowing an `ALL` vector index to `KEYS_ONLY` or a tight `INCLUDE` and "
1819 "hydrating the rest with `BatchGetItem` cuts search bytes by orders of "
1820 "magnitude. The projection is immutable, so that is a new-index migration." ,
1821 ]
1822
1823 return out
1824
1825
1826 # =============================================================================
1827 # CLI.
1828 # =============================================================================
1829 def main ():
1830 parser = argparse.ArgumentParser( description = "DynamoDB Cost Calculator — reads data model JSON" )
1831 parser.add_argument( "--model" , required = True , help = "Path to dynamodb_data_model.json" )
1832 parser.add_argument(
1833 "--requirements" ,
1834 "-r" ,
1835 help = "Path to requirements artifact JSON " "(enables write-action storage modeling)" ,
1836 )
1837 parser.add_argument(
1838 "--output" , "-o" , help = "Output file (default: cost_report.md " "in same directory)"
1839 )
1840 args = parser.parse_args()
1841
1842 model_path = Path(args.model)
1843 try :
1844 model = load_model( str (model_path))
1845 except (json.JSONDecodeError, FileNotFoundError ) as e:
1846 print ( f "Error loading model: { e } " , file = sys.stderr)
1847 sys.exit( 1 )
1848
1849 requirements = None
1850 if args.requirements:
1851 try :
1852 requirements = load_model(args.requirements)
1853 except (json.JSONDecodeError, FileNotFoundError ) as e:
1854 print ( f "Warning: Could not load requirements ( { e } ), using defaults." , file = sys.stderr)
1855
1856 if not model.get( "access_patterns" ):
1857 print ( "Error: No access patterns found in the data model." , file = sys.stderr)
1858 sys.exit( 1 )
1859
1860 # Hard-fail a malformed or impossible vector design BEFORE pricing it. Without this
1861 # gate the failure is silent and expensive: a vector index missing `vector_attribute`,
1862 # or a SearchVectors pattern whose `index` does not resolve, still produces a
1863 # plausible-looking report with vector write capacity priced at $0 — and $0 on the
1864 # line that is usually the largest in the estimate reads as "vectors are cheap"
1865 # rather than "the model is wrong". Observed exactly that: a model using `name`
1866 # instead of `index_name`, omitting `vector_attribute`, and `vector_index` instead of
1867 # `index` costed the whole vector dimension at zero without a word of complaint.
1868 vector_errors = validate_vector_model(
1869 model.get( "tables" ) or [], model.get( "access_patterns" ) or []
1870 )
1871 if vector_errors:
1872 print (
1873 "Error: the vector portion of this data model cannot be priced as written." ,
1874 file = sys.stderr,
1875 )
1876 # Not `e`: this function already binds `e` in two `except ... as e` blocks above,
1877 # and Python deletes that name on leaving the handler.
1878 for problem in vector_errors:
1879 print ( f " - { problem } " , file = sys.stderr)
1880 # Print the correct shape, not just the complaint. Observed recovery behaviour
1881 # when given only an error: the vector index gets DELETED from the model so the
1882 # calculator will run, and the vector cost silently leaves the estimate — a worse
1883 # outcome than the original mistake. A copyable snippet makes fixing the field
1884 # names the path of least resistance.
1885 print (
1886 " \n Fix the model — do NOT delete the vector index to get past this, and do "
1887 "NOT estimate the vector cost by hand. Both drop a real cost that is often "
1888 "the largest line in the estimate. The correct shape is: \n "
1889 ' \n "tables": [{ \n '
1890 ' "table_name": "Products", \n '
1891 ' "vector_indexes": [{ \n '
1892 ' "index_name": "ProductEmbeddingIndex", \n '
1893 ' "vector_attribute": "Embedding", \n '
1894 ' "dimensions": 1024, \n '
1895 ' "distance_function": "COSINE", \n '
1896 ' "projection": { "type": "ALL" }, \n '
1897 ' "search_schema": { "partition_key": "category" } \n '
1898 " }] \n "
1899 " }], \n "
1900 ' "access_patterns": [{ \n '
1901 ' "pattern_id": "FindSimilarProducts", \n '
1902 ' "operation": "SearchVectors", \n '
1903 ' "table": "Products", \n '
1904 ' "index": "ProductEmbeddingIndex", \n '
1905 ' "peak_rps": 60, \n '
1906 ' "top_k": 50 \n '
1907 " }, { \n "
1908 ' "pattern_id": "PutProduct", \n '
1909 ' "operation": "PutItem", \n '
1910 ' "table": "Products", \n '
1911 ' "peak_rps": 200, \n '
1912 ' "attributes_written": ["Embedding"] \n '
1913 " }] \n "
1914 " \n Note `index_name` (not `name`), `index` (not `vector_index`), and that a "
1915 "write storing the embedding lists it in `attributes_written`. Full schema: "
1916 "references/cost-model-schema.md ('Vector index' and 'Access pattern')." ,
1917 file = sys.stderr,
1918 )
1919 sys.exit( 1 )
1920
1921 report = calculate_and_report(model, requirements)
1922
1923 output_path = Path(args.output) if args.output else model_path.parent / "cost_report.md"
1924 output_path.write_text(report)
1925 print ( f "Cost report written to { output_path } " )
1926 print (
1927 f "Analyzed { len (model[ 'access_patterns' ]) } access patterns "
1928 f "across { len (model.get( 'tables' , [])) } tables."
1929 )
1930
1931
1932 if __name__ == "__main__" :
1933 main()