Setting the file. One moment.
Serverless Estimator · Amazon Elasticache · aws/agent-toolkit-for-aws · Skills Docs
ContentsBack to the top of the page 10
Setup DevOps Agent
33
AWS Deployment
61.5
Recipe Gallery
74 of 79 scripts/ serverless_estimator.py
Python · 668 lines · 28 KB
16 This is a quick approximation for initial cost comparison.
17
18 Detailed: Also provide per-command stats (from INFO commandstats).
19 Classifies commands as simple (1 ECPU) or complex
20 (estimated as MAX(calls, usec/3) ECPUs based on the
21 assumption that 1 ECPU corresponds to approximately
22 3 microseconds of vCPU time, derived from the AWS pricing
23 blog example). Takes the higher of the CPU and network
24 components. More accurate than simple mode but still an
25 approximation.
26
27 IMPORTANT: The ECPU formulas used in this tool are heuristic approximations
28 based on publicly available AWS documentation and pricing examples. AWS does
29 not publish exact ECPU calculation formulas. The 3-microsecond-per-ECPU
30 assumption for complex commands is derived from the AWS pricing blog and may
31 not apply uniformly to all command types. See command_classifier.py for
32 details on command classification.
33
34 Pricing is fetched live from the AWS Bulk Pricing API and reflects current
35 published rates. See https://aws.amazon.com/elasticache/pricing/
36
37 Note: INFO commandstats returns cumulative counts that keep incrementing
38 from server start. If you include an "uptime_days" column in the
39 commandstats CSV, the estimator divides calls and usec by that value to
40 normalize to a daily rate. The --endpoint mode handles this automatically
41 by taking two snapshots and computing the delta.
42
43 Usage:
44 python3 serverless_estimator.py --input clusters.csv
45 python3 serverless_estimator.py --input clusters.csv --commandstats stats.csv
46 python3 serverless_estimator.py --endpoint my-cluster.abc123.use1.cache.amazonaws.com:6379
47 python3 serverless_estimator.py --endpoint my-cluster.abc123.use1.cache.amazonaws.com --instance-type cache.r7g.large
48 python3 serverless_estimator.py --input clusters.csv --pricing pricing.csv
49 python3 serverless_estimator.py --input clusters.csv --output estimate.csv --json
50 """
51 import argparse
52 import csv
53 import json
54 import math
55 import os
56 import re
57 import sys
58 from typing import Dict, List, Optional
59
60 sys.path.insert( 0 , os.path.dirname(os.path.abspath( __file__ )))
61
62 from command_classifier import estimate_ecpus_from_commandstats
63 from pricing import PricingLoader
64
65 HOURS_PER_MONTH = 730
66
67 MIN_STORAGE_GB = {
68 "valkey" : 0.1 , # 100 MB
69 "redis" : 1.0 , # 1 GB
70 "memcached" : 1.0 , # 1 GB
71 }
72
73 # Number of sample points for sinusoidal workload modeling.
74 # 24 = one sample per hour over a daily cycle.
75 _SIN_SAMPLES = 24
76
77
78 def _sine_wave_samples (min_val, max_val, n = _SIN_SAMPLES ):
79 """Generate n hourly samples of a sine wave oscillating between min and max.
80
81 Models a daily traffic pattern:
82 value(t) = min + (max - min) * (1 + sin(2*pi*t/n)) / 2
83
84 Returns a list of n float values representing the workload at each hour.
85 """
86 amplitude = (max_val - min_val) / 2.0
87 midpoint = (max_val + min_val) / 2.0
88 return [midpoint + amplitude * math.sin( 2 * math.pi * i / n) for i in range (n)]
89
90
91 def compute_burst_multiplier (avg_val, min_val = None , max_val = None , peak_to_avg = None ):
92 """Compute a cost multiplier for bursty (sinusoidal) workloads.
93
94 Serverless bills per-hour. A workload that swings between min and peak
95 follows a sine curve over the day. This function samples the curve at
96 24 hourly points and returns the ratio of the sine-wave average to the
97 customer-provided average.
98
99 Why this matters: if the customer says "my average is X" but their actual
100 traffic swings from min to peak, the true average of the sine curve
101 ((min+max)/2) may differ from X. The multiplier corrects for that.
102
103 The caller can specify the burst in two ways:
104 1. min_val + max_val: explicit floor and ceiling of the daily cycle
105 2. peak_to_avg: ratio of peak to average (e.g., 3.0 means peak is 3x avg)
106
107 Returns 1.0 (no adjustment) if no burst parameters are provided or if
108 the workload is flat.
109 """
110 if peak_to_avg is not None and peak_to_avg > 1.0 :
111 max_val = avg_val * peak_to_avg
112 min_val = max ( 0 , avg_val * ( 2 - peak_to_avg))
113
114 if min_val is None or max_val is None :
115 return 1.0
116 if max_val <= min_val or max_val <= 0 :
117 return 1.0
118 if avg_val <= 0 :
119 return 1.0
120
121 samples = _sine_wave_samples(min_val, max_val)
122 sine_avg = sum (samples) / len (samples)
123
124 return sine_avg / avg_val
125
126
127 def load_clusters (path: str ) -> List[Dict]:
128 """Load cluster data from CSV.
129
130 Required columns:
131 cluster_name, instance_type, region, engine, node_count,
132 avg_memory_gb, daily_commands
133
134 Optional columns:
135 primary_nodes -- defaults to max(1, node_count // 2). This is a
136 rough heuristic; for CMD clusters (1 primary +
137 N replicas) or CME clusters (1 primary per shard),
138 provide the actual value via --primary-nodes or
139 the primary_nodes CSV column for accurate results.
140 current_monthly_cost -- if known; otherwise computed from list price
141 avg_payload_kb -- average payload size in KB (default 1.0).
142 ECPUs scale linearly with payload: 3.2 KB = 3.2 ECPUs per op.
143 peak_commands -- peak daily commands (for bursty workload modeling)
144 min_commands -- minimum daily commands (for bursty workload modeling)
145 peak_to_avg_ratio -- ratio of peak to average commands (alternative to min/max)
146 peak_memory_gb -- peak memory in GB (for bursty storage modeling)
147 min_memory_gb -- minimum memory in GB (for bursty storage modeling)
148 """
149 clusters = []
150 with open (path, newline = "" ) as f:
151 reader = csv.DictReader(f)
152 for row in reader:
153 node_count = int (row.get( "node_count" , 1 ))
154 primary_nodes = int (row.get( "primary_nodes" , 0 ))
155 if primary_nodes == 0 :
156 # Heuristic: assumes half the nodes are primaries. This is only
157 # accurate for 1-replica setups; actual primary count depends on
158 # cluster topology (CMD: 1 primary + N replicas, CME: 1 primary
159 # per shard). Provide the primary_nodes column for accuracy.
160 primary_nodes = max ( 1 , node_count // 2 )
161
162 clusters.append({
163 "cluster_name" : row[ "cluster_name" ].strip(),
164 "instance_type" : row.get( "instance_type" , "" ).strip(),
165 "region" : row.get( "region" , "us-east-1" ).strip(),
166 "engine" : row.get( "engine" , "valkey" ).strip().lower(),
167 "node_count" : node_count,
168 "primary_nodes" : primary_nodes,
169 "avg_memory_gb" : float (row.get( "avg_memory_gb" , 0 )),
170 "daily_commands" : float (row.get( "daily_commands" , 0 )),
171 "current_monthly_cost" : float (row.get( "current_monthly_cost" , 0 ) or 0 ),
172 "avg_payload_kb" : float (row.get( "avg_payload_kb" , 1.0 ) or 1.0 ),
173 "peak_commands" : float (row.get( "peak_commands" , 0 ) or 0 ),
174 "min_commands" : float (row.get( "min_commands" , 0 ) or 0 ),
175 "peak_to_avg_ratio" : float (row.get( "peak_to_avg_ratio" , 0 ) or 0 ),
176 "peak_memory_gb" : float (row.get( "peak_memory_gb" , 0 ) or 0 ),
177 "min_memory_gb" : float (row.get( "min_memory_gb" , 0 ) or 0 ),
178 })
179 return clusters
180
181
182 def parse_commandstats_file (path: str ) -> Dict[ str , dict ]:
183 """Parse a commandstats CSV.
184
185 Expected columns: cluster_name, command, calls, usec
186
187 Optional columns:
188 uptime_days - days since server restart. When present, calls and
189 usec are divided by this value to normalize cumulative
190 INFO commandstats output to a daily rate. Without it
191 the raw cumulative values are used as-is.
192
193 Returns a dict keyed by cluster_name. Each value is a dict of
194 {command: {"calls": int, "usec": int}}. A special key
195 "_normalized" (bool) indicates whether uptime_days normalization
196 was applied for that cluster.
197 """
198 result = {}
199 with open (path, newline = "" ) as f:
200 for row in csv.DictReader(f):
201 cluster = row[ "cluster_name" ].strip()
202 cmd = row[ "command" ].strip().lower()
203 calls = int (row.get( "calls" , 0 ))
204 usec = int (row.get( "usec" , 0 ))
205
206 uptime_raw = row.get( "uptime_days" , "" ).strip()
207 uptime_days = float (uptime_raw) if uptime_raw else None
208
209 if uptime_days and uptime_days > 0 :
210 calls = int (calls / uptime_days)
211 usec = int (usec / uptime_days)
212 normalized = True
213 else :
214 normalized = False
215
216 if cluster not in result:
217 result[cluster] = { "_normalized" : normalized} # type: ignore[dict-item]
218 result[cluster][cmd] = { "calls" : calls, "usec" : usec} # type: ignore[assignment]
219 # If any row for this cluster has uptime_days, mark as normalized
220 if normalized:
221 result[cluster][ "_normalized" ] = True
222 return result
223
224
225 def parse_commandstats_info (text: str ) -> dict :
226 """Parse raw INFO commandstats output from valkey-cli (or redis-cli).
227
228 Example line: cmdstat_get:calls=1000,usec=1500,usec_per_call=1.50,rejected_calls=0,failed_calls=0
229 """
230 result = {}
231 for line in text.strip().splitlines():
232 line = line.strip()
233 if not line or line.startswith( "#" ):
234 continue
235 match = re.match( r "cmdstat_ (\w + ) :calls= (\d + ) ,usec= (\d + ) " , line)
236 if match:
237 cmd = match.group( 1 ).lower()
238 result[cmd] = {
239 "calls" : int (match.group( 2 )),
240 "usec" : int (match.group( 3 )),
241 }
242 return result
243
244
245 def estimate_cluster (
246 cluster: Dict,
247 pricing: PricingLoader,
248 commandstats: Optional[ dict ] = None ,
249 ) -> Dict:
250 """Estimate serverless cost for one cluster."""
251 region = cluster[ "region" ]
252 engine = cluster[ "engine" ]
253 node_count = cluster[ "node_count" ]
254 instance_type = cluster[ "instance_type" ]
255
256 # --- Current provisioned cost ---
257 if cluster[ "current_monthly_cost" ] > 0 :
258 current_cost = cluster[ "current_monthly_cost" ]
259 cost_source = "provided"
260 elif instance_type:
261 try :
262 hourly = pricing.get_node_hourly_rate(region, instance_type, engine)
263 current_cost = hourly * HOURS_PER_MONTH * node_count
264 cost_source = "list_price"
265 except ValueError :
266 current_cost = 0
267 cost_source = "unknown"
268 else :
269 current_cost = 0
270 cost_source = "unknown"
271
272 # --- Serverless storage ---
273 avg_memory_gb = cluster[ "avg_memory_gb" ]
274 min_gb = MIN_STORAGE_GB .get(engine, 1.0 )
275 billing_memory_gb = max (avg_memory_gb, min_gb)
276 used_minimum = avg_memory_gb < min_gb
277
278 # Bursty storage: if peak/min memory provided, adjust
279 storage_burst = compute_burst_multiplier(
280 avg_memory_gb,
281 min_val = cluster.get( "min_memory_gb" ) or None ,
282 max_val = cluster.get( "peak_memory_gb" ) or None ,
283 )
284 adjusted_memory_gb = billing_memory_gb * storage_burst
285
286 monthly_gb_hours = adjusted_memory_gb * HOURS_PER_MONTH
287 storage_rate = pricing.get_serverless_storage_rate(region, engine)
288 storage_cost = monthly_gb_hours * storage_rate
289
290 # --- Serverless ECPUs ---
291 ecpu_rate = pricing.get_serverless_ecpu_rate(region, engine)
292
293 # Bursty commands: if peak/min commands or peak_to_avg_ratio provided, adjust
294 ecpu_burst = compute_burst_multiplier(
295 cluster[ "daily_commands" ],
296 min_val = cluster.get( "min_commands" ) or None ,
297 max_val = cluster.get( "peak_commands" ) or None ,
298 peak_to_avg = cluster.get( "peak_to_avg_ratio" ) or None ,
299 )
300
301 if commandstats and engine == "memcached" :
302 print ( "WARNING: commandstats mode uses Redis/Valkey INFO format, which is "
303 "incompatible with Memcached. ECPU estimates for Memcached cluster ' {} ' "
304 "will be inaccurate. Falling back to simple estimation." .format(
305 cluster[ "cluster_name" ]),
306 file = sys.stderr)
307 commandstats = None
308
309 if commandstats:
310 # _normalized is a metadata flag, not a command; exclude before estimation
311 cs_normalized = commandstats.get( "_normalized" , False )
312 cs_data = {k: v for k, v in commandstats.items() if k != "_normalized" }
313 ecpu_result = estimate_ecpus_from_commandstats(cs_data)
314 daily_cpu_ecpus = ecpu_result[ "total_ecpus" ]
315 # Network component: each command costs MAX(1, avg_payload_kb) ECPUs.
316 # This approximates the "1 ECPU per KB transferred" pricing rule.
317 # See: https://aws.amazon.com/elasticache/pricing/
318 avg_payload_kb = cluster.get( "avg_payload_kb" , 1.0 ) or 1.0
319 ecpu_per_request = max ( 1.0 , avg_payload_kb)
320 daily_net_ecpus = cluster[ "daily_commands" ] * ecpu_per_request
321 # Serverless charges the higher of vCPU time or data transferred.
322 # See: https://aws.amazon.com/blogs/database/unlock-on-demand-cost-optimized-performance-with-amazon-elasticache-serverless/
323 daily_ecpus = max (daily_cpu_ecpus, daily_net_ecpus)
324 monthly_ecpus = daily_ecpus * 30 * ecpu_burst
325 ecpu_mode = "detailed"
326 fixed_ecpus = ecpu_result[ "fixed_ecpus" ] * 30
327 nonfixed_ecpus = ecpu_result[ "nonfixed_ecpus" ] * 30
328 internal_calls = ecpu_result[ "internal_calls" ] * 30
329 else :
330 avg_payload_kb = cluster.get( "avg_payload_kb" , 1.0 ) or 1.0
331 ecpu_per_request = max ( 1.0 , avg_payload_kb)
332 monthly_ecpus = cluster[ "daily_commands" ] * 30 * ecpu_per_request * ecpu_burst
333 ecpu_mode = "simple"
334 fixed_ecpus = None
335 nonfixed_ecpus = None
336 internal_calls = None
337
338 ecpu_cost = monthly_ecpus * ecpu_rate
339 total_serverless = storage_cost + ecpu_cost
340
341 # --- Comparison ---
342 diff = total_serverless - current_cost if current_cost > 0 else None
343 diff_pct = (diff / current_cost * 100 ) if diff is not None and current_cost > 0 else None
344
345 # --- Notes ---
346 notes = []
347 if used_minimum:
348 notes.append( "Min storage applied ( {} GB)" .format(min_gb))
349 if cost_source == "list_price" :
350 notes.append( "Current cost: on-demand list price" )
351 if cost_source == "unknown" :
352 notes.append( "Current cost unknown - provide current_monthly_cost or instance_type" )
353 if ecpu_mode == "simple" :
354 if avg_payload_kb > 1 :
355 notes.append( "ECPU: {} ECPUs per request ( {} KB avg payload)" .format(
356 round (ecpu_per_request, 1 ), round (avg_payload_kb, 1 )))
357 else :
358 notes.append( "ECPU: 1 cmd = 1 ECPU (provide avg_payload_kb for better accuracy)" )
359 if commandstats and not cs_normalized:
360 notes.append( "Commandstats not normalized by uptime; provide uptime_days column for accuracy" )
361 if ecpu_burst > 1.0 :
362 notes.append( "Burst adjustment applied: {:.2f} x (peak/avg commands)" .format(ecpu_burst))
363 if storage_burst > 1.0 :
364 notes.append( "Burst adjustment applied: {:.2f} x (peak/avg memory)" .format(storage_burst))
365
366 # Serverless compatibility warnings
367 if engine in ( "redis" , "valkey" ):
368 notes.append( "Serverless requires cluster-mode-enabled clients and TLS; Global Data Store and Data Tiering are not supported" )
369
370 return {
371 "cluster_name" : cluster[ "cluster_name" ],
372 "region" : region,
373 "engine" : engine,
374 "instance_type" : instance_type,
375 "node_count" : node_count,
376 "primary_nodes" : cluster[ "primary_nodes" ],
377 "current_monthly_cost" : round (current_cost, 2 ),
378 "cost_source" : cost_source,
379 "avg_memory_gb" : round (avg_memory_gb, 4 ),
380 "billing_memory_gb" : round (billing_memory_gb, 4 ),
381 "monthly_gb_hours" : round (monthly_gb_hours, 2 ),
382 "storage_rate_per_gb_hour" : storage_rate,
383 "storage_cost" : round (storage_cost, 2 ),
384 "monthly_ecpus" : round (monthly_ecpus),
385 "ecpu_rate_per_million" : round (ecpu_rate * 1_000_000 , 4 ),
386 "ecpu_cost" : round (ecpu_cost, 2 ),
387 "ecpu_mode" : ecpu_mode,
388 "fixed_ecpus" : round (fixed_ecpus) if fixed_ecpus is not None else None ,
389 "nonfixed_ecpus" : round (nonfixed_ecpus) if nonfixed_ecpus is not None else None ,
390 "internal_calls_excluded" : round (internal_calls) if internal_calls is not None else None ,
391 "serverless_total" : round (total_serverless, 2 ),
392 "diff" : round (diff, 2 ) if diff is not None else None ,
393 "diff_pct" : round (diff_pct, 1 ) if diff_pct is not None else None ,
394 "notes" : "; " .join(notes),
395 }
396
397
398 def write_csv (results: List[Dict], path: str ):
399 """Write results to CSV."""
400 if not results:
401 return
402 fields = [
403 "cluster_name" , "region" , "engine" , "instance_type" ,
404 "node_count" , "primary_nodes" ,
405 "current_monthly_cost" , "cost_source" ,
406 "avg_memory_gb" , "billing_memory_gb" ,
407 "monthly_gb_hours" , "storage_cost" ,
408 "monthly_ecpus" , "ecpu_cost" , "ecpu_mode" ,
409 "serverless_total" , "diff" , "diff_pct" , "notes" ,
410 ]
411 with open (path, "w" , newline = "" ) as f:
412 writer = csv.DictWriter(f, fieldnames = fields, extrasaction = "ignore" )
413 writer.writeheader()
414 writer.writerows(results)
415 print ( "Output written to: {} " .format(path))
416
417
418 def print_summary (results: List[Dict]):
419 """Print summary to console."""
420 total_current = sum (r[ "current_monthly_cost" ] for r in results)
421 total_sl = sum (r[ "serverless_total" ] for r in results)
422 total_stor = sum (r[ "storage_cost" ] for r in results)
423 total_ecpu = sum (r[ "ecpu_cost" ] for r in results)
424 cheaper = sum ( 1 for r in results if r[ "diff" ] is not None and r[ "diff" ] < 0 )
425 more_exp = sum ( 1 for r in results if r[ "diff" ] is not None and r[ "diff" ] > 0 )
426
427 print ()
428 print ( "=" * 65 )
429 print ( " ElastiCache Serverless Cost Estimate" )
430 print ( "=" * 65 )
431 print ( " Clusters analyzed: {} " .format( len (results)))
432 print ( " Total nodes: {} " .format( sum (r[ "node_count" ] for r in results)))
433 print ()
434 print ( " Current provisioned cost: $ {:>12,.2f} /month" .format(total_current))
435 print ( " Estimated serverless: $ {:>12,.2f} /month" .format(total_sl))
436 print ( " Data storage: $ {:>12,.2f} " .format(total_stor))
437 print ( " ECPUs: $ {:>12,.2f} " .format(total_ecpu))
438 print ()
439 if total_current > 0 :
440 savings = total_current - total_sl
441 pct = savings / total_current * 100
442 print ( " Estimated savings: $ {:>12,.2f} ( {:.1f} %)" .format(savings, pct))
443 print ( " Clusters cheaper on SL: {} " .format(cheaper))
444 print ( " Clusters more expensive: {} " .format(more_exp))
445 print ( "=" * 65 )
446 print ()
447 print ( " {:<30s} {:>10s} {:>10s} {:>8s} " .format(
448 "Cluster" , "Current" , "Serverless" , "Change" ))
449 print ( " {} {} {} {} " .format( "-" * 30 , "-" * 10 , "-" * 10 , "-" * 8 ))
450 for r in sorted (results, key =lambda x: x.get( "diff" ) or 0 ):
451 name = r[ "cluster_name" ][: 30 ]
452 cur = "$ {:,.0f} " .format(r[ "current_monthly_cost" ])
453 sl = "$ {:,.0f} " .format(r[ "serverless_total" ])
454 chg = " {:+.0f} %" .format(r[ "diff_pct" ]) if r[ "diff_pct" ] is not None else "N/A"
455 print ( " {:<30s} {:>10s} {:>10s} {:>8s} " .format(name, cur, sl, chg))
456 print ()
457 print ( " ECPU estimates assume 1 KB avg payload unless avg_payload_kb is provided." )
458 print ( " ECPUs scale linearly with payload size (e.g. 3 KB = 3x ECPUs)." )
459 print ()
460 print ( " IMPORTANT: These are approximate estimates. Actual serverless costs depend" )
461 print ( " on real-time workload characteristics and may differ. For accurate billing," )
462 print ( " use CloudWatch metrics on a running serverless cache." )
463 print ( " See https://aws.amazon.com/elasticache/pricing/" )
464 print ()
465
466
467 def collect_from_endpoint (endpoint, cluster_name = None , instance_type = "" ,
468 region = "us-east-1" , engine = "valkey" , node_count = 2 ,
469 avg_payload_kb = 1.0 , use_tls = True , sample_seconds = 60 ):
470 """Connect to a live Valkey/Redis endpoint and collect metrics.
471
472 Takes two INFO snapshots separated by sample_seconds and computes
473 deltas. This gives the actual traffic rate during the sample window,
474 which is more accurate than dividing cumulative totals by uptime.
475
476 Returns (cluster_dict, commandstats_dict) ready for estimate_cluster().
477 """
478 try :
479 import valkey as client_lib
480 except ImportError :
481 print ( "Error: the 'valkey' package is required for --endpoint mode." )
482 print ( "Install with: pip install valkey" )
483 sys.exit( 1 )
484
485 import time
486
487 # Parse host:port
488 if ":" in endpoint and not endpoint.startswith( "[" ):
489 host, port_str = endpoint.rsplit( ":" , 1 )
490 port = int (port_str)
491 else :
492 host = endpoint
493 port = 6379
494
495 if not cluster_name:
496 cluster_name = host.split( "." )[ 0 ]
497
498 print ( "Connecting to {} : {}{} ..." .format(host, port, " (TLS)" if use_tls else "" ))
499 r = client_lib.Redis( host = host, port = port, ssl = use_tls,
500 ssl_cert_reqs = None , decode_responses = True ,
501 socket_connect_timeout = 10 )
502 print ( " PING: {} " .format(r.ping()))
503
504 # Collect memory and replication (point-in-time, no delta needed)
505 mem_info = r.info( "memory" )
506 repl_info = r.info( "replication" )
507 dataset_bytes = mem_info.get( "used_memory_dataset" , mem_info.get( "used_memory" , 0 ))
508 dataset_gb = dataset_bytes / ( 1024 ** 3 )
509 role = repl_info.get( "role" , "unknown" )
510
511 print ( " Role: {} " .format(role))
512 print ( " Memory: {:.4f} GB ( {:,.0f} bytes)" .format(dataset_gb, dataset_bytes))
513
514 # Snapshot 1
515 print ( " Taking snapshot 1..." )
516 cs1 = r.info( "commandstats" )
517 stats1 = r.info( "stats" )
518 t1 = time.time()
519
520 # Wait
521 print ( " Sampling for {} seconds..." .format(sample_seconds))
522 time.sleep(sample_seconds)
523
524 # Snapshot 2
525 print ( " Taking snapshot 2..." )
526 cs2 = r.info( "commandstats" )
527 stats2 = r.info( "stats" )
528 t2 = time.time()
529
530 elapsed = t2 - t1
531 elapsed_days = elapsed / 86400
532
533 # Compute commandstats deltas
534 commandstats = {}
535 total_delta_calls = 0
536 for key in cs2:
537 cmd = key.replace( "cmdstat_" , "" )
538 calls_delta = cs2[key].get( "calls" , 0 ) - cs1.get(key, {}).get( "calls" , 0 )
539 usec_delta = cs2[key].get( "usec" , 0 ) - cs1.get(key, {}).get( "usec" , 0 )
540 if calls_delta > 0 :
541 # Scale to daily rate
542 daily_calls = int (calls_delta / elapsed_days)
543 daily_usec = int (usec_delta / elapsed_days)
544 commandstats[cmd] = { "calls" : daily_calls, "usec" : daily_usec}
545 total_delta_calls += calls_delta
546
547 commandstats[ "_normalized" ] = True # type: ignore[assignment]
548 daily_commands = total_delta_calls / elapsed_days if elapsed_days > 0 else 0
549
550 # Also get total commands delta for simple mode
551 total_cmds_delta = stats2.get( "total_commands_processed" , 0 ) - stats1.get( "total_commands_processed" , 0 )
552 daily_commands_total = total_cmds_delta / elapsed_days if elapsed_days > 0 else 0
553
554 print ( " Sample window: {:.0f} seconds" .format(elapsed))
555 print ( " Commands in window: {:,} " .format(total_cmds_delta))
556 print ( " Daily rate: {:,.0f} commands/day" .format(daily_commands_total))
557
558 top_cmds = sorted (commandstats.items(),
559 key =lambda x: x[ 1 ].get( "calls" , 0 ) if isinstance (x[ 1 ], dict ) else 0 ,
560 reverse = True )[: 5 ]
561 print ( " Top commands: {} " .format(
562 ", " .join( " {} ( {:,} /day)" .format(k, v[ "calls" ])
563 for k, v in top_cmds if isinstance (v, dict ))))
564
565 cluster = {
566 "cluster_name" : cluster_name,
567 "instance_type" : instance_type,
568 "region" : region,
569 "engine" : engine.lower(),
570 "node_count" : node_count,
571 "primary_nodes" : max ( 1 , node_count // 2 ),
572 "avg_memory_gb" : dataset_gb,
573 "daily_commands" : daily_commands_total,
574 "current_monthly_cost" : 0 ,
575 "avg_payload_kb" : avg_payload_kb,
576 "peak_commands" : 0 ,
577 "min_commands" : 0 ,
578 "peak_to_avg_ratio" : 0 ,
579 "peak_memory_gb" : 0 ,
580 "min_memory_gb" : 0 ,
581 }
582
583 return cluster, commandstats
584
585
586 def main ():
587 parser = argparse.ArgumentParser(
588 description = "Estimate ElastiCache Serverless costs from provisioned cluster metrics"
589 )
590 parser.add_argument( "--input" , "-i" ,
591 help = "CSV with cluster data (see README for format)" )
592 parser.add_argument( "--endpoint" , "-e" ,
593 help = "Connect directly to a Valkey/Redis endpoint (host:port or host)" )
594 parser.add_argument( "--cluster-name" , default = None ,
595 help = "Cluster name (used with --endpoint, default: derived from host)" )
596 parser.add_argument( "--instance-type" , default = "" ,
597 help = "Instance type (used with --endpoint, e.g., cache.r7g.large)" )
598 parser.add_argument( "--region" , default = "us-east-1" ,
599 help = "AWS region (used with --endpoint, default: us-east-1)" )
600 parser.add_argument( "--engine" , default = "valkey" ,
601 help = "Engine (used with --endpoint, default: valkey)" )
602 parser.add_argument( "--node-count" , type = int , default = 2 ,
603 help = "Node count (used with --endpoint, default: 2)" )
604 parser.add_argument( "--avg-payload-kb" , type = float , default = 1.0 ,
605 help = "Average payload size in KB (default: 1.0)" )
606 parser.add_argument( "--no-tls" , action = "store_true" ,
607 help = "Disable TLS when connecting via --endpoint" )
608 parser.add_argument( "--sample-seconds" , type = int , default = 60 ,
609 help = "Seconds to sample when using --endpoint (default: 60). "
610 "Takes two snapshots this far apart and computes deltas." )
611 parser.add_argument( "--commandstats" , "-c" ,
612 help = "CSV with per-command stats for detailed ECPU estimation" )
613 parser.add_argument( "--pricing" , "-p" ,
614 help = "Pricing CSV (optional - fetches live from AWS if omitted)" )
615 parser.add_argument( "--output" , "-o" , default = "serverless_estimate.csv" ,
616 help = "Output CSV path (default: serverless_estimate.csv)" )
617 parser.add_argument( "--json" , action = "store_true" ,
618 help = "Also write JSON output" )
619 args = parser.parse_args()
620
621 if not args.input and not args.endpoint:
622 parser.error( "Either --input (CSV) or --endpoint (host:port) is required" )
623
624 pricing = PricingLoader(args.pricing)
625
626 if args.endpoint:
627 # Direct endpoint mode: connect, collect, normalize, estimate
628 cluster, commandstats = collect_from_endpoint(
629 args.endpoint,
630 cluster_name = args.cluster_name,
631 instance_type = args.instance_type,
632 region = args.region,
633 engine = args.engine,
634 node_count = args.node_count,
635 avg_payload_kb = args.avg_payload_kb,
636 use_tls =not args.no_tls,
637 sample_seconds = args.sample_seconds,
638 )
639 results = [estimate_cluster(cluster, pricing, commandstats)]
640 else :
641 print ( "Loading clusters from: {} " .format(args.input))
642 clusters = load_clusters(args.input)
643 print ( " {} clusters loaded" .format( len (clusters)))
644
645 all_commandstats = None
646 if args.commandstats:
647 print ( "Loading commandstats from: {} " .format(args.commandstats))
648 all_commandstats = parse_commandstats_file(args.commandstats)
649 print ( " Stats for {} clusters" .format( len (all_commandstats)))
650
651 results = []
652 for cluster in clusters:
653 cs = all_commandstats.get(cluster[ "cluster_name" ]) if all_commandstats else None
654 results.append(estimate_cluster(cluster, pricing, cs))
655
656 write_csv(results, args.output)
657 print_summary(results)
658
659 if args.json:
660 base, _ = os.path.splitext(args.output)
661 json_path = base + ".json"
662 with open (json_path, "w" ) as f:
663 json.dump(results, f, indent = 2 , default = str )
664 print ( "JSON written to: {} " .format(json_path))
665
666
667 if __name__ == "__main__" :
668 main()