Setting the file. One moment.
Deploy Ic · Hf Cloud Sagemaker Production Defaults · huggingface/skills · Skills Docs
Back to the top of the page
20 KB scripts/ deploy_ic.py
Python · 506 lines · 20 KB
14
--env SM_VLLM_TRUST_REMOTE_CODE=false
15
16 Four pieces make zero work, and all four are required:
17 1. endpoint config: ManagedInstanceScaling with MinInstanceCount=0
18 2. scalable target on inference-component/<name> with MinCapacity=0
19 3. target-tracking policy on concurrency (handles 1..n)
20 4. step-scaling policy + NoCapacityInvocationFailures alarm (handles 0 -> 1)
21
22 Target tracking alone cannot leave zero, so without (4) the endpoint scales to
23 zero once and never wakes.
24
25 Measured on ml.g5.xlarge with Qwen3-0.6B from the Hub (July 2026):
26 endpoint InService 4 min (it starts empty) -> component InService +6 min ->
27 idle scale-in to 0 copies 11 min -> 0 instances +12 min -> wake from zero
28 9 min 24 s from the first rejected request to a 200.
29
30 See SKILL.md, section "Scale to zero for real-time endpoints".
31 """
32
33 import argparse
34 import json
35 import sys
36 import time
37 from typing import Any
38
39 import boto3
40 from botocore.exceptions import ClientError
41
42 from _common import (
43 build_tags,
44 create_model,
45 log as _log,
46 make_endpoint_name,
47 parse_env,
48 wait_for_endpoint,
49 )
50
51
52 # Defaults — change here, not at call sites.
53 DEFAULTS = {
54 "initial_instance_count" : 1 ,
55 "min_instance_count" : 0 ,
56 "max_instance_count" : 2 ,
57 "min_copy_count" : 0 ,
58 "max_copy_count" : 4 ,
59 # Concurrent requests per copy. Target tracking scales out above this.
60 "concurrency_target_per_copy" : 5 ,
61 "scale_in_cooldown_seconds" : 300 ,
62 "scale_out_cooldown_seconds" : 300 ,
63 "wake_cooldown_seconds" : 60 ,
64 "wake_step_size" : 1 ,
65 "model_data_download_timeout" : 3600 ,
66 # 3600 is the API maximum, but a crash-looping container holds the component
67 # in Creating for the whole timeout. 1800 plus the log scan below surfaces a
68 # broken container in about a minute instead.
69 "container_startup_timeout" : 1800 ,
70 # A scheduling reservation, not a cap: the container may use more. Keep it
71 # small. ml.g5.xlarge accepts 1024 and rejects 4096, far below its 16 GiB.
72 "min_memory_required_mb" : 1024 ,
73 "accelerator_devices" : 1 ,
74 "alarm_evaluation_periods" : 1 ,
75 "alarm_period_seconds" : 300 ,
76 "alarm_5xx_threshold_count" : 5 ,
77 "environment_tag" : "dev" ,
78 }
79
80 IC_DIMENSION = "sagemaker:inference-component:DesiredCopyCount"
81 # High-resolution concurrency, not invocations/minute: it reacts up to 6x faster
82 # and its idle zeros are what drive scale-in to zero.
83 CONCURRENCY_METRIC = "SageMakerInferenceComponentConcurrentRequestsPerCopyHighResolution"
84 WAKE_METRIC = "NoCapacityInvocationFailures"
85
86 # supervisord restarts a dying server, so the component status stays Creating.
87 # These markers are the only early evidence of a crash loop.
88 CRASH_MARKERS = (
89 "exited: app" ,
90 "not expected" ,
91 "Worker died" ,
92 "Load model failed" ,
93 "ImportError" ,
94 "Traceback (most recent call last)" ,
95 "api_server.py: error:" ,
96 )
97
98
99 def log (msg: str ) -> None :
100 _log( "deploy_ic" , msg)
101
102
103 def create_endpoint_config (
104 sm: Any, * , config_name: str , instance_type: str , role_arn: str ,
105 initial_instance_count: int , min_instance_count: int , max_instance_count: int ,
106 inference_ami_version: str | None , tags: list[ dict ],
107 ) -> str :
108 log( f "Creating IC endpoint config: { config_name } " )
109
110 variant: dict[ str , Any] = {
111 "VariantName" : "AllTraffic" ,
112 "InstanceType" : instance_type,
113 "InitialInstanceCount" : initial_instance_count,
114 "ModelDataDownloadTimeoutInSeconds" : DEFAULTS [ "model_data_download_timeout" ],
115 "ContainerStartupHealthCheckTimeoutInSeconds" : DEFAULTS [ "container_startup_timeout" ],
116 # MinInstanceCount=0 is what allows the endpoint to drop every instance.
117 "ManagedInstanceScaling" : {
118 "Status" : "ENABLED" ,
119 "MinInstanceCount" : min_instance_count,
120 "MaxInstanceCount" : max_instance_count,
121 },
122 "RoutingConfig" : { "RoutingStrategy" : "LEAST_OUTSTANDING_REQUESTS" },
123 }
124
125 # Same vLLM CUDA 13+ rule as deploy.py, and it coexists with
126 # ManagedInstanceScaling on the same variant (verified).
127 if inference_ami_version:
128 variant[ "InferenceAmiVersion" ] = inference_ami_version
129 log( f " InferenceAmiVersion set to: { inference_ami_version } " )
130
131 # Two differences from a model-based config: no ModelName on the variant
132 # (the component carries the model) and ExecutionRoleArn moves up here.
133 try :
134 sm.create_endpoint_config(
135 EndpointConfigName = config_name,
136 ExecutionRoleArn = role_arn,
137 ProductionVariants = [variant],
138 Tags = tags,
139 )
140 except ClientError as e:
141 if "Cannot create already existing endpoint configuration" in str (e):
142 log( f "Endpoint config { config_name } already exists — reusing" )
143 else :
144 raise
145 return config_name
146
147
148 def create_endpoint (sm: Any, * , endpoint_name: str , config_name: str , tags: list[ dict ]) -> None :
149 log( f "Creating endpoint: { endpoint_name } " )
150 sm.create_endpoint(
151 EndpointName = endpoint_name,
152 EndpointConfigName = config_name,
153 Tags = tags,
154 )
155
156
157 def create_inference_component (
158 sm: Any, * , ic_name: str , endpoint_name: str , model_name: str ,
159 accelerator_devices: int , min_memory_mb: int , tags: list[ dict ],
160 ) -> None :
161 log( f "Creating inference component: { ic_name } " )
162 sm.create_inference_component(
163 InferenceComponentName = ic_name,
164 EndpointName = endpoint_name,
165 VariantName = "AllTraffic" ,
166 Specification = {
167 "ModelName" : model_name,
168 "StartupParameters" : {
169 "ModelDataDownloadTimeoutInSeconds" : DEFAULTS [ "model_data_download_timeout" ],
170 "ContainerStartupHealthCheckTimeoutInSeconds" : DEFAULTS [ "container_startup_timeout" ],
171 },
172 "ComputeResourceRequirements" : {
173 "MinMemoryRequiredInMb" : min_memory_mb,
174 "NumberOfAcceleratorDevicesRequired" : accelerator_devices,
175 },
176 },
177 RuntimeConfig = { "CopyCount" : 1 },
178 Tags = tags,
179 )
180
181
182 def scan_component_logs (ic_name: str , region: str , limit: int = 200 ) -> list[ str ]:
183 """Return crash-marker lines from the component's newest log stream.
184
185 Returns [] when the log group does not exist yet, or when the caller cannot
186 read it. A diagnostic that is unavailable never blocks the deployment.
187 """
188 logs = boto3.client( "logs" , region_name = region)
189 group = f "/aws/sagemaker/InferenceComponents/ { ic_name } "
190 try :
191 streams = logs.describe_log_streams(
192 logGroupName = group, orderBy = "LastEventTime" , descending = True , limit = 1
193 )[ "logStreams" ]
194 except ClientError:
195 return []
196 if not streams:
197 return []
198 try :
199 events = logs.get_log_events(
200 logGroupName = group,
201 logStreamName = streams[ 0 ][ "logStreamName" ],
202 limit = limit,
203 startFromHead = False ,
204 )[ "events" ]
205 except ClientError:
206 return []
207 return [
208 event[ "message" ].strip()[: 500 ]
209 for event in events
210 if any (marker in event[ "message" ] for marker in CRASH_MARKERS )
211 ]
212
213
214 def wait_for_component (sm: Any, ic_name: str , region: str , timeout_minutes: int = 30 ) -> None :
215 """Poll DescribeInferenceComponent until InService.
216
217 Also scans the container log on every poll. A crash loop never changes the
218 component status — it stays Creating until the startup timeout expires — so
219 the log is the only early signal.
220 """
221 log( f "Waiting for inference component { ic_name } (up to { timeout_minutes } min)..." )
222 start = time.time()
223 deadline = start + timeout_minutes * 60
224 polls = 0
225
226 while time.time() < deadline:
227 desc = sm.describe_inference_component( InferenceComponentName = ic_name)
228 status = desc[ "InferenceComponentStatus" ]
229 elapsed = int (time.time() - start)
230
231 if status == "InService" :
232 log( f "Component InService after { elapsed } s" )
233 return
234 if status == "Failed" :
235 reason = desc.get( "FailureReason" , "(no reason given)" )
236 raise RuntimeError ( f "Inference component failed after { elapsed } s: { reason } " )
237
238 polls += 1
239 log( f " status= { status } elapsed= { elapsed } s" )
240 # One grace poll: the first seconds of any boot look noisy.
241 if polls >= 2 :
242 hits = scan_component_logs(ic_name, region)
243 if hits:
244 log( "Container is crash-looping. Log evidence:" )
245 for line in hits[ - 6 :]:
246 log( f " { line } " )
247 raise RuntimeError (
248 "Container does not stay up. Fix the container error above; "
249 "the component status alone would have hidden this for "
250 f " { DEFAULTS [ 'container_startup_timeout' ] } s."
251 )
252 time.sleep( 30 )
253
254 raise TimeoutError ( f "Component did not reach InService within { timeout_minutes } minutes" )
255
256
257 def register_autoscaling (
258 * , ic_name: str , min_copies: int , max_copies: int , concurrency_target: int ,
259 region: str ,
260 ) -> str :
261 """Register the scalable target and both policies. Returns the step ARN.
262
263 Two policies, both required:
264 1. target tracking on concurrency per copy — scales 1..n, and its idle
265 zeros are what take the count down to zero
266 2. step scaling — the only way out of zero, since target tracking cannot
267 divide by zero copies
268 """
269 log( f "Registering component autoscaling: copies= { min_copies } - { max_copies } "
270 f "concurrency-target= { concurrency_target } /copy" )
271 appscaling = boto3.client( "application-autoscaling" , region_name = region)
272 resource_id = f "inference-component/ { ic_name } "
273
274 appscaling.register_scalable_target(
275 ServiceNamespace = "sagemaker" ,
276 ResourceId = resource_id,
277 ScalableDimension = IC_DIMENSION ,
278 MinCapacity = min_copies,
279 MaxCapacity = max_copies,
280 )
281 appscaling.put_scaling_policy(
282 PolicyName = f " { ic_name } -concurrency-target-tracking" ,
283 ServiceNamespace = "sagemaker" ,
284 ResourceId = resource_id,
285 ScalableDimension = IC_DIMENSION ,
286 PolicyType = "TargetTrackingScaling" ,
287 TargetTrackingScalingPolicyConfiguration = {
288 "TargetValue" : float (concurrency_target),
289 "PredefinedMetricSpecification" : { "PredefinedMetricType" : CONCURRENCY_METRIC },
290 "ScaleInCooldown" : DEFAULTS [ "scale_in_cooldown_seconds" ],
291 "ScaleOutCooldown" : DEFAULTS [ "scale_out_cooldown_seconds" ],
292 },
293 )
294 resp = appscaling.put_scaling_policy(
295 PolicyName = f " { ic_name } -step-wake-from-zero" ,
296 ServiceNamespace = "sagemaker" ,
297 ResourceId = resource_id,
298 ScalableDimension = IC_DIMENSION ,
299 PolicyType = "StepScaling" ,
300 StepScalingPolicyConfiguration = {
301 "AdjustmentType" : "ChangeInCapacity" ,
302 "MetricAggregationType" : "Maximum" ,
303 "Cooldown" : DEFAULTS [ "wake_cooldown_seconds" ],
304 "StepAdjustments" : [
305 { "MetricIntervalLowerBound" : 0 , "ScalingAdjustment" : DEFAULTS [ "wake_step_size" ]},
306 ],
307 },
308 )
309 return resp[ "PolicyARN" ]
310
311
312 def create_alarms (
313 * , ic_name: str , endpoint_name: str , step_policy_arn: str ,
314 sns_topic_arn: str | None , region: str ,
315 ) -> None :
316 """Wake alarm (drives the step policy) plus an error alarm."""
317 cw = boto3.client( "cloudwatch" , region_name = region)
318
319 # The wake alarm's action is the step policy, never an SNS topic. Period 30
320 # with one datapoint keeps the wake latency near a minute.
321 log( f "Creating wake alarm: { ic_name } -wake-from-zero" )
322 cw.put_metric_alarm(
323 AlarmName = f " { ic_name } -wake-from-zero" ,
324 AlarmDescription = "Component invoked with zero copies — wake from zero" ,
325 AlarmActions = [step_policy_arn],
326 MetricName = WAKE_METRIC ,
327 Namespace = "AWS/SageMaker" ,
328 Statistic = "Maximum" ,
329 Dimensions = [{ "Name" : "InferenceComponentName" , "Value" : ic_name}],
330 Period = 30 ,
331 EvaluationPeriods = 1 ,
332 DatapointsToAlarm = 1 ,
333 Threshold = 1 ,
334 ComparisonOperator = "GreaterThanOrEqualToThreshold" ,
335 TreatMissingData = "missing" ,
336 )
337
338 alarm_actions = [sns_topic_arn] if sns_topic_arn else []
339 cw.put_metric_alarm(
340 AlarmName = f " { endpoint_name } -Invocation5XXErrors" ,
341 AlarmDescription = "5XX errors > 5 in 5min" ,
342 AlarmActions = alarm_actions,
343 MetricName = "Invocation5XXErrors" ,
344 Namespace = "AWS/SageMaker" ,
345 Statistic = "Sum" ,
346 Dimensions = [
347 { "Name" : "EndpointName" , "Value" : endpoint_name},
348 { "Name" : "VariantName" , "Value" : "AllTraffic" },
349 ],
350 Period = DEFAULTS [ "alarm_period_seconds" ],
351 EvaluationPeriods = DEFAULTS [ "alarm_evaluation_periods" ],
352 Threshold = DEFAULTS [ "alarm_5xx_threshold_count" ],
353 ComparisonOperator = "GreaterThanThreshold" ,
354 TreatMissingData = "notBreaching" ,
355 )
356 if not sns_topic_arn:
357 log( "WARNING: no --sns-alarm-topic — the error alarm won't notify anyone." )
358
359
360 def main () -> int :
361 p = argparse.ArgumentParser( description = __doc__ , formatter_class = argparse.RawDescriptionHelpFormatter)
362
363 # Required
364 p.add_argument( "--model-name" , required = True )
365 p.add_argument( "--image-uri" , required = True , help = "From serving-image-selection" )
366 p.add_argument( "--role-arn" , required = True , help = "From sagemaker-iam-preflight" )
367 p.add_argument( "--instance-type" , required = True , help = "e.g. ml.g5.xlarge" )
368 p.add_argument( "--region" , required = True , help = "From aws-context-discovery" )
369
370 # Conditional
371 p.add_argument( "--model-s3-uri" , default = None ,
372 help = "Pre-staged weights. Strongly recommended here: they are "
373 "re-downloaded on every wake from zero." )
374 p.add_argument( "--env" , action = "append" , default = [], help = "KEY=VALUE; repeatable" )
375 p.add_argument( "--inference-ami-version" , default = None ,
376 help = "REQUIRED for vLLM DLC with CUDA 13+ "
377 "(al2-ami-sagemaker-inference-gpu-3-1)" )
378
379 # Naming
380 p.add_argument( "--endpoint-name" , default = None , help = "Default: <model-name>-<timestamp>" )
381 p.add_argument( "--inference-component-name" , default = None , help = "Default: <model-name>-ic" )
382 p.add_argument( "--project" , default = None , help = "Tag value (default: model name)" )
383 p.add_argument( "--environment" , default = DEFAULTS [ "environment_tag" ])
384
385 # Capacity
386 p.add_argument( "--initial-instance-count" , type = int , default = DEFAULTS [ "initial_instance_count" ])
387 p.add_argument( "--min-instance-count" , type = int , default = DEFAULTS [ "min_instance_count" ],
388 help = "0 enables scale-to-zero. Anything else defeats the point." )
389 p.add_argument( "--max-instance-count" , type = int , default = DEFAULTS [ "max_instance_count" ])
390 p.add_argument( "--min-copy-count" , type = int , default = DEFAULTS [ "min_copy_count" ])
391 p.add_argument( "--max-copy-count" , type = int , default = DEFAULTS [ "max_copy_count" ])
392 p.add_argument( "--concurrency-target-per-copy" , type = int ,
393 default = DEFAULTS [ "concurrency_target_per_copy" ])
394
395 # Component resources
396 p.add_argument( "--accelerator-devices" , type = int , default = DEFAULTS [ "accelerator_devices" ],
397 help = "Must match SM_VLLM_TENSOR_PARALLEL_SIZE for multi-GPU models" )
398 p.add_argument( "--min-memory-mb" , type = int , default = DEFAULTS [ "min_memory_required_mb" ])
399
400 # Alarms
401 p.add_argument( "--sns-alarm-topic" , default = None , help = "SNS topic ARN for the error alarm" )
402 p.add_argument( "--no-alarms" , action = "store_true" ,
403 help = "NOT RECOMMENDED: without the wake alarm the endpoint never leaves zero" )
404 p.add_argument( "--no-autoscaling" , action = "store_true" ,
405 help = "NOT RECOMMENDED: creates the endpoint without any scaling" )
406
407 args = p.parse_args()
408
409 env_dict = parse_env(args.env)
410 endpoint_name = make_endpoint_name(args.model_name, args.endpoint_name)
411 config_name = f " { endpoint_name } -config"
412 ic_name = args.inference_component_name or f " { args.model_name.replace( '_' , '-' ).lower() } -ic" [: 63 ]
413
414 sts = boto3.client( "sts" , region_name = args.region)
415 sm = boto3.client( "sagemaker" , region_name = args.region)
416 identity = sts.get_caller_identity()
417
418 if args.min_instance_count != 0 :
419 log( f "NOTE: --min-instance-count { args.min_instance_count } — this endpoint "
420 "will NOT scale to zero instances." )
421
422 tags = build_tags(
423 project = args.project or args.model_name,
424 caller_arn = identity[ "Arn" ],
425 environment = args.environment,
426 model_s3_uri = args.model_s3_uri,
427 extra = { "InferenceMode" : "realtime-ic-scale-to-zero" },
428 )
429
430 create_model(
431 sm, model_name = args.model_name, image_uri = args.image_uri,
432 role_arn = args.role_arn, model_s3_uri = args.model_s3_uri,
433 env = env_dict, tags = tags, log_prefix = "deploy_ic" ,
434 )
435 create_endpoint_config(
436 sm, config_name = config_name, instance_type = args.instance_type,
437 role_arn = args.role_arn, initial_instance_count = args.initial_instance_count,
438 min_instance_count = args.min_instance_count,
439 max_instance_count = args.max_instance_count,
440 inference_ami_version = args.inference_ami_version, tags = tags,
441 )
442 create_endpoint(sm, endpoint_name = endpoint_name, config_name = config_name, tags = tags)
443 # An IC endpoint starts empty, so this is fast (~4 min): no model loads yet.
444 wait_for_endpoint(sm, endpoint_name, log_prefix = "deploy_ic" )
445
446 create_inference_component(
447 sm, ic_name = ic_name, endpoint_name = endpoint_name, model_name = args.model_name,
448 accelerator_devices = args.accelerator_devices, min_memory_mb = args.min_memory_mb,
449 tags = tags,
450 )
451 wait_for_component(sm, ic_name, args.region)
452
453 step_policy_arn = None
454 if not args.no_autoscaling:
455 step_policy_arn = register_autoscaling(
456 ic_name = ic_name, min_copies = args.min_copy_count,
457 max_copies = args.max_copy_count,
458 concurrency_target = args.concurrency_target_per_copy, region = args.region,
459 )
460 else :
461 log( "WARNING: autoscaling skipped. This endpoint will not scale at all." )
462
463 if not args.no_alarms and step_policy_arn:
464 create_alarms(
465 ic_name = ic_name, endpoint_name = endpoint_name,
466 step_policy_arn = step_policy_arn, sns_topic_arn = args.sns_alarm_topic,
467 region = args.region,
468 )
469 elif not args.no_alarms:
470 log( "WARNING: no step policy, so no wake alarm. Nothing wakes this endpoint." )
471
472 # Summary
473 log( "" )
474 log( f "Deployment complete: { endpoint_name } " )
475 log( f " Component: { ic_name } " )
476 log( f " Instance: { args.instance_type } "
477 f "( { args.min_instance_count } - { args.max_instance_count } )" )
478 log( f " Copies: { args.min_copy_count } - { args.max_copy_count } " )
479 log( f " Scale to zero: { 'YES' if args.min_instance_count == 0 and args.min_copy_count == 0 else 'NO' } " )
480 log( "" )
481 log( "Idle behaviour: copies reach 0 after ~11 min without traffic, instances" )
482 log( "~12 min later. The first request after that returns a 400 and wakes the" )
483 log( "endpoint; a 200 follows about 9 min later (Hub download included)." )
484 log( "" )
485 log( f "Test: python3 invoke_endpoint.py --endpoint-name { endpoint_name } \\ " )
486 log( f " --inference-component-name { ic_name } \\ " )
487 log( " --payload '{ \" prompt \" : \" hello \" , \" max_tokens \" : 16}' \\ " )
488 log( f " --wait-for-capacity 900 --region { args.region } " )
489 log( f "Teardown: python3 teardown.py { endpoint_name } { args.region } " )
490
491 print (json.dumps({
492 "endpoint_name" : endpoint_name,
493 "endpoint_config_name" : config_name,
494 "model_name" : args.model_name,
495 "inference_component_name" : ic_name,
496 "step_policy_arn" : step_policy_arn,
497 "region" : args.region,
498 "instance_type" : args.instance_type,
499 "scales_to_zero" : args.min_instance_count == 0 and args.min_copy_count == 0 ,
500 }))
501
502 return 0
503
504
505 if __name__ == "__main__" :
506 sys.exit(main())