Setting the file. One moment.
Deploy · Hf Cloud Sagemaker Production Defaults · huggingface/skills · Skills Docs
Back to the top of the page Next
Script Invoke Endpoint
scripts/ deploy.py
Python · 327 lines · 13 KB
import
time
16 from typing import Any
17
18 import boto3
19 from botocore.exceptions import ClientError
20
21 from _common import (
22 build_tags,
23 create_model,
24 log as _log,
25 make_endpoint_name,
26 parse_env,
27 wait_for_endpoint,
28 )
29
30
31 # Defaults — change here, not at call sites.
32 # Reasoning lives in references/deployment-template.md.
33 DEFAULTS = {
34 "initial_instance_count" : 1 ,
35 "min_capacity" : 1 ,
36 "max_capacity" : 4 ,
37 "target_invocations_per_instance" : 20 ,
38 "scale_in_cooldown_seconds" : 300 ,
39 "scale_out_cooldown_seconds" : 60 ,
40 "data_capture_sampling_percent" : 100 ,
41 "alarm_latency_threshold_ms" : 30_000 ,
42 "alarm_5xx_threshold_count" : 5 ,
43 "alarm_overhead_threshold_ms" : 2_000 ,
44 "alarm_evaluation_periods" : 1 ,
45 "alarm_period_seconds" : 300 ,
46 "environment_tag" : "dev" ,
47 }
48
49
50 def log (msg: str ) -> None :
51 _log( "deploy" , msg)
52
53
54 def create_endpoint_config (
55 sm: Any, * , config_name: str , model_name: str , instance_type: str ,
56 initial_instance_count: int , inference_ami_version: str | None ,
57 data_capture_enabled: bool , data_capture_s3_uri: str | None ,
58 tags: list[ dict ],
59 ) -> str :
60 log( f "Creating endpoint config: { config_name } " )
61
62 production_variant: dict[ str , Any] = {
63 "VariantName" : "AllTraffic" ,
64 "ModelName" : model_name,
65 "InstanceType" : instance_type,
66 "InitialInstanceCount" : initial_instance_count,
67 "InitialVariantWeight" : 1.0 ,
68 }
69
70 # InferenceAmiVersion required for vLLM DLC with CUDA 13+. Without it the
71 # container dies on startup with no logs. See hf-cloud-serving-image-selection skill.
72 if inference_ami_version:
73 production_variant[ "InferenceAmiVersion" ] = inference_ami_version
74 log( f " InferenceAmiVersion set to: { inference_ami_version } " )
75
76 kwargs: dict[ str , Any] = {
77 "EndpointConfigName" : config_name,
78 "ProductionVariants" : [production_variant],
79 "Tags" : tags,
80 }
81
82 if data_capture_enabled:
83 if not data_capture_s3_uri:
84 raise ValueError ( "data_capture_enabled but no data_capture_s3_uri provided" )
85 kwargs[ "DataCaptureConfig" ] = {
86 "EnableCapture" : True ,
87 "InitialSamplingPercentage" : DEFAULTS [ "data_capture_sampling_percent" ],
88 "DestinationS3Uri" : data_capture_s3_uri,
89 "CaptureOptions" : [{ "CaptureMode" : "Input" }, { "CaptureMode" : "Output" }],
90 "CaptureContentTypeHeader" : { "JsonContentTypes" : [ "application/json" ]},
91 }
92
93 try :
94 sm.create_endpoint_config( ** kwargs)
95 except ClientError as e:
96 if "Cannot create already existing endpoint configuration" in str (e):
97 log( f "Endpoint config { config_name } already exists — reusing" )
98 else :
99 raise
100 return config_name
101
102
103 def create_endpoint (sm: Any, * , endpoint_name: str , config_name: str , tags: list[ dict ]) -> None :
104 log( f "Creating endpoint: { endpoint_name } " )
105 sm.create_endpoint(
106 EndpointName = endpoint_name,
107 EndpointConfigName = config_name,
108 Tags = tags,
109 )
110
111
112 def register_autoscaling (
113 * , endpoint_name: str , variant_name: str , min_capacity: int , max_capacity: int ,
114 target_invocations: int , scale_in_cooldown: int , scale_out_cooldown: int , region: str ,
115 ) -> None :
116 log( f "Registering autoscaling: min= { min_capacity } max= { max_capacity } target= { target_invocations } /min" )
117 appscaling = boto3.client( "application-autoscaling" , region_name = region)
118 resource_id = f "endpoint/ { endpoint_name } /variant/ { variant_name } "
119
120 appscaling.register_scalable_target(
121 ServiceNamespace = "sagemaker" ,
122 ResourceId = resource_id,
123 ScalableDimension = "sagemaker:variant:DesiredInstanceCount" ,
124 MinCapacity = min_capacity,
125 MaxCapacity = max_capacity,
126 )
127 appscaling.put_scaling_policy(
128 PolicyName = f " { endpoint_name } -target-tracking" ,
129 ServiceNamespace = "sagemaker" ,
130 ResourceId = resource_id,
131 ScalableDimension = "sagemaker:variant:DesiredInstanceCount" ,
132 PolicyType = "TargetTrackingScaling" ,
133 TargetTrackingScalingPolicyConfiguration = {
134 "TargetValue" : float (target_invocations),
135 "PredefinedMetricSpecification" : {
136 "PredefinedMetricType" : "SageMakerVariantInvocationsPerInstance" ,
137 },
138 "ScaleInCooldown" : scale_in_cooldown,
139 "ScaleOutCooldown" : scale_out_cooldown,
140 },
141 )
142
143
144 def create_alarms ( * , endpoint_name: str , variant_name: str , sns_topic_arn: str | None , region: str ) -> None :
145 log( f "Creating CloudWatch alarms for { endpoint_name } " )
146 cw = boto3.client( "cloudwatch" , region_name = region)
147 actions = [sns_topic_arn] if sns_topic_arn else []
148 common_dims = [
149 { "Name" : "EndpointName" , "Value" : endpoint_name},
150 { "Name" : "VariantName" , "Value" : variant_name},
151 ]
152
153 alarms = [
154 {
155 "AlarmName" : f " { endpoint_name } -ModelLatencyP99" ,
156 "MetricName" : "ModelLatency" ,
157 "ExtendedStatistic" : "p99" ,
158 "Threshold" : DEFAULTS [ "alarm_latency_threshold_ms" ] * 1000 , # microseconds
159 "ComparisonOperator" : "GreaterThanThreshold" ,
160 "AlarmDescription" : "Model inference latency p99 > 30s" ,
161 },
162 {
163 "AlarmName" : f " { endpoint_name } -Invocation5XXErrors" ,
164 "MetricName" : "Invocation5XXErrors" ,
165 "Statistic" : "Sum" ,
166 "Threshold" : DEFAULTS [ "alarm_5xx_threshold_count" ],
167 "ComparisonOperator" : "GreaterThanThreshold" ,
168 "AlarmDescription" : "5XX errors > 5 in 5min" ,
169 },
170 {
171 "AlarmName" : f " { endpoint_name } -OverheadLatencyP99" ,
172 "MetricName" : "OverheadLatency" ,
173 "ExtendedStatistic" : "p99" ,
174 "Threshold" : DEFAULTS [ "alarm_overhead_threshold_ms" ] * 1000 ,
175 "ComparisonOperator" : "GreaterThanThreshold" ,
176 "AlarmDescription" : "Platform overhead latency p99 > 2s" ,
177 },
178 ]
179
180 for spec in alarms:
181 params = {
182 "AlarmName" : spec[ "AlarmName" ],
183 "AlarmDescription" : spec[ "AlarmDescription" ],
184 "MetricName" : spec[ "MetricName" ],
185 "Namespace" : "AWS/SageMaker" ,
186 "Dimensions" : common_dims,
187 "Period" : DEFAULTS [ "alarm_period_seconds" ],
188 "EvaluationPeriods" : DEFAULTS [ "alarm_evaluation_periods" ],
189 "Threshold" : spec[ "Threshold" ],
190 "ComparisonOperator" : spec[ "ComparisonOperator" ],
191 "TreatMissingData" : "notBreaching" ,
192 "AlarmActions" : actions,
193 }
194 if "Statistic" in spec:
195 params[ "Statistic" ] = spec[ "Statistic" ]
196 if "ExtendedStatistic" in spec:
197 params[ "ExtendedStatistic" ] = spec[ "ExtendedStatistic" ]
198 cw.put_metric_alarm( ** params)
199
200 if not sns_topic_arn:
201 log( "WARNING: no --sns-alarm-topic — alarms exist but won't notify anyone." )
202
203
204 def main () -> int :
205 p = argparse.ArgumentParser( description = __doc__ , formatter_class = argparse.RawDescriptionHelpFormatter)
206
207 # Required
208 p.add_argument( "--model-name" , required = True )
209 p.add_argument( "--image-uri" , required = True , help = "From hf-cloud-serving-image-selection" )
210 p.add_argument( "--role-arn" , required = True , help = "From hf-cloud-sagemaker-iam-preflight" )
211 p.add_argument( "--instance-type" , required = True , help = "e.g. ml.g5.xlarge" )
212 p.add_argument( "--region" , required = True , help = "From hf-cloud-aws-context-discovery" )
213
214 # Conditional
215 p.add_argument( "--model-s3-uri" , default = None ,
216 help = "S3 model artifact extracted to /opt/ml/model (preferred); omit only for Hub-at-runtime" )
217 p.add_argument( "--env" , action = "append" , default = [], help = "KEY=VALUE; repeatable" )
218 p.add_argument(
219 "--inference-ami-version" , default = None ,
220 help = (
221 "REQUIRED for vLLM DLC with CUDA 13+ (e.g. al2-ami-sagemaker-inference-gpu-3-1). "
222 "Without this, container dies on startup with no logs. "
223 "See hf-cloud-serving-image-selection's 'vLLM AMI requirement' table to map a tag to the AMI version."
224 ),
225 )
226
227 # Naming
228 p.add_argument( "--endpoint-name" , default = None , help = "Default: <model-name>-<timestamp>" )
229 p.add_argument( "--project" , default = None , help = "Tag value (default: model name)" )
230 p.add_argument( "--environment" , default = DEFAULTS [ "environment_tag" ])
231
232 # Capacity / scaling
233 p.add_argument( "--initial-instance-count" , type = int , default = DEFAULTS [ "initial_instance_count" ])
234 p.add_argument( "--min-capacity" , type = int , default = DEFAULTS [ "min_capacity" ])
235 p.add_argument( "--max-capacity" , type = int , default = DEFAULTS [ "max_capacity" ])
236 p.add_argument( "--target-invocations-per-instance" , type = int , default = DEFAULTS [ "target_invocations_per_instance" ])
237 p.add_argument( "--no-autoscaling" , action = "store_true" , help = "NOT RECOMMENDED" )
238
239 # Data capture (off by default)
240 p.add_argument( "--enable-data-capture" , action = "store_true" , help = "Log requests/responses to S3" )
241 p.add_argument( "--data-capture-s3-uri" , default = None )
242
243 # Alarms
244 p.add_argument( "--sns-alarm-topic" , default = None , help = "SNS topic ARN for alarm notifications" )
245 p.add_argument( "--no-alarms" , action = "store_true" )
246
247 args = p.parse_args()
248
249 env_dict = parse_env(args.env)
250 endpoint_name = make_endpoint_name(args.model_name, args.endpoint_name)
251 config_name = f " { endpoint_name } -config"
252
253 sts = boto3.client( "sts" , region_name = args.region)
254 sm = boto3.client( "sagemaker" , region_name = args.region)
255 caller_arn = sts.get_caller_identity()[ "Arn" ]
256 account_id = sts.get_caller_identity()[ "Account" ]
257
258 if args.enable_data_capture and not args.data_capture_s3_uri:
259 args.data_capture_s3_uri = f "s3://sagemaker- { args.region } - { account_id } / { endpoint_name } /data-capture/"
260 log( f "Data capture URI defaulted to: { args.data_capture_s3_uri } " )
261
262 tags = build_tags(
263 project = args.project or args.model_name,
264 caller_arn = caller_arn,
265 environment = args.environment,
266 model_s3_uri = args.model_s3_uri,
267 )
268
269 create_model(
270 sm, model_name = args.model_name, image_uri = args.image_uri,
271 role_arn = args.role_arn, model_s3_uri = args.model_s3_uri,
272 env = env_dict, tags = tags, log_prefix = "deploy" ,
273 )
274 create_endpoint_config(
275 sm, config_name = config_name, model_name = args.model_name,
276 instance_type = args.instance_type, initial_instance_count = args.initial_instance_count,
277 inference_ami_version = args.inference_ami_version,
278 data_capture_enabled = args.enable_data_capture,
279 data_capture_s3_uri = args.data_capture_s3_uri, tags = tags,
280 )
281 create_endpoint(sm, endpoint_name = endpoint_name, config_name = config_name, tags = tags)
282 wait_for_endpoint(sm, endpoint_name, log_prefix = "deploy" )
283
284 if not args.no_autoscaling:
285 register_autoscaling(
286 endpoint_name = endpoint_name, variant_name = "AllTraffic" ,
287 min_capacity = args.min_capacity, max_capacity = args.max_capacity,
288 target_invocations = args.target_invocations_per_instance,
289 scale_in_cooldown = DEFAULTS [ "scale_in_cooldown_seconds" ],
290 scale_out_cooldown = DEFAULTS [ "scale_out_cooldown_seconds" ],
291 region = args.region,
292 )
293 else :
294 log( "WARNING: autoscaling skipped. Endpoint won't scale with traffic." )
295
296 if not args.no_alarms:
297 create_alarms(
298 endpoint_name = endpoint_name, variant_name = "AllTraffic" ,
299 sns_topic_arn = args.sns_alarm_topic, region = args.region,
300 )
301
302 # Summary
303 log( "" )
304 log( f "Deployment complete: { endpoint_name } " )
305 log( f " Instance: { args.instance_type } " )
306 log( f " Autoscaling: { 'OFF' if args.no_autoscaling else f ' { args.min_capacity } - { args.max_capacity } instances' } " )
307 log( f " Data capture: { args.data_capture_s3_uri if args.enable_data_capture else 'OFF (pass --enable-data-capture)' } " )
308 log( "" )
309 log( f "Test: python3 invoke_endpoint.py --endpoint-name { endpoint_name } \\ " )
310 log( f " --payload ' {{\" prompt \" : \" hello \"}} ' --region { args.region } " )
311 log( " (BOM-safe + cross-platform; use 'python' on Windows)" )
312 log( f "Teardown: python3 teardown.py { endpoint_name } { args.region } " )
313
314 # Machine-readable summary for downstream scripting
315 print (json.dumps({
316 "endpoint_name" : endpoint_name,
317 "endpoint_config_name" : config_name,
318 "model_name" : args.model_name,
319 "region" : args.region,
320 "instance_type" : args.instance_type,
321 }))
322
323 return 0
324
325
326 if __name__ == "__main__" :
327 sys.exit(main())