Setting the file. One moment.
Test Poc Shapes · Agent Advisor · aws/agent-toolkit-for-aws · Skills Docs
ContentsBack to the top of the page Add Capabilities
81
Creating Amazon Aurora Db Cluster With Instances
104
Routing Traffic With Route53 And CloudFront
Resilience Program Design
Creating API Gateway Stage
— line 93
This file
Number 23.74
Position 74 of 81
Type Python
Size 14 KB
Lines 309 scripts/ test_poc_shapes.py
Python · 309 lines · 14 KB
/
"decision-refs"
/
"poc-shapes.md"
)
14 POC_MD = (pathlib.Path( __file__ ).parent.parent
15 / "references" / "phases" / "poc" / "poc.md" )
16 POC_REPORT_MD = (pathlib.Path( __file__ ).parent.parent
17 / "references" / "phases" / "poc" / "poc-report.md" )
18 SKILL_MD = pathlib.Path( __file__ ).parent.parent / "SKILL.md"
19
20
21 def _norm (path):
22 return re.sub( r " \s + " , " " , path.read_text())
23
24
25 def test_files_exist ():
26 assert SHAPES_MD .exists(), SHAPES_MD
27
28
29 def test_ecs_create_allowlist_and_never_list ():
30 text = _norm( SHAPES_MD )
31 assert "Terraform may create ONLY" in text
32 assert "Terraform must NEVER create" in text
33 for never in [ "VPC" , "subnets" , "NAT gateway" , "internet gateway" , "ALB" ]:
34 assert never in text, f "ECS never-list item missing: { never !r} "
35 # No-ALB smoke path: a service that deploys but can't be invoked isn't a POC.
36 assert "No ALB" in text
37 assert "run-task" in text or "execute-command" in text
38
39
40 def test_ecs_networking_never_creates_vpc ():
41 text = _norm( SHAPES_MD )
42 assert "default VPC" in text
43 assert "never create one" in text or "never create a VPC" in text.lower()
44
45
46 def test_eks_unique_namespace_and_no_iam ():
47 text = _norm( SHAPES_MD )
48 assert "agent-advisor-poc-<run_id>" in text, (
49 "EKS namespace must be unique per run — a fixed name like 'poc' risks "
50 "deleting a user's pre-existing namespace on teardown" )
51 assert "Never a fixed name" in text
52 assert "ClusterIP" in text
53 assert "No LoadBalancer" in text
54 assert "creates no IAM resources" in text
55
56
57 def test_eks_never_creates_cluster ():
58 assert "Never creates an EKS cluster" in _norm( SHAPES_MD )
59
60
61 def test_lambda_url_auth_is_iam_never_none ():
62 text = _norm( SHAPES_MD )
63 assert "AWS_IAM" in text
64 assert "never `NONE`" in text or "never NONE" in text, (
65 "a public Bedrock-invoking Function URL is an open cost hole" )
66
67
68 def test_microvms_disabled_fallback ():
69 text = _norm( SHAPES_MD )
70 assert "microvms.tf.disabled" in text
71 assert "MicroVMs config pending verification" in text
72 assert "No deployable claim" in text
73
74
75 def test_temporal_tls_contract_complete ():
76 # All six connection vars, explicitly — the API-key-implies-TLS binary
77 # broke mTLS self-hosted servers (Codex review, Critical 2).
78 for var in [ "TEMPORAL_ADDRESS" , "TEMPORAL_NAMESPACE" , "TEMPORAL_TLS" ,
79 "TEMPORAL_API_KEY" , "TEMPORAL_TLS_SERVER_NAME" ,
80 "TEMPORAL_TLS_CA_PATH" , "TEMPORAL_TLS_CERT_PATH" ,
81 "TEMPORAL_TLS_KEY_PATH" ]:
82 assert var in _norm( SHAPES_MD ), f "TLS contract var missing: { var } "
83 assert "never inferred" in _norm( SHAPES_MD )
84
85
86 def test_temporal_smoke_queue_isolated ():
87 text = _norm( SHAPES_MD )
88 assert "poc-smoke-<run_id>" in text
89 assert "never the user's real queues" in text.lower() or \
90 "never touched production queues" in text
91
92
93 def test_temporal_apply_proves_nothing ():
94 # terraform apply succeeding is not pickup proof — the starter's result is.
95 text = _norm( SHAPES_MD )
96 assert "apply alone proves nothing" in text or \
97 "proves nothing by itself" in text
98
99
100 def test_temporal_teardown_wording ():
101 text = _norm( SHAPES_MD )
102 assert "not deletable resources" in text
103 assert "ages out" in text
104
105
106 def test_temporal_secrets_never_inline ():
107 text = _norm( SHAPES_MD )
108 assert "SSM Parameter Store" in text or "SSM SecureString" in text
109 assert "never inline" in text
110
111
112 def test_poc_md_dispatches_to_shapes ():
113 text = _norm( POC_MD )
114 assert "poc-shapes.md" in text
115 assert "Per-unit runtime dispatch" in text
116 # The old skip message must be gone.
117 assert "supports AgentCore only" not in text
118
119
120 def test_poc_plan_eks_uses_kubectl_not_terraform_runtask ():
121 # Codex round-4 #1: the EKS shape is kubectl apply / port-forward / kubectl delete — NOT
122 # Terraform + run-task. The plan's staged steps must give EKS its own kubectl path.
123 text = _norm( POC_MD )
124 assert re.search( r "eks . {0,120} kubectl apply" , text, re. IGNORECASE ), \
125 "EKS plan must deploy with kubectl apply, not terraform"
126 assert re.search( r "eks . {0,80} kubectl delete" , text, re. IGNORECASE ), \
127 "EKS rollback must be kubectl delete, not terraform destroy / agentcore destroy"
128
129
130 def test_poc_plan_covers_lambda_microvms ():
131 # Codex round-4 #4: lambda_microvms is dispatched as its own runtime, so the plan's staged
132 # steps AND rollback list must include it (it was missing from both).
133 text = _norm( POC_MD )
134 # It appears in the staged-steps runtime list and the rollback list.
135 assert text.count( "lambda_microvms" ) >= 2 , \
136 "poc.md plan must cover lambda_microvms in both staged steps and rollback"
137
138
139 def test_poc_md_dispatches_temporal_worker_poll ():
140 # M2: temporal_worker_poll units dispatch to the temporal shape in poc-shapes.
141 text = _norm( POC_MD )
142 assert "temporal_worker_poll" in text
143 assert "Temporal worker POC" in text
144 assert "shape in" in text and "poc-shapes.md" in text
145
146
147 def test_skill_md_rows ():
148 # SKILL.md's execution moved from a hand-written state-machine table to the
149 # interpreter-loop delegation (Routing & gates orchestration prose). The two
150 # invariants this test pins are unchanged — only where they live moved.
151 text = _norm( SKILL_MD )
152 # The temporal branch must not be routable.
153 assert "load `references/phases/temporal-worker" not in text, \
154 "the temporal branch must not be routable"
155 # Gate 2 / poc is not AgentCore-gated: any winning runtime gets a POC shape.
156 gate2 = re.search( r " \*\* Gate 2 → `poc` \*\* (. *? ) - Persisting" , text)
157 assert gate2, "SKILL.md must carry the Gate 2 → poc semantics"
158 assert "Any winning runtime" in gate2.group( 1 )
159 assert "== agentcore" not in gate2.group( 1 )
160
161
162 def test_poc_report_uses_v3_shell ():
163 # BUG -A fix: poc-report.md must reference the v3 document shell.
164 text = _norm( POC_REPORT_MD )
165 assert "report-shell.md" in text, "poc-report must reference report-shell.md"
166 assert "SHARED_SHELL_CSS" in text or "shared shell" in text.lower(), \
167 "poc-report must inline the shared shell CSS"
168 assert "SHARED_SHELL_MERMAID_TAG" in text or "SRI-pinned mermaid" in text.lower(), \
169 "poc-report must use the SRI-pinned mermaid script tag"
170 assert ".doc-head" in text, "poc-report must use .doc-head"
171 assert ".help-strip" in text or "HELP_URL" in text, \
172 "poc-report must reference .help-strip or HELP_URL"
173
174
175 def test_poc_model_resolved_per_unit ():
176 # BUG -C fix: poc.md Step 2 must resolve model per-unit from design.json.units.
177 text = _norm( POC_MD )
178 # Step 2 mentions resolving per unit, not just design_blocks[0] / FIRST entry alone.
179 assert "per unit" in text.lower() or "for each unit" in text.lower() or \
180 "units[<id>]" in text or "units[<unit-id>]" in text, \
181 "Step 2 must resolve model per-unit"
182 # design.json is the authoritative model source (already reconciled by the plan);
183 # the POC resolves each unit's model from its OWN entry.
184 assert "single source of truth" in text.lower() or "authoritative" in text.lower(), \
185 "Step 2 must state design.json is the authoritative model source"
186 # The multi-unit dispatch in Step 3 must mention using each unit's model.
187 assert "THAT unit's model" in text or "unit's model id" in text.lower(), \
188 "Step 3 dispatch must use each unit's model id"
189
190
191 def test_poc_model_not_taken_from_first_block_for_all_units ():
192 # BUG -C regression (multi-unit, plan-backed): the plan-backed cross-check must NOT
193 # tell the POC to read design_blocks[0] for every unit — that takes the first unit's
194 # model for all units. It must match each unit's OWN block by evidence overlap.
195 text = _norm( POC_MD )
196 assert "NOT `design_blocks[0]`" in text or "not `design_blocks[0]`" in text.lower(), \
197 "Step 2 plan-backed cross-check must warn against using design_blocks[0] for all units"
198 assert "source_paths" in text and "evidence" in text, \
199 "Step 2 plan-backed cross-check must match a unit to its own block by evidence overlap"
200
201
202 def test_poc_dispatches_on_effective_runtime_not_raw_verdict ():
203 # Codex P1 #1: a consolidated platform decision must actually drive the POC. The
204 # dispatch resolves an "effective runtime" (platform.runtime under consolidated,
205 # else the unit verdict) rather than always deploying each unit's raw split verdict.
206 text = _norm( POC_MD )
207 assert "effective runtime" in text.lower(), \
208 "poc.md must resolve an effective runtime before dispatch"
209 assert re.search( r 'platform \. mode \s * == \s * " ? consolidated" ? ' , text), \
210 "poc.md must key the consolidated case on platform.mode == consolidated"
211 assert "platform.runtime" in text, \
212 "poc.md consolidated case must deploy on platform.runtime (the superset)"
213
214
215 def test_poc_postcondition_dispatches_on_effective_runtime ():
216 # Codex round-6 #1: the _postcondition must judge the POC by effective_runtime, not the raw
217 # verdict — a consolidated unit's correct ECS POC must not be failed for not matching its
218 # raw split verdict (agentcore/lambda/batch).
219 text = _norm( POC_MD )
220 # Grab the frontmatter (between the first two --- fences).
221 fm = text.split( "---" , 2 )[ 1 ] if text.count( "---" ) >= 2 else ""
222 assert "effective_runtime" in fm, \
223 "poc.md _postcondition must dispatch on effective_runtime"
224 assert re.search( r "NOT the raw split verdict | not the raw . {0,20} verdict" , fm, re. IGNORECASE ), \
225 "poc.md _postcondition must explicitly not use the raw split verdict"
226
227
228 def test_poc_dispatch_includes_fargate ():
229 # Codex P1 #2: W5/W6 produce a fargate verdict; dispatch must cover it. fargate reuses
230 # the ecs shape (ECS-on-Fargate), so no separate shape is authored.
231 text = _norm( POC_MD )
232 assert "fargate" in text, "poc.md dispatch must include fargate"
233 shapes = _norm( SHAPES_MD )
234 assert re.search( r "fargate . {0,80} ecs shape | `fargate` . {0,80} ecs | ecs . {0,120} fargate" ,
235 shapes, re. IGNORECASE ), \
236 "poc-shapes.md ecs shape must state it also serves fargate verdicts"
237 # Codex round-2 #4: W5 is "Fargate behind ALB" but the ecs shape has no ALB. The shape must
238 # honestly scope-limit: the POC validates the container + Bedrock, NOT the ALB ingress.
239 assert re.search( r "W5 . {0,200} ( does NOT provision an ALB | not the public ALB | without . {0,20} ALB ) " ,
240 shapes, re. IGNORECASE | re. DOTALL ), \
241 "ecs shape must state the W5 ALB scope limit (POC validates container, not ALB ingress)"
242
243
244 def test_batch_shape_exists ():
245 # BUG -8 fix: batch POC shape must exist in poc-shapes.md.
246 text = _norm( SHAPES_MD )
247 assert "## batch" in text, "poc-shapes.md must have a ## batch section"
248
249
250 def test_batch_create_allowlist_and_never_list ():
251 text = _norm( SHAPES_MD )
252 # Batch section must have create allowlist and never-list.
253 assert "aws_batch_compute_environment" in text
254 assert "aws_batch_job_queue" in text
255 assert "aws_batch_job_definition" in text
256 for never in [ "VPC" , "subnets" , "NAT gateway" , "internet gateway" , "always-on compute" ]:
257 assert never in text, f "Batch never-list item missing: { never !r} "
258
259
260 def test_batch_secrets_never_inline ():
261 text = _norm( SHAPES_MD )
262 # Batch shape must use SSM/Secrets Manager, never inline.
263 assert "SSM Parameter Store" in text or "SSM SecureString" in text or "Secrets Manager" in text
264 assert "never inline" in text
265
266
267 def test_batch_teardown_wording ():
268 text = _norm( SHAPES_MD )
269 # Batch teardown must mention resources not deletable while jobs run.
270 assert "not deletable" in text or "not deletable resources" in text
271 assert "drain" in text or "ages out" in text
272
273
274 def test_poc_md_dispatches_batch ():
275 # BUG -8 fix: poc.md Step 3 dispatch must include batch.
276 text = _norm( POC_MD )
277 assert "batch" in text
278 # The dispatch list should include batch alongside ecs/eks/lambda/lambda_microvms.
279 assert "batch / ecs / eks / lambda / lambda_microvms" in text or \
280 "batch/ecs/eks/lambda/lambda_microvms" in text or \
281 ( "batch" in text and "ecs / eks / lambda / lambda_microvms" in text), \
282 "poc.md Step 3 dispatch must include batch"
283
284
285 def test_poc_md_multiunit_dispatch_is_unconditionally_first ():
286 # BUG -9 fix: Step 3 must unambiguously gate on unit count FIRST, before runtime dispatch.
287 text = _norm( POC_MD )
288 # Multi-unit decision must be explicit at top of Step 3.
289 assert "Multi-unit system decision gate" in text or \
290 "equals design.json.units[].length" in text, \
291 "Step 3 must have explicit multi-unit decision gate"
292 # Must state that Temporal systems with multiple units are multi-unit, not one app.
293 assert "Temporal system" in text and "multi-unit" in text.lower(), \
294 "Step 3 must clarify Temporal systems with >1 unit are multi-unit"
295 assert "NOT one app" in text or "never one" in text, \
296 "Step 3 must state multi-unit Temporal systems don't collapse to one poc/app/"
297 # The agentcore branches (3-F/3-H/3a-e) are per-unit shapes, not whole-system.
298 assert "PER-UNIT shape" in text or "per-unit shape" in text, \
299 "Step 3 must clarify 3-F/3-H/3a-e are per-unit shapes"
300 # Per-unit runtime dispatch heading must NOT say "FIRST" — only the multi-unit gate is first.
301 assert "Per-unit runtime dispatch" in text, \
302 "Step 3 must have 'Per-unit runtime dispatch' heading"
303 # The old "RUNTIME DISPATCH FIRST" heading must be gone or reworded.
304 step3_text = re.search( r "## Step 3 — . *? ## Step 4" , text, re. DOTALL )
305 assert step3_text, "Could not find Step 3 section"
306 # Count "FIRST" occurrences in Step 3 — there should be exactly 0 or 1 (the decision gate).
307 first_count = step3_text.group( 0 ).lower().count( "dispatch first" )
308 assert first_count <= 1 , \
309 f "Step 3 has { first_count } 'DISPATCH FIRST' headings; only the multi-unit gate may be first"