Setting the file. One moment. Add Capabilities
81
Creating Amazon Aurora Db Cluster With Instances
104
Routing Traffic With Route53 And CloudFront
Resilience Program Design
Creating API Gateway Stage
— line 135
This file
- Number
- 23.73
- Position
- 73 of 81
- Type
- Python
- Size
- 25 KB
- Lines
- 659
scripts/test_openai_model_recommendation.py
Python·659 lines·25 KB
OPENAI_CATALOG
=
model_recommendation.load_openai_catalog()
15
16
17def _workload(**overrides):
18 workload = {
19 "workload_id": "chat-svc",
20 "source": {
21 "provider": "openai",
22 "model_ids": ["gpt-5.4"],
23 "sdk": "openai",
24 "api_surface": "responses",
25 "source_paths": ["src/app.py"],
26 },
27 "requirements": {
28 "priority": "balanced",
29 "critical_features": [],
30 },
31 "detected_features": [],
32 }
33 for key, value in overrides.items():
34 if key in {"source", "requirements"}:
35 workload[key].update(value)
36 else:
37 workload[key] = value
38 return workload
39
40
41def _recommend(workload=None):
42 return oai.recommend_openai_workload(
43 workload or _workload(), "us-east-2", OPENAI_CATALOG
44 )
45
46
47def _codes(items):
48 return {item["code"] for item in items}
49
50
51def _delta_codes(rec):
52 return {d["code"] for d in rec["migration_deltas"]}
53
54
55# --- 11.1 Provider and path selection -------------------------------------
56
57def test_responses_source_with_continuity_recommends_mantle_responses():
58 rec = _recommend(
59 _workload(requirements={"api_continuity": "required"})
60 )
61 assert rec["provider_module"] == "openai"
62 assert rec["decision_status"] == "recommended"
63 assert rec["api_path"] == "mantle_openai_responses"
64 assert rec["primary_model"] == "openai.gpt-5.6-sol"
65
66
67def test_chat_completions_gpt5_target_selects_responses_not_chat():
68 rec = _recommend(
69 _workload(
70 source={"api_surface": "chat_completions"},
71 requirements={"api_continuity": "required"},
72 )
73 )
74 assert rec["api_path"] == "mantle_openai_responses"
75 assert rec["api_path"] != "mantle_openai_chat"
76 assert "chat_completions_to_responses_required" in _codes(rec["blocks"])
77
78
79def test_chat_completions_case_includes_full_reshape_deltas():
80 rec = _recommend(
81 _workload(
82 source={"api_surface": "chat_completions"},
83 requirements={"api_continuity": "required"},
84 detected_features=["tool_or_function_calling", "conversation_state"],
85 )
86 )
87 codes = _delta_codes(rec)
88 assert "chat_to_responses_request" in codes
89 assert "chat_to_responses_response" in codes
90 assert "chat_to_responses_tools" in codes
91 assert "chat_to_responses_state" in codes
92
93
94def test_governance_selects_runtime_converse():
95 rec = _recommend(
96 _workload(requirements={"governance": ["guardrails", "invocation_logging"]})
97 )
98 assert rec["decision_status"] == "recommended"
99 assert rec["api_path"] == "runtime_converse"
100 assert rec["primary_model"] == "anthropic.claude-sonnet-5"
101
102
103def test_continuity_plus_runtime_only_returns_decision_required():
104 rec = _recommend(
105 _workload(
106 requirements={
107 "api_continuity": "required",
108 "governance": ["guardrails"],
109 }
110 )
111 )
112 assert rec["decision_status"] == "decision_required"
113 assert rec["primary_model"] is None
114 assert rec["api_path"] is None
115 assert {o["api_path"] for o in rec["decision_options"]} == {
116 "mantle_openai_responses",
117 "runtime_converse",
118 }
119 assert "model_path_decision_required" in _codes(rec["blocks"])
120
121
122def test_decision_options_explain_tradeoffs():
123 rec = _recommend(
124 _workload(
125 requirements={"api_continuity": "required", "governance": ["guardrails"]}
126 )
127 )
128 reasons = {o["api_path"]: o["reason"] for o in rec["decision_options"]}
129 assert "OpenAI SDK" in reasons["mantle_openai_responses"]
130 assert "Bedrock-native" in reasons["runtime_converse"]
131
132
133# --- 11.2 Model-generation analysis ---------------------------------------
134
135def test_gpt4_to_gpt5_emits_model_generation_finding():
136 rec = _recommend(
137 _workload(
138 source={"model_ids": ["gpt-4o"], "api_surface": "chat_completions"},
139 requirements={"api_continuity": "required"},
140 )
141 )
142 assert rec["source_analysis"]["source_family"] == "legacy"
143 assert rec["source_analysis"]["model_generation_changes"] is True
144 assert "model_generation_hop" in _delta_codes(rec)
145
146
147def test_oseries_source_is_reasoning_not_legacy():
148 rec = _recommend(_workload(source={"model_ids": ["o3-mini"]}))
149 assert rec["source_analysis"]["source_family"] == "reasoning"
150 assert rec["source_analysis"]["model_generation_changes"] is False
151
152
153def test_opaque_deployment_name_stays_unknown():
154 assert oai.detect_family("prod-default") == "unknown"
155 rec = _recommend(_workload(source={"model_ids": ["prod-default"]}))
156 assert rec["source_analysis"]["source_family"] == "unknown"
157
158
159def test_reasoning_workload_requires_output_headroom():
160 rec = _recommend() # gpt-5.4 reasoning
161 assert "reasoning_token_headroom" in _codes(rec["tuning"])
162
163
164def test_reasoning_headroom_is_a_starting_heuristic_not_a_guarantee():
165 rec = _recommend()
166 msg = next(
167 f["remediation"] for f in rec["tuning"] if f["code"] == "reasoning_token_headroom"
168 )
169 assert "2.5x" in msg
170 assert "STARTING heuristic" in msg or "not a guaranteed" in msg
171
172
173# --- 11.3 Parameter and API behavior --------------------------------------
174
175def test_sampling_is_target_derived_not_source_derived():
176 # G01: the Mantle target is openai.gpt-5.4 (accepts sampling), so BOTH an
177 # early-GPT-5 source and a GPT-5.4 source get the same target-derived finding.
178 early = _recommend(_workload(source={"model_ids": ["gpt-5.1"]}))
179 later = _recommend(_workload(source={"model_ids": ["gpt-5.4"]}))
180 assert "sampling_params_accepted" in _codes(early["tuning"])
181 assert "sampling_params_accepted" in _codes(later["tuning"])
182 # G10: penalties/logprobs/stop are called out as rejected separately.
183 assert "sampling_penalties_rejected" in _codes(early["tuning"])
184
185
186def test_gpt54_does_not_get_anthropic_sampling_removal():
187 rec = _recommend(_workload(source={"model_ids": ["gpt-5.4"]}))
188 assert "sampling_parameters_removed" not in _codes(rec["blocks"])
189 assert "sampling_params_accepted" in _codes(rec["tuning"])
190
191
192def test_n_usage_emits_repeated_call_requirement():
193 rec = _recommend(_workload(requirements={"uses_n": True}))
194 assert "responses_no_n" in _delta_codes(rec)
195
196
197def test_tool_results_emit_function_call_output_delta():
198 rec = _recommend(_workload(detected_features=["tool_or_function_calling"]))
199 assert "tool_result_shape" in _delta_codes(rec)
200
201
202def test_structured_output_emits_text_format_mapping():
203 rec = _recommend(_workload(detected_features=["structured_output_json"]))
204 assert "structured_output_text_format" in _delta_codes(rec)
205
206
207def test_multiturn_state_emits_previous_response_or_app_state():
208 rec = _recommend(_workload(requirements={"uses_hosted_state": True}))
209 assert "conversation_state_ownership" in _delta_codes(rec)
210
211
212# --- 11.4 Architecture impacts --------------------------------------------
213
214def _impact_features(rec):
215 return {a["feature"] for a in rec["architecture_impacts"]}
216
217
218def test_hosted_web_search_not_reported_as_live_native_search():
219 rec = _recommend(_workload(detected_features=["web_search"]))
220 impacts = {a["feature"]: a["impact"] for a in rec["architecture_impacts"]}
221 assert "web_search" in impacts
222 assert "passthrough" in impacts["web_search"].lower()
223
224
225def test_file_search_and_vector_stores_produce_retrieval_impacts():
226 rec = _recommend(
227 _workload(detected_features=["file_search_retrieval", "files_api", "vector_stores"])
228 )
229 assert {"file_search_retrieval", "files_api", "vector_stores"} <= _impact_features(rec)
230
231
232def test_assistants_threads_produce_state_impacts():
233 rec = _recommend(_workload(detected_features=["assistants_threads"]))
234 assert "assistants_threads" in _impact_features(rec)
235
236
237def test_modalities_are_separate_capability_paths():
238 rec = _recommend(
239 _workload(detected_features=["audio_modality", "embeddings", "images"])
240 )
241 assert {"audio_modality", "embeddings", "images"} <= _impact_features(rec)
242
243
244def test_agentic_workload_uses_trajectory_evaluation():
245 rec = _recommend(_workload(detected_features=["tool_or_function_calling"]))
246 assert rec["evaluation"]["mode"] == "trajectory"
247
248
249# --- 11.5 Catalog and verification ----------------------------------------
250
251def test_context_requirement_beyond_every_catalog_window_requires_decision():
252 # 2M exceeds every cataloged window (GPT-5.6 is 1M as of 2026-08-21 — a 400K
253 # requirement now legitimately passes, which an earlier version of this test
254 # used as its probe value). This is the KNOWN-limit-too-small case; the
255 # unknown-limit invariant has its own probe below.
256 rec = _recommend(_workload(requirements={"min_context_tokens": 2000000}))
257 assert rec["decision_status"] == "decision_required"
258
259
260def test_unknown_limits_do_not_pass_hard_numeric_requirement():
261 # Fail-closed invariant: an unknown catalog limit cannot satisfy a hard numeric
262 # need. output_token_ceiling is "unknown" on every entry as of 2026-08-21, so it
263 # probes the invariant the way min_context_tokens no longer can.
264 rec = _recommend(_workload(requirements={"expected_output_tokens": 100000}))
265 assert rec["decision_status"] == "decision_required"
266 assert "unverified_capacity" in _codes(rec["blocks"])
267
268
269def test_cris_only_path_with_unresolvable_residency_blocks_explicitly():
270 # GPT-5.6 on bedrock-runtime is CRIS-only. A residency posture permitting
271 # neither Global nor Geo CRIS must surface an explicit block, not a silent
272 # invocation_model_id: None.
273 rec = _recommend(_workload(
274 source={"model_ids": ["gpt-5.6-terra"], "api_surface": "responses"},
275 requirements={"governance": ["guardrails"], "data_residency": "in_region_required"},
276 ))
277 assert "cris_residency_unresolved" in _codes(rec["blocks"])
278
279def test_no_aws_account_leaves_probe_not_run_and_provisional():
280 rec = _recommend()
281 assert rec["verification"]["probe_status"] == "not_run"
282 assert rec["verification"]["availability_claim"] == "provisional"
283
284
285def test_decision_required_verification_is_not_applicable():
286 rec = _recommend(
287 _workload(requirements={"api_continuity": "required", "governance": ["guardrails"]})
288 )
289 assert rec["verification"]["probe_status"] == "not_applicable"
290 assert rec["verification"]["availability_claim"] == "not_selected"
291
292
293def test_catalog_limits_are_sourced_or_unknown():
294 # Limits must be positive sourced integers or the explicit string "unknown" —
295 # never a fabricated placeholder. Every model must cite its capability source.
296 for model in OPENAI_CATALOG["models"].values():
297 for limit in (model["context_window"], model["output_token_ceiling"]):
298 assert limit == "unknown" or (isinstance(limit, int) and limit > 0)
299 assert model.get("capability_source")
300
301
302# --- Mixed provider provenance (via orchestrator) -------------------------
303
304def test_mixed_provider_run_keeps_catalog_provenance():
305 data = {
306 "schema_version": 2,
307 "region": "us-east-2",
308 "primary_unit": "claude-agent",
309 "workloads": [
310 {
311 "workload_id": "claude-agent",
312 "source": {
313 "provider": "anthropic",
314 "model_ids": ["claude-3-7-sonnet-latest"],
315 "sdk": "anthropic",
316 "api_surface": "messages",
317 "source_paths": ["a.py"],
318 },
319 "requirements": {"priority": "balanced", "critical_features": ["tool_use"]},
320 "detected_features": [],
321 },
322 {
323 "workload_id": "openai-svc",
324 "source": {
325 "provider": "openai",
326 "model_ids": ["gpt-5.4"],
327 "sdk": "openai",
328 "api_surface": "responses",
329 "source_paths": ["b.py"],
330 },
331 "requirements": {"priority": "balanced", "critical_features": [], "api_continuity": "required"},
332 "detected_features": [],
333 },
334 ],
335 }
336 out = model_recommendation.recommend(data)
337 assert out["workloads"]["claude-agent"]["provider_module"] == "anthropic"
338 assert out["workloads"]["openai-svc"]["provider_module"] == "openai"
339 assert out["catalog_provenance"]["claude-agent"]["provider"] == "anthropic"
340 assert out["catalog_provenance"]["openai-svc"]["provider"] == "openai"
341 # Output validates against the schema.
342 schema = json.loads((SCRIPTS / "schemas" / "model-recommendation.json").read_text())
343 jsonschema.validate(out, schema)
344
345
346def test_openai_output_validates_against_schema():
347 data = {
348 "schema_version": 2,
349 "region": "us-east-2",
350 "primary_unit": "chat-svc",
351 "workloads": [_workload(requirements={"api_continuity": "required"})],
352 }
353 schemas = SCRIPTS / "schemas"
354 jsonschema.validate(
355 data, json.loads((schemas / "model-recommendation-input.json").read_text())
356 )
357 out = model_recommendation.recommend(data)
358 jsonschema.validate(
359 out, json.loads((schemas / "model-recommendation.json").read_text())
360 )
361
362
363# --- Regression tests for the Codex gap review (G01-G10) -------------------
364
365def _delta_by_code(rec, code):
366 return next((d for d in rec["migration_deltas"] if d["code"] == code), None)
367
368
369def test_g01_runtime_target_is_not_reported_as_gpt5():
370 # A Bedrock-native (Nova) target must not carry target_version gpt-5.x.
371 rec = _recommend(_workload(requirements={"governance": ["guardrails"]}))
372 assert rec["api_path"] == "runtime_converse"
373 assert rec["source_analysis"]["target_version"] == "5" # Claude Sonnet 5, not gpt-5.x
374 assert rec["source_analysis"]["target_version"] != "gpt-5.x"
375
376
377def test_g01_same_version_source_and_target_not_marked_changed():
378 # gpt-5.6 source -> openai.gpt-5.6-sol target: version did not change.
379 rec = _recommend(_workload(source={"model_ids": ["gpt-5.6"]}))
380 assert rec["source_analysis"]["target_version"] == "5.6"
381 assert rec["source_analysis"]["version_changed"] is False
382
383
384def test_g01_legacy_source_to_reasoning_target_marks_generation_change():
385 rec = _recommend(
386 _workload(source={"model_ids": ["gpt-4o"], "api_surface": "chat_completions"},
387 requirements={"api_continuity": "required"})
388 )
389 assert rec["source_analysis"]["model_generation_changes"] is True
390 assert rec["source_analysis"]["version_changed"] is True
391
392
393def test_g02_runtime_selection_fails_closed_on_unproven_capability():
394 # No runtime_converse candidate has streaming evidence in the catalog, so a
395 # workload requiring it must NOT get a silent native claim — it fails closed.
396 rec = _recommend(
397 _workload(
398 requirements={"governance": ["guardrails"], "critical_features": ["streaming"]}
399 )
400 )
401 assert rec["decision_status"] == "decision_required"
402 assert "unverified_capability" in _codes(rec["blocks"])
403
404
405def test_g02_native_only_lists_catalog_evidenced_features():
406 # Detected structured output is only feature-probed on gpt-5.4, so the engine
407 # falls back past gpt-5.6-sol (no structured-output evidence) to gpt-5.4 and
408 # native lists exactly what that catalog entry supports.
409 rec = _recommend(
410 _workload(detected_features=["tool_or_function_calling", "structured_output_json"])
411 )
412 assert rec["decision_status"] == "recommended"
413 assert rec["primary_model"] == "openai.gpt-5.4" # evidence-driven fallback
414 assert set(rec["compatibility"]["native"]) == {
415 "tool_or_function_calling",
416 "structured_output_json",
417 }
418
419
420def test_g02_vision_requirement_selects_the_evidenced_candidate():
421 # image_input_vision is evidenced on gpt-5.6-sol (model card) but not on
422 # gpt-5.5/terra/luna/5.4 — the engine must select the evidenced candidate,
423 # never claim native on an unevidenced one.
424 rec = _recommend(
425 _workload(
426 requirements={"critical_features": ["image_input_vision"]},
427 )
428 )
429 assert rec["decision_status"] == "recommended"
430 assert rec["primary_model"] == "openai.gpt-5.6-sol"
431 assert "image_input_vision" in rec["compatibility"]["native"]
432
433
434def test_g03_mantle_keeps_typed_responses_parse():
435 rec = _recommend(_workload(detected_features=["structured_output_json"]))
436 delta = _delta_by_code(rec, "structured_output_text_format")
437 assert delta is not None
438 assert "responses.parse" in delta["description"]
439 assert "no direct" not in delta["description"].lower()
440
441
442def test_g04_mantle_state_offers_hosted_and_manual_modes():
443 rec = _recommend(_workload(requirements={"uses_hosted_state": True}))
444 delta = _delta_by_code(rec, "conversation_state_ownership")
445 assert delta is not None
446 assert "store=True" in delta["description"]
447 assert "store=False" in delta["description"]
448
449
450def test_g08_feature_status_unknown_blocks_readiness():
451 rec = _recommend(
452 _workload(
453 requirements={"critical_features": ["structured_output_json"]},
454 feature_status={"structured_output_json": "unknown"},
455 )
456 )
457 assert "feature_scan_incomplete" in _codes(rec["blocks"])
458
459
460def test_g08_feature_assessment_is_populated():
461 rec = _recommend(
462 _workload(
463 detected_features=["tool_or_function_calling"],
464 feature_status={"web_search": "absent"},
465 )
466 )
467 assert rec["feature_assessment"]["tool_or_function_calling"] == "detected"
468 assert rec["feature_assessment"]["web_search"] == "absent"
469
470
471def test_g06_separate_modalities_emit_additional_targets():
472 rec = _recommend(
473 _workload(detected_features=["embeddings", "audio_modality", "images"])
474 )
475 targets = {t["capability"]: t for t in rec["additional_targets"]}
476 assert set(targets) == {"embeddings", "audio_modality", "images"}
477 # Unknown target model stays unresolved with a named service, never a fake ID.
478 for cap, t in targets.items():
479 assert t["status"] == "unresolved"
480 assert t["candidate"] is None
481 assert t["service"]
482
483
484def test_g06_no_modalities_means_empty_additional_targets():
485 rec = _recommend(_workload(detected_features=["tool_or_function_calling"]))
486 assert rec["additional_targets"] == []
487
488
489# --- Regression tests for the consistency re-review (N01-N04, G08 enforcement) ---
490
491import json as _json # noqa: E402
492import pathlib as _pathlib # noqa: E402
493
494_OUT_SCHEMA = _json.loads(
495 (SCRIPTS / "schemas" / "model-recommendation.json").read_text()
496)
497
498
499def _full_recommend(workload):
500 data = {
501 "schema_version": 2,
502 "region": "us-east-2",
503 "primary_unit": workload["workload_id"],
504 "workloads": [workload],
505 }
506 return model_recommendation.recommend(data)
507
508
509def test_n01_decision_required_openai_output_is_schema_valid():
510 # A continuity/governance conflict is decision_required; the full artifact must
511 # validate (model_generation_changes null before selection is allowed).
512 out = _full_recommend(
513 _workload(requirements={"api_continuity": "required", "governance": ["guardrails"]})
514 )
515 jsonschema.validate(out, _OUT_SCHEMA)
516 rec = out["workloads"]["chat-svc"]
517 assert rec["decision_status"] == "decision_required"
518 assert rec["source_analysis"]["model_generation_changes"] is None
519
520
521def test_n02_absent_feature_is_not_reported_native_or_delta():
522 rec = _recommend(
523 _workload(feature_status={"structured_output_json": "absent"})
524 )
525 assert "structured_output_json" not in rec["compatibility"]["native"]
526 assert "structured_output_text_format" not in _delta_codes(rec)
527
528
529def test_n02_status_only_detected_feature_flows_to_behavior():
530 # Detected via feature_status (not the array) must still drive native/delta.
531 rec = _recommend(
532 _workload(detected_features=[], feature_status={"tool_or_function_calling": "detected"})
533 )
534 assert "tool_or_function_calling" in rec["compatibility"]["native"]
535 assert "tool_result_shape" in _delta_codes(rec)
536
537
538def test_g08_unknown_required_feature_forces_decision_required():
539 rec = _recommend(
540 _workload(
541 requirements={"critical_features": ["structured_output_json"]},
542 feature_status={"structured_output_json": "unknown"},
543 )
544 )
545 assert rec["decision_status"] == "decision_required"
546 assert "feature_scan_incomplete" in _codes(rec["blocks"])
547 assert rec["primary_model"] is None
548
549
550def test_n03_tool_requirement_never_lands_on_unevidenced_nova():
551 # Nova has no asserted capabilities; a governance workload requiring tools must
552 # land on the evidenced Converse candidate (Claude), never claim Nova native.
553 rec = _recommend(
554 _workload(
555 requirements={
556 "governance": ["guardrails"],
557 "critical_features": ["tool_or_function_calling"],
558 }
559 )
560 )
561 assert rec["decision_status"] == "recommended"
562 assert rec["primary_model"] == "anthropic.claude-sonnet-5"
563 assert "tool_or_function_calling" in rec["compatibility"]["native"]
564
565
566def test_n04_logprobs_and_stop_not_declared_rejected():
567 rec = _recommend(_workload(source={"model_ids": ["gpt-5.4"]}))
568 finding = next(
569 f for f in rec["tuning"] if f["code"] == "sampling_penalties_rejected"
570 )
571 # Only the two probed penalties are declared rejected in the message.
572 assert "frequency_penalty" in finding["message"]
573 assert "presence_penalty" in finding["message"]
574 assert "logprobs" not in finding["message"]
575 assert "stop" not in finding["message"]
576 # logprobs/stop are routed to verification, not rejection.
577 assert "verify" in finding["remediation"].lower()
578
579
580# --- Converse tier mapping (source OpenAI tier -> matching Claude tier) ----
581
582def test_tier_map_sol_source_maps_to_opus_on_converse():
583 # Verified 2026-08-21: GPT-5.6 runs on bedrock-runtime via CRIS, so a 5.6
584 # source with governance requirements keeps ITS OWN model — the same-model
585 # candidate outranks the Claude tier. (Before 2026-08-21 the docs said
586 # mantle-only, and this test asserted the Opus fallback.)
587 rec = _recommend(
588 _workload(source={"model_ids": ["gpt-5.6-sol"]},
589 requirements={"governance": ["guardrails"]})
590 )
591 assert rec["primary_model"] == "openai.gpt-5.6-sol"
592 assert rec["api_path"] == "runtime_converse"
593
594
595def test_tier_map_sol_cross_family_fallback_is_opus():
596 # The Claude tier map is preserved as the cross-family option behind the
597 # same-model candidate.
598 from openai_model_recommendation import _converse_candidate_order
599 order = _converse_candidate_order({"model_ids": ["gpt-5.6-sol"]})
600 assert order[0] == "openai_gpt_5_6_sol"
601 assert order[1] == "anthropic_claude_opus_4_8"
602
603def test_tier_map_luna_source_maps_to_haiku_on_converse():
604 # Same-model-first (2026-08-21): Luna keeps Luna; Haiku is the cross-family
605 # fallback in the candidate order.
606 rec = _recommend(
607 _workload(source={"model_ids": ["gpt-5.6-luna"]},
608 requirements={"governance": ["guardrails"]})
609 )
610 assert rec["primary_model"] == "openai.gpt-5.6-luna"
611 from openai_model_recommendation import _converse_candidate_order
612 order = _converse_candidate_order({"model_ids": ["gpt-5.6-luna"]})
613 assert order[0] == "openai_gpt_5_6_luna"
614 assert order[1] == "anthropic_claude_haiku_4_5"
615
616def test_tier_map_terra_and_55_sources_map_to_sonnet_on_converse():
617 # Terra (5.6) keeps itself — same-model runtime path exists. GPT-5.5 has NO
618 # runtime path (mantle-only, re-verified 2026-08-21), so its governance
619 # Converse target remains the Claude balanced tier.
620 rec_terra = _recommend(
621 _workload(source={"model_ids": ["gpt-5.6-terra"]},
622 requirements={"governance": ["guardrails"]})
623 )
624 assert rec_terra["primary_model"] == "openai.gpt-5.6-terra"
625 rec_55 = _recommend(
626 _workload(source={"model_ids": ["gpt-5.5"]},
627 requirements={"governance": ["guardrails"]})
628 )
629 assert rec_55["primary_model"] == "anthropic.claude-sonnet-5"
630
631def test_tier_map_falls_back_across_tiers_on_capability_evidence():
632 # The capability-evidence fallback mechanism: when the first candidate in the
633 # order lacks evidence for a critical feature, the engine advances to the next
634 # tier that covers it. Exercised with an explicit order because the Luna
635 # source no longer produces a Haiku-first order — its same-model candidate
636 # carries reasoning evidence and satisfies the requirement directly
637 # (2026-08-21 same-model-first change).
638 from model_recommendation import load_openai_catalog
639 from openai_model_recommendation import _catalog_model_for_path
640 catalog = load_openai_catalog()
641 hit, unmet = _catalog_model_for_path(
642 catalog, "runtime_converse", detected_features=[],
643 requirements={"critical_features": ["reasoning"]},
644 candidate_order=["anthropic_claude_haiku_4_5", "anthropic_claude_sonnet_5"],
645 )
646 assert hit is not None
647 assert hit[0] == "anthropic_claude_sonnet_5"
648
649
650def test_same_model_governance_still_recommended_end_to_end():
651 # A 5.6 source with governance + reasoning requirement lands on itself.
652 rec = _recommend(
653 _workload(source={"model_ids": ["gpt-5.6-luna"]},
654 requirements={"governance": ["guardrails"],
655 "critical_features": ["reasoning"]})
656 )
657 assert rec["decision_status"] == "recommended"
658 assert rec["primary_model"] == "openai.gpt-5.6-luna"
659 assert rec["api_path"] == "runtime_converse"