Setting the file. One moment.
Test Resolve Source Model · LLM To Bedrock · aws/agent-toolkit-for-aws · Skills Docs
Back to the top of the page 81
Creating Amazon Aurora Db Cluster With Instances
104
Routing Traffic With Route53 And CloudFront
Resilience Program Design
Creating API Gateway Stage
— line 94
This file
Number 30.31
Position 31 of 35
Type Python
Size 6 KB
Lines 162 scripts/ test_resolve_source_model.py
Python · 162 lines · 6 KB
14
15 def test_exact_match_wins ():
16 out = resolve([ "gpt-5.4" , "gpt-5.4-2026-03-05" ], "gpt-5.4" )
17 assert out == { "status" : "exact" , "resolved_id" : "gpt-5.4" }
18
19
20 def test_date_suffix_is_same_line ():
21 out = resolve([ "gpt-5.4-2026-03-05" ], "gpt-5.4" )
22 assert out[ "status" ] == "prefix"
23 assert out[ "resolved_id" ] == "gpt-5.4-2026-03-05"
24
25
26 def test_version_suffix_is_same_line ():
27 out = resolve([ "claude-3-5-sonnet-2" ], "claude-3-5-sonnet" )
28 assert out[ "status" ] == "prefix"
29
30
31 def test_shortest_safe_variant_picked ():
32 out = resolve([ "gpt-5.4-2026-03-05-1234" , "gpt-5.4-2026-03-05" ], "gpt-5.4" )
33 assert out[ "resolved_id" ] == "gpt-5.4-2026-03-05"
34
35
36 def test_alphabetic_suffixes_never_auto_resolve ():
37 # mini/pro/latest are different model lines or moving aliases — the exact
38 # failure mode the hard rule exists to prevent.
39 catalog = [ "gpt-5.4-mini" , "gpt-5.4-pro" , "gpt-5.4-latest" ]
40 out = resolve(catalog, "gpt-5.4" )
41 assert out[ "status" ] == "not_found"
42 assert out.get( "ambiguous_prefix" ) is True
43 assert set (out[ "candidates" ]) == set (catalog)
44
45
46 def test_cross_line_never_matches ():
47 # gpt-5 and gpt-54 must not satisfy a gpt-5.4 plan id.
48 out = resolve([ "gpt-5" , "gpt-54" , "gpt-5.5" ], "gpt-5.4" )
49 assert out[ "status" ] == "not_found"
50 assert "ambiguous_prefix" not in out
51
52
53 def test_no_match_ranks_by_longest_common_prefix ():
54 out = resolve([ "claude-3-5-haiku" , "gemini-1.5-pro" , "gpt-4o" ], "claude-3-5-sonnet" )
55 assert out[ "status" ] == "not_found"
56 assert out[ "candidates" ][ 0 ] == "claude-3-5-haiku"
57
58
59 def test_safe_variant_boundary ():
60 assert safe_variant( "gpt-5.4-2026-03-05" , "gpt-5.4" )
61 assert safe_variant( "gpt-5.4-3.1" , "gpt-5.4" )
62 assert not safe_variant( "gpt-5.4-turbo" , "gpt-5.4" )
63 assert not safe_variant( "gpt-5.4-2026-03-05-preview" , "gpt-5.4" )
64
65
66 def test_cli_no_key_exits_2_without_network ():
67 # An env file with no recognized provider key must produce {"status":
68 # "no_key"} and exit 2 before any network call is attempted.
69 script = pathlib.Path( __file__ ).parent / "resolve_source_model.py"
70 with tempfile.TemporaryDirectory() as tmp:
71 env_file = pathlib.Path(tmp) / ".source-provider-env"
72 env_file.write_text( "SOMETHING_ELSE=x \n " , encoding = "utf-8" )
73 proc = subprocess.run( # nosec B603 — test-only, fixed args
74 [sys.executable, str (script), str (env_file)],
75 capture_output = True , text = True ,
76 env = { "PATH" : "/usr/bin:/bin" , "PLAN_MODEL_ID" : "gpt-4o" },
77 )
78 assert proc.returncode == 2
79 assert json.loads(proc.stdout) == { "status" : "no_key" }
80
81
82 def test_provider_selected_from_file_not_ambient_env (tmp_path, monkeypatch):
83 # Review finding: an ambient OPENAI_API_KEY must not select OpenAI when
84 # the migration's env file carries a GEMINI_API_KEY — the model ID would
85 # go to the wrong provider's API.
86 import resolve_source_model as rsm
87 monkeypatch.setenv( "OPENAI_API_KEY" , "sk-ambient" )
88 p = tmp_path / ".source-provider-env"
89 p.write_text( "GEMINI_API_KEY=AIza-file \n " , encoding = "utf-8" )
90 pairs = rsm.load_env_file( str (p))
91 assert rsm.pick_provider(pairs) == "GEMINI_API_KEY"
92
93
94 def test_anthropic_pagination_follows_has_more (monkeypatch):
95 import resolve_source_model as rsm
96 calls = []
97
98 def fake_get (url, headers):
99 calls.append(url)
100 if "after_id" not in url:
101 return { "data" : [{ "id" : "claude-3-5-haiku-20241022" }],
102 "has_more" : True , "last_id" : "claude-3-5-haiku-20241022" }
103 return { "data" : [{ "id" : "claude-3-5-sonnet-20241022" }], "has_more" : False }
104
105 monkeypatch.setattr(rsm, "_get_json" , fake_get)
106 monkeypatch.setenv( "ANTHROPIC_API_KEY" , "sk-ant" )
107 ids = rsm.list_anthropic()
108 assert len (calls) == 2 and "after_id=claude-3-5-haiku-20241022" in calls[ 1 ]
109 # A page-2 model must resolve, not report not_found.
110 assert rsm.resolve(ids, "claude-3-5-sonnet" )[ "status" ] == "prefix"
111
112
113 def test_gemini_pagination_follows_next_page_token (monkeypatch):
114 import resolve_source_model as rsm
115 calls = []
116
117 def fake_get (url, headers):
118 calls.append(url)
119 if "pageToken" not in url:
120 return { "models" : [{ "name" : "models/gemini-1.5-flash" }],
121 "nextPageToken" : "tok2" }
122 return { "models" : [{ "name" : "models/gemini-1.5-pro" }]}
123
124 monkeypatch.setattr(rsm, "_get_json" , fake_get)
125 monkeypatch.setenv( "GEMINI_API_KEY" , "AIza" )
126 ids = rsm.list_gemini()
127 assert len (calls) == 2 and "pageToken=tok2" in calls[ 1 ]
128 assert "gemini-1.5-pro" in ids
129
130
131 def test_source_provider_env_is_authoritative (monkeypatch):
132 # Mirror of the runner's rule: SOURCE_PROVIDER beats file-order guessing.
133 import resolve_source_model as rsm
134 pairs = { "OPENAI_API_KEY" : "a" , "GEMINI_API_KEY" : "c" }
135 monkeypatch.setenv( "SOURCE_PROVIDER" , "google" )
136 assert rsm.pick_provider(pairs) == "GEMINI_API_KEY"
137 monkeypatch.setenv( "SOURCE_PROVIDER" , "anthropic" ) # not in file
138 assert rsm.pick_provider(pairs) is None
139 monkeypatch.delenv( "SOURCE_PROVIDER" )
140 assert rsm.pick_provider(pairs) == "OPENAI_API_KEY"
141
142
143 def test_network_exception_never_reaches_stderr_unredacted (tmp_path, monkeypatch, capsys):
144 # Adjudicated finding: an unhandled exception from the catalog fetch put
145 # its text (which can embed the auth header) on stderr as a traceback.
146 # main() must catch it and emit a REDACTED JSON error instead.
147 import resolve_source_model as rsm
148 env = tmp_path / ".source-provider-env"
149 env.write_text( "OPENAI_API_KEY=redaction-unit-test-123 \n " , encoding = "utf-8" ) # test placeholder, not a credential
150
151 def boom (url, headers):
152 raise ValueError ( "Invalid header value b'Bearer redaction-unit-test-123 \\ rX'" )
153
154 monkeypatch.setattr(rsm, "_get_json" , boom)
155 monkeypatch.setenv( "PLAN_MODEL_ID" , "gpt-4o" )
156 monkeypatch.setattr(sys, "argv" , [ "resolve_source_model.py" , str (env)])
157 rc = rsm.main()
158 captured = capsys.readouterr()
159 assert rc == 3
160 assert "redaction-unit-test-123" not in captured.out and "redaction-unit-test-123" not in captured.err
161 out = json.loads(captured.out)
162 assert out[ "status" ] == "error" and "***" in out[ "detail" ]