Setting the file. One moment. tests/test.sh
tests/test.sh
Shell·220 lines·9 KB
=
$(
mktemp
-d
"
$repo_root
/.operational-value-designer-test.XXXXXX"
)
8trap 'rm -rf "$work_dir"' EXIT HUP INT TERM
9
10grep -Fq 'Choose `script` for compact, workflow-specific Bash' "$skill_dir/SKILL.md"
11grep -Fq 'Choose `run` when the Bash is large enough to obscure the workflow' "$skill_dir/SKILL.md"
12grep -Fq '65,536 UTF-8 bytes, inclusive' "$skill_dir/SKILL.md"
13grep -Fq '4,096 Unicode characters, inclusive' "$skill_dir/SKILL.md"
14grep -Fq '`run` does not reduce the generated workflow size' "$skill_dir/SKILL.md"
15grep -Fq 'comment the workflow intent and the meaning of every emitted metric' "$skill_dir/SKILL.md"
16grep -Fq 'ultimate operational condition' "$skill_dir/SKILL.md"
17grep -Fq 'Select the furthest downstream rung that satisfies all three' "$skill_dir/SKILL.md"
18grep -Fq 'observable at grading time, attributable to this run or its exact subject, and independently verifiable' "$skill_dir/SKILL.md"
19grep -Fq 'why stronger downstream effects are unavailable' "$skill_dir/SKILL.md"
20grep -Fq 'Never reward output merely for existing' "$skill_dir/SKILL.md"
21grep -Fq 'Missing evidence for the selected rung returns `null`; it never causes runtime fallback to a weaker rung' "$skill_dir/SKILL.md"
22
23evaluator_path=$("$skill_dir/scripts/operational-value-evaluator-path.sh" daily-file-diet)
24[[ $evaluator_path == .github/graders/daily-file-diet-operational-value.sh ]]
25if "$skill_dir/scripts/operational-value-evaluator-path.sh" ../escape >/dev/null 2>&1; then
26 printf 'invalid workflow name was accepted\n' >&2
27 exit 1
28fi
29
30valid_evaluator="$work_dir/valid.sh"
31cat > "$valid_evaluator" <<'EOF'
32#!/usr/bin/env bash
33
34set -euo pipefail
35
36[[ $# -eq 0 ]]
37request=$(cat)
38printf '%s\n' "$request" | jq -e '
39 .schemaVersion == 1
40 and .run.id == "1"
41 and .run.attempt == 1
42 and .run.repository == "owner/repo"
43 and .run.workflow == "Verification workflow"
44 and .run.ref == "refs/heads/main"
45 and .run.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
46 and .run.eventName == "workflow_dispatch"
47 and (.event | type == "object")
48 and (.outputs | type == "array")
49 and .config.verification == true
50' >/dev/null
51printf 'verification diagnostic\n' >&2
52case_name=$(printf '%s\n' "$request" | jq -r '.config.case // "attained"')
53case "$case_name" in
54 attained)
55 printf '%s\n' '[{"id":"triaged-issues","value":2.5},{"id":"net-cost","value":-3.25}]'
56 ;;
57 missed)
58 printf '%s\n' '[{"id":"triaged-issues","value":0.5},{"id":"net-cost","value":-7.75}]'
59 ;;
60 unavailable|malformed)
61 printf '%s\n' '[{"id":"triaged-issues","value":null},{"id":"net-cost","value":null}]'
62 ;;
63 *)
64 exit 1
65 ;;
66esac
67EOF
68chmod +x "$valid_evaluator"
69
70"$skill_dir/scripts/verify-operational-value-evaluator.sh" "$valid_evaluator" >/dev/null
71
72fixtures="$work_dir/fixtures.json"
73jq -n '
74 def request($case): {
75 schemaVersion: 1,
76 run: {
77 id: "1",
78 attempt: 1,
79 repository: "owner/repo",
80 workflow: "Verification workflow",
81 ref: "refs/heads/main",
82 sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
83 eventName: "workflow_dispatch"
84 },
85 event: {},
86 outputs: [],
87 config: {verification: true, case: $case}
88 };
89 [
90 {name: "attained", request: request("attained"), expected: [{id: "triaged-issues", value: 2.5}, {id: "net-cost", value: -3.25}]},
91 {name: "missed", request: request("missed"), expected: [{id: "triaged-issues", value: 0.5}, {id: "net-cost", value: -7.75}]},
92 {name: "unavailable", request: request("unavailable"), expected: [{id: "triaged-issues", value: null}, {id: "net-cost", value: null}]},
93 {name: "malformed", request: request("malformed"), expected: [{id: "triaged-issues", value: null}, {id: "net-cost", value: null}]}
94 ]
95' > "$fixtures"
96"$skill_dir/scripts/verify-operational-value-evaluator.sh" "$valid_evaluator" "$fixtures" >/dev/null
97
98assert_fixtures_rejected() {
99 local name=$1
100 local invalid_fixtures=$2
101 if "$skill_dir/scripts/verify-operational-value-evaluator.sh" "$valid_evaluator" "$invalid_fixtures" >/dev/null 2>&1; then
102 printf 'invalid evaluator fixtures were accepted: %s\n' "$name" >&2
103 exit 1
104 fi
105}
106
107missing_fixture="$work_dir/missing-fixture.json"
108jq 'map(select(.name != "malformed"))' "$fixtures" > "$missing_fixture"
109assert_fixtures_rejected missing-required-case "$missing_fixture"
110
111mismatched_fixture="$work_dir/mismatched-fixture.json"
112jq 'map(if .name == "attained" then .expected[0].value = 0.5 else . end)' "$fixtures" > "$mismatched_fixture"
113assert_fixtures_rejected mismatched-output "$mismatched_fixture"
114
115reversed_fixture="$work_dir/reversed-fixture.json"
116jq 'map(
117 if .name == "attained" then .request.config.case = "missed" | .expected[0].value = 0
118 elif .name == "missed" then .request.config.case = "attained" | .expected[0].value = 1
119 else . end
120)' "$fixtures" > "$reversed_fixture"
121assert_fixtures_rejected reversed-semantics "$reversed_fixture"
122
123contract_repo="$work_dir/contract-repo"
124mkdir -p "$contract_repo/.github/graders" "$contract_repo/.github/workflows"
125git -C "$contract_repo" init -q
126git -C "$contract_repo" config user.email test@example.com
127git -C "$contract_repo" config user.name "Operational Value Test"
128printf '%s\n' '#!/usr/bin/env bash' > "$contract_repo/.github/graders/example-operational-value.sh"
129printf '%s\n' '# Example' > "$contract_repo/.github/workflows/example.md"
130git -C "$contract_repo" add .
131git -C "$contract_repo" commit -qm baseline
132baseline_commit=$(git -C "$contract_repo" rev-parse HEAD)
133
134printf '%s\n' 'printf changed' >> "$contract_repo/.github/graders/example-operational-value.sh"
135if git -C "$contract_repo" -c advice.detachedHead=false \
136 --no-pager diff --quiet "$baseline_commit" -- .github/graders/example-operational-value.sh; then
137 printf 'contract test evaluator did not change\n' >&2
138 exit 1
139fi
140if (cd "$contract_repo" && "$skill_dir/scripts/verify-operational-value-contract-change.sh" "$baseline_commit") >/dev/null 2>&1; then
141 printf 'unpaired evaluator change was accepted\n' >&2
142 exit 1
143fi
144(cd "$contract_repo" && "$skill_dir/scripts/verify-operational-value-contract-change.sh" --correction "$baseline_commit") >/dev/null
145
146printf '%s\n' 'Updated contract.' >> "$contract_repo/.github/workflows/example.md"
147(cd "$contract_repo" && "$skill_dir/scripts/verify-operational-value-contract-change.sh" "$baseline_commit") >/dev/null
148
149make_evaluator() {
150 local name=$1
151 local evaluator_output=$2
152 local generated_evaluator="$work_dir/$name.sh"
153 cat > "$generated_evaluator" <<EOF
154#!/usr/bin/env bash
155set -euo pipefail
156cat <<'JSON'
157$evaluator_output
158JSON
159EOF
160 chmod +x "$generated_evaluator"
161 printf '%s\n' "$generated_evaluator"
162}
163
164assert_rejected() {
165 local name=$1
166 local evaluator_output=$2
167 local generated_evaluator
168 generated_evaluator=$(make_evaluator "$name" "$evaluator_output")
169 if "$skill_dir/scripts/verify-operational-value-evaluator.sh" "$generated_evaluator" >/dev/null 2>&1; then
170 printf 'invalid evaluator output was accepted: %s\n' "$name" >&2
171 exit 1
172 fi
173}
174
175assert_rejected empty-array '[]'
176assert_rejected top-level-object '{"id":"score","value":1}'
177assert_rejected duplicate-ids '[{"id":"same","value":1},{"id":"same","value":0}]'
178assert_rejected empty-id '[{"id":" ","value":1}]'
179assert_rejected boolean-value '[{"id":"score","value":true}]'
180assert_rejected string-value '[{"id":"score","value":"1"}]'
181assert_rejected extra-field '[{"id":"score","value":1,"message":"not allowed"}]'
182assert_rejected multiple-documents $'[{"id":"first","value":1}]\n[{"id":"second","value":0}]'
183assert_rejected invalid-json 'not json'
184
185nul_evaluator="$work_dir/nul-output.sh"
186cat > "$nul_evaluator" <<'EOF'
187#!/usr/bin/env bash
188printf '[{"id":"score","value":1}]\0'
189EOF
190chmod +x "$nul_evaluator"
191if "$skill_dir/scripts/verify-operational-value-evaluator.sh" "$nul_evaluator" >/dev/null 2>&1; then
192 printf 'evaluator output containing a NUL byte was accepted\n' >&2
193 exit 1
194fi
195
196failed_evaluator="$work_dir/failed.sh"
197cat > "$failed_evaluator" <<'EOF'
198#!/usr/bin/env bash
199exit 1
200EOF
201chmod +x "$failed_evaluator"
202if "$skill_dir/scripts/verify-operational-value-evaluator.sh" "$failed_evaluator" >/dev/null 2>&1; then
203 printf 'unsuccessful evaluator was accepted\n' >&2
204 exit 1
205fi
206
207oversized_evaluator="$work_dir/oversized.sh"
208cat > "$oversized_evaluator" <<'EOF'
209#!/usr/bin/env bash
210set -euo pipefail
211printf '[{"id":"score","value":1}]'
212dd if=/dev/zero bs=1048576 count=1 2>/dev/null | tr '\0' ' '
213EOF
214chmod +x "$oversized_evaluator"
215if "$skill_dir/scripts/verify-operational-value-evaluator.sh" "$oversized_evaluator" >/dev/null 2>&1; then
216 printf 'oversized evaluator output was accepted\n' >&2
217 exit 1
218fi
219
220printf 'operational-value-designer skill tests passed\n'