Setting the file. One moment. scripts/verify-operational-value-evaluator.sh
scripts/verify-operational-value-evaluator.sh
Shell·138 lines·6 KB
11
12evaluator=$1
13fixtures=${2:-}
14[[ -f $evaluator ]] || fail "operational-value evaluator not found: $evaluator"
15[[ -x $evaluator ]] || fail "operational-value evaluator is not executable: $evaluator"
16command -v jq >/dev/null 2>&1 || fail "jq is required"
17bash -n "$evaluator"
18
19max_output_bytes=$((1024 * 1024))
20
21validate_output() {
22 local output_file=$1
23 local context=$2
24 local output_bytes
25 local text_bytes
26
27 output_bytes=$(LC_ALL=C wc -c < "$output_file" | tr -d ' ')
28 (( output_bytes <= max_output_bytes )) || fail "$context output exceeds 1 MiB"
29 text_bytes=$(LC_ALL=C tr -d '\000' < "$output_file" | LC_ALL=C wc -c | tr -d ' ')
30 (( text_bytes == output_bytes )) || fail "$context output must not contain NUL bytes"
31
32 jq -se '
33 length == 1
34 and (.[0] | type == "array" and length > 0)
35 and (.[0] | all(.[];
36 type == "object"
37 and keys == ["id", "value"]
38 and (.id | type == "string" and test("[^[:space:]]"))
39 and (.value == null or (.value | type == "number" and isfinite))))
40 and (.[0] | [.[].id] | unique | length) == (.[0] | length)
41 ' "$output_file" >/dev/null || fail "$context must print exactly one non-empty array of unique {id,value} metrics"
42}
43
44output_file=$(mktemp "${TMPDIR:-/tmp}/operational-value-output.XXXXXX")
45trap 'rm -f "$output_file"' EXIT HUP INT TERM
46
47request=$(jq -cn '{
48 schemaVersion: 1,
49 run: {
50 id: "1",
51 attempt: 1,
52 repository: "owner/repo",
53 workflow: "Verification workflow",
54 ref: "refs/heads/main",
55 sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
56 eventName: "workflow_dispatch"
57 },
58 event: {},
59 outputs: [],
60 config: {verification: true}
61}')
62
63if ! printf '%s\n' "$request" | "$evaluator" > "$output_file"; then
64 fail "operational-value evaluator exited unsuccessfully"
65fi
66validate_output "$output_file" "operational-value evaluator"
67
68if [[ -n $fixtures ]]; then
69 [[ -f $fixtures ]] || fail "operational-value fixtures not found: $fixtures"
70 fixture_count=$(jq -er '
71 . as $fixtures
72 | if (($fixtures | type) == "array"
73 and ($fixtures | length) >= 4
74 and all($fixtures[];
75 type == "object"
76 and keys == ["expected", "name", "request"]
77 and (.name | type == "string" and length > 0)
78 and (.request | type == "object")
79 and (.expected | type == "array"))
80 and ([$fixtures[].name] | unique | length) == ($fixtures | length)
81 and (["attained", "missed", "unavailable", "malformed"] - [$fixtures[].name] | length == 0))
82 then ($fixtures | length)
83 else error("invalid fixtures")
84 end
85 ' "$fixtures") || fail "operational-value fixtures are invalid"
86
87 fixture_index=0
88 attained_value=null
89 missed_value=null
90 unavailable_value=false
91 malformed_value=false
92 metric_ids=
93 while (( fixture_index < fixture_count )); do
94 fixture_name=$(jq -er ".[$fixture_index].name" "$fixtures")
95 fixture_request=$(jq -c ".[$fixture_index].request" "$fixtures")
96 fixture_expected=$(jq -cS ".[$fixture_index].expected" "$fixtures")
97
98 printf '%s\n' "$fixture_expected" > "$output_file"
99 validate_output "$output_file" "fixture $fixture_name expected"
100 if ! printf '%s\n' "$fixture_request" | "$evaluator" > "$output_file"; then
101 fail "operational-value evaluator exited unsuccessfully for fixture $fixture_name"
102 fi
103 validate_output "$output_file" "fixture $fixture_name"
104 fixture_actual=$(jq -cS . "$output_file")
105 [[ $fixture_actual == "$fixture_expected" ]] || fail "fixture $fixture_name output does not match expected metrics"
106 fixture_metric_ids=$(jq -c '[.[].id]' "$output_file")
107 if [[ -z $metric_ids ]]; then
108 metric_ids=$fixture_metric_ids
109 else
110 [[ $fixture_metric_ids == "$metric_ids" ]] || fail "fixture $fixture_name changes metric IDs or order"
111 fi
112
113 if ! printf '%s\n' "$fixture_request" | "$evaluator" > "$output_file"; then
114 fail "operational-value evaluator exited unsuccessfully when repeating fixture $fixture_name"
115 fi
116 repeated_actual=$(jq -cS . "$output_file")
117 [[ $repeated_actual == "$fixture_actual" ]] || fail "fixture $fixture_name is not deterministic"
118
119 if [[ $fixture_name == attained ]]; then
120 attained_value=$(jq -c '.[0].value' "$output_file")
121 elif [[ $fixture_name == missed ]]; then
122 missed_value=$(jq -c '.[0].value' "$output_file")
123 elif [[ $fixture_name == unavailable ]]; then
124 unavailable_value=$(jq -c '.[0].value' "$output_file")
125 elif [[ $fixture_name == malformed ]]; then
126 malformed_value=$(jq -c '.[0].value' "$output_file")
127 fi
128 fixture_index=$((fixture_index + 1))
129 done
130
131 jq -en --argjson attained "$attained_value" --argjson missed "$missed_value" '
132 $attained != null and $missed != null and $attained != $missed
133 ' >/dev/null || fail "attained and missed fixtures must have distinct primary values"
134 [[ $unavailable_value == null ]] || fail "unavailable fixture primary metric must be null"
135 [[ $malformed_value == null ]] || fail "malformed fixture primary metric must be null"
136fi
137
138printf 'verified %s\n' "$evaluator"