Setting the file. One moment.
Unit · Graphos Factory · apollographql/skills · Skills Docs
Back to the top of the page 10.20
Release
graphos-factory-core/scripts/unit.sh
graphos-factory-core/scripts/ unit.sh
Shell · 306 lines · 13 KB
15 # given and a plugin is already cached. We seed the cache with the pin.
16 # * it prints "TEST RESULTS: FAILED" and still exits 0. Its exit status
17 # cannot gate anything; the summary line is the verdict, and a run with no
18 # summary at all fails closed.
19 #
20 # Zero cases is not a pass: rover reports an empty suite as SUCCESSFUL with
21 # "0 passed; 0 failed", and nothing was proven. Cases are counted per suite
22 # from rover's TEST SUITE / TEST CASE listing, subdirectories of tests/
23 # included; a suite rover lists that unit.sh cannot match to a file, or the
24 # reverse, fails the run. Every suite that ran no case must cite, in the
25 # first sentence of its header comment, the decision behind it (a D-nnnn
26 # or random D-k7m2qx id from the decision log), or the run fails (exit
27 # 1), the way live.sh fails an exclusion with no reason. When no suite ran a
28 # case, the run exits 3 (not_run) with "unit: no runnable cases:" and each
29 # suite's sentence; when others did, it passes and names the empty ones. A
30 # `skip: true` case is not a pass either: any skipped count fails. A missing
31 # suite file is still a failure.
32 #
33 # Exit codes: 0 pass · 1 fail · 3 not_run (no suite ran a case, each citing
34 # its decision; or APOLLO_ELV2_LICENSE is not `accept`, see elv2.sh) · 127 a
35 # tool is missing.
36 #
37 # --only PATTERN runs only the `tests:` entries (rover's own unit of "one
38 # test") whose quoted `name:` contains PATTERN as a literal substring, across
39 # every suite file. rover has no name-level filter, so a suite that has any
40 # match is rewritten to a temporary copy holding only the matching entries
41 # under its own `config:` header, and that copy is what gets tested — a
42 # suite with zero matches is left out entirely. A PATTERN matching no entry
43 # in any suite fails closed (exit 1, no rover invocation) rather than let
44 # rover report a vacuous "0 passed; 0 failed" as a green summary.
45 set -euo pipefail
46
47 WORKSPACE = "."
48 ONLY = ""
49 while [ $# -gt 0 ]; do
50 case " $1 " in
51 --only )
52 if [ $# -lt 2 ] || [ -z " $2 " ]; then echo "unit: --only requires a non-empty pattern" >&2 ; exit 1 ; fi
53 ONLY = " $2 " ; shift 2 ;;
54 *)
55 WORKSPACE = " $1 " ; shift ;;
56 esac
57 done
58 OUT = "$( mktemp -d )"
59 trap 'rm -rf "$OUT"' EXIT
60
61 # Rewrites $2 to hold only the $1 suite's `tests:` entries whose quoted
62 # `name:` contains $3 as a substring, keeping $1's own `config:` header.
63 # Prints the number of entries kept.
64 extract_only () {
65 local src = " $1 " dst = " $2 " pattern = " $3 "
66 local tests_line body_end
67 tests_line = "$( grep -n '^tests:$' " $src " | head -1 | cut -d: -f1 )"
68 if [ -z " $tests_line " ]; then
69 # A deliberately empty suite (`tests: []`) has nothing to match.
70 if grep -Eq '^tests:[[:space:]]*\[[[:space:]]*\][[:space:]]*(#.*)?$' " $src " ; then echo 0 ; return 0 ; fi
71 echo "unit: $src has no top-level 'tests:' key" >&2
72 return 1
73 fi
74 body_end = "$( awk -v start=" $tests_line " '
75 NR>start && /^[A-Za-z_][A-Za-z0-9_-]*:/ && !found { boundary=NR-1; found=1 }
76 END { if (found) print boundary; else print NR }
77 ' " $src ")"
78 sed -n "1,${ tests_line }p" " $src " > " $dst "
79 awk -v pattern=" $pattern " -v from="$(( tests_line + 1 ))" -v to=" $body_end " '
80 NR<from || NR>to { next }
81 /^ - name: "/ {
82 if (block != "" && index(name, pattern) > 0) printf "%s", block
83 block = $0 "\n"
84 name = $0
85 sub(/^ - name: "/, "", name)
86 sub(/"$/, "", name)
87 next
88 }
89 { block = block $0 "\n" }
90 END { if (block != "" && index(name, pattern) > 0) printf "%s", block }
91 ' " $src " >> " $dst "
92 # `grep -c` exits 1 on a zero count, which is an ordinary outcome here
93 # (this suite just has no matching entry) — never let it read as the
94 # function's own failure, which is reserved for the missing-`tests:` case.
95 grep -c '^ - name: "' " $dst " || true
96 }
97
98 # The product binary, and a refusal (exit 78) when it is older than
99 # the version these scripts pin — see resolve-bin.sh for the order and why.
100 # shellcheck source=SCRIPTDIR/resolve-bin.sh
101 . "$( dirname " $0 ")/resolve-bin.sh"
102
103 if ! command -v rover > /dev/null 2>&1 ; then
104 echo "unit: rover is not installed" >&2
105 exit 127
106 fi
107
108 # shellcheck source=SCRIPTDIR/elv2.sh
109 . "$( dirname " $0 ")/elv2.sh"
110 elv2_require unit
111
112 shopt -s nullglob
113 suites = ( " $WORKSPACE "/tests/ * .connector.yaml )
114 shopt -u nullglob
115 if [ ${ # suites[ @ ]} -eq 0 ]; then
116 echo "unit: no tests/*.connector.yaml suite in $WORKSPACE " >&2
117 echo "unit: every selected operation whose arguments are scalars needs one — this is a failure, not a skip" >&2
118 exit 1
119 fi
120
121 # --unit also rewrites static {$env.NAME} to {$config.NAME} in the temporary
122 # copy: the framework injects $config, $args, $context, $this and $batch, but
123 # not $env. The committed schema keeps {$env...}.
124 eval "$( " $RC " render " $WORKSPACE " --out " $OUT " --unit )"
125
126 # rover reads the user's own APOLLO_ELV2_LICENSE=accept (checked above) for
127 # the plugin install and the test run alike.
128 if ! ls " $HOME /.rover/bin"/supergraph-v * > /dev/null 2>&1 ; then
129 echo "unit: no supergraph plugin cached; installing the pinned v $FEDERATION_VERSION "
130 rover install --plugin "supergraph@= $FEDERATION_VERSION "
131 fi
132
133 TESTDIR = " $WORKSPACE /tests"
134 if [ -n " $ONLY " ]; then
135 mkdir -p " $OUT /only"
136 total = 0
137 for suite in "${ suites [ @ ]}" ; do
138 n = "$( extract_only " $suite " " $OUT /only/$( basename " $suite ")" " $ONLY ")" || exit 1
139 total = $(( total + n ))
140 [ " $n " -gt 0 ] || rm -f " $OUT /only/$( basename " $suite ")"
141 done
142 if [ " $total " -eq 0 ]; then
143 echo "unit: --only ' $ONLY ' matched no case in ${ # suites [ @ ]} suite(s) under $TESTDIR " >&2
144 exit 1
145 fi
146 TESTDIR = " $OUT /only"
147 echo "unit: --only ' $ONLY ' matched $total case(s)"
148 fi
149
150 echo "unit: $( rover --version | head -1 ), ${ # suites [ @ ]} suite(s)"
151 # The temporary schema reads each static credential as {$config.NAME}, and
152 # only the suite supplies it: a suite with a case that lacks one fails every
153 # case for the wrong reason. Each suite this run executes (with --only, the
154 # ones holding a matching case) must hold every name render listed under
155 # config.common.variables.$config; a suite with no case needs none.
156 if [ -n "${ CONFIG_VARS :- }" ]; then
157 if ! command -v jq > /dev/null 2>&1 ; then
158 echo "unit: jq is not installed (it reads each suite's \$ config variables)" >&2
159 exit 127
160 fi
161 read -ra config_vars <<< " $CONFIG_VARS "
162 config_missing = 0
163 for suite in "${ suites [ @ ]}" ; do
164 if [ -n " $ONLY " ] && [ ! -f " $OUT /only/$( basename " $suite ")" ]; then continue ; fi
165 if ! supplied = "$( " $RC " yaml2json " $suite " | jq -r '
166 if type != "object" then "!suite"
167 elif ((.tests // []) | length) == 0 then "-"
168 else (.config.common.variables["$config"] // {}) as $c
169 | if ($c | type) == "object" then ($c | keys[]) else "!config" end
170 end')" ; then
171 echo "unit: FAIL — $suite is not readable YAML" >&2
172 exit 1
173 fi
174 case " $supplied " in
175 - ) continue ;;
176 '!suite' )
177 echo "unit: FAIL — $suite is not a mapping: a suite is a YAML mapping with config: and tests: keys" >&2
178 config_missing = 1 ; continue ;;
179 '!config' )
180 echo "unit: FAIL — $suite : config.common.variables. \$ config is not a mapping; give it one key per variable (${ config_vars [0]}: test-value)" >&2
181 config_missing = 1 ; continue ;;
182 esac
183 lacking = ()
184 for var in "${ config_vars [ @ ]}" ; do
185 grep -qxF -- " $var " <<< " $supplied " || lacking += ( " $var " )
186 done
187 if [ ${ # lacking[ @ ]} -gt 0 ]; then
188 echo "unit: FAIL — $suite does not supply ${ lacking [ * ]}: add each under config.common.variables. \$ config (config: {common: {variables: { \$ config: {${ lacking [0]}: test-value}}}})" >&2
189 config_missing = 1
190 fi
191 done
192 [ " $config_missing " -eq 0 ] || exit 1
193 fi
194
195 log = " $OUT /result.log"
196 NO_COLOR = 1 rover connector --skip-update --skip-update-check test \
197 --schema " $RENDERED_SCHEMA " -d " $TESTDIR " --no-fail-fast | tee " $log "
198
199 # rover's exit status does not reflect suite failures; the summary line does,
200 # so the failure quotes it (evidence records this line as the reason).
201 if grep -q "TEST RESULTS: FAILED" " $log " || ! grep -q "TEST RESULTS: SUCCESSFUL" " $log " ; then
202 verdict = "$( grep -m1 -o 'TEST RESULTS:.*' " $log " || true )"
203 echo "unit: FAIL — ${ verdict :- rover printed no TEST RESULTS summary line }" >&2
204 exit 1
205 fi
206
207
208 summary = "$( grep -m1 '^TEST RESULTS: SUCCESSFUL' " $log ")"
209
210 # A `skip: true` case never runs; rover counts it as skipped and still says
211 # SUCCESSFUL. Skipped is not passed, and it is not the zero-case not_run
212 # either (the case exists and asserts nothing), so any skipped count fails.
213 skipped = "$( sed -nE 's/.*[^0-9]([0-9]+)[[:space:]]+skipped.*/\1/p' <<< " $summary ")"
214 if [ -n " $skipped " ] && [ " $skipped " -gt 0 ]; then
215 echo "unit: FAIL — $skipped case(s) marked skip: true did not run; a skipped case is not a pass (make it run, or delete it and record the decision)" >&2
216 exit 1
217 fi
218
219 # Each suite's reason for having no cases: the first sentence of its header
220 # comment, past the generic title line.
221 suite_reason () {
222 awk '
223 { sub(/\r$/, "") }
224 !/^#/ { exit }
225 { sub(/^#[ \t]*/, ""); if (tolower($0) ~ /^rover connector test suite/) next; text = text (text == "" ? "" : " ") $0 }
226 END {
227 if (text == "") { print "the suite header gives no reason"; exit }
228 n = index(text, ". ")
229 print (n > 0 ? substr(text, 1, n) : text)
230 }
231 ' " $1 "
232 }
233
234 # Cases per suite, from rover's own listing: each `TEST SUITE: <path>` line
235 # is followed by one `TEST CASE:` line per case it ran (a skipped case prints
236 # none). The FAILURES section repeats both, so counting stops there. Printed
237 # as `<count>\t<path>`, keyed by the whole path rover gave.
238 suite_cases = "$( awk '
239 /^FAILURES:/ || /^TEST RESULTS:/ { exit }
240 /^TEST SUITE: / {
241 cur = $0; sub(/^TEST SUITE: /, "", cur); sub(/[ \t]+$/, "", cur)
242 if (!(cur in count)) { order[++k] = cur; count[cur] = 0 }
243 next
244 }
245 /^TEST CASE: / && cur != "" { count[cur]++ }
246 END { for (i = 1; i <= k; i++) print count[order[i]] "\t" order[i] }
247 ' " $log ")"
248
249 # rover walks subdirectories of $TESTDIR, so a suite is its path relative to
250 # $TESTDIR. Every suite rover listed must be a file there, and every suite
251 # file there must be one rover listed; anything else fails rather than
252 # escaping the count.
253 cases = 0
254 reasons = ""
255 uncited = 0
256 listed = ""
257 while IFS = $' \t ' read -r n path ; do
258 [ -n " $path " ] || continue
259 rel = "${ path # " $TESTDIR " / }"
260 if [ " $rel " = " $path " ] || [ ! -f " $TESTDIR / $rel " ]; then
261 echo "unit: FAIL — rover ran $path , which unit.sh cannot account for as a suite under $TESTDIR " >&2
262 exit 1
263 fi
264 listed = " $listed$rel "$' \n '
265 cases = $(( cases + n ))
266 [ " $n " -eq 0 ] || continue
267 reason = "$( suite_reason " $TESTDIR / $rel ")"
268 # The reason is only as good as the decision it cites: without one, an
269 # empty suite (or a header that says nothing about it) would clear the
270 # publication gate on the fallback text alone.
271 if ! grep -Eq '(^|[^[:alnum:]])D-([0-9]{4,}|[0-9a-z]{6})([^[:alnum:]]|$)' <<< " $reason " ; then
272 echo "unit: FAIL — $rel has no cases and its header's first sentence cites no decision (D-0019 or D-k7m2qx): $reason " >&2
273 uncited = $(( uncited + 1 ))
274 fi
275 reasons = "${ reasons : + $reasons ; } $rel : $reason "
276 done <<< " $suite_cases "
277 while IFS = read -r suite ; do
278 rel = "${ suite # " $TESTDIR " / }"
279 if ! grep -Fxq -- " $rel " <<< " $listed " ; then
280 echo "unit: FAIL — rover printed no TEST SUITE line for $rel , so its cases cannot be counted" >&2
281 exit 1
282 fi
283 done < <( find " $TESTDIR " -type f -name '*.connector.yaml')
284 [ " $uncited " -eq 0 ] || exit 1
285
286
287 # rover's count and its listing must agree on whether anything ran; if its
288 # output format moves, fail rather than guess.
289 if grep -Eq '^TEST RESULTS: SUCCESSFUL[[:space:]]+0[[:space:]]+passed;[[:space:]]+0[[:space:]]+failed' <<< " $summary " ; then
290 zero_summary = 1
291 else
292 zero_summary = 0
293 fi
294 if { [ " $cases " -eq 0 ] && [ " $zero_summary " -eq 0 ]; } || { [ " $cases " -gt 0 ] && [ " $zero_summary " -eq 1 ]; }; then
295 echo "unit: FAIL — rover listed $cases case(s) but summarised: $summary " >&2
296 exit 1
297 fi
298
299 if [ " $cases " -eq 0 ]; then
300 echo "unit: no runnable cases: $reasons (not_run)"
301 exit 3
302 fi
303 if [ -n " $reasons " ]; then
304 echo "unit: empty beside suites that ran: $reasons "
305 fi
306 echo "unit: pass"