Setting the file. One moment.
Live · Graphos Factory · apollographql/skills · Skills Docs
Back to the top of the page 10.20
Release
(opens in a new tab)
graphos-factory-core/scripts/ live.sh
Shell · 200 lines · 10 KB
15
# .data.x_draw.cards # response; must evaluate to boolean true
16 # | length == 2
17 # - name: project # a case may take GraphQL variables from an
18 # variables: # EARLIER case's response: `from` names the
19 # projectId: # case, `path` is a jq path into its response.
20 # from: list_projects # A path that yields null, or a case that has
21 # path: .data.x_listProjects.results[0].id # not run or failed,
22 # # fails this case (never a silent skip).
23 # exclusions: # what this layer cannot exercise, and why
24 # - operation: "get:/users/me" # a selected operation
25 # reason: "accepts only a user-scoped token; the CI key is account-level"
26 # - field: "X_Payment.card" # a relationship field, <Type>.<field>
27 # reason: "its only parent is a write; no write is sent live"
28 #
29 # Each entry names exactly one of `operation` or `field`. An operation
30 # exclusion is printed as `EXCLUDED: <operation> — <reason>` and lands in
31 # evidence as that operation's live status `excluded` with the reason, so an
32 # operation nobody could test live is visible, not an anonymous `n/a`. A
33 # field exclusion is printed as `EXCLUDED FIELD: <Type>.<field> — <reason>`
34 # and lands in the live layer's findings: a relationship field has no
35 # operations row, and an excluded one is never reported as passed.
36 #
37 # Baseline per case: no `errors`, non-empty `data`. `require:` is for the
38 # nullable fields the baseline cannot see (references/testing.md).
39 #
40 # Credentials come from the environment, or from a gitignored smoke.env found
41 # from the workspace up to its checkout's top (or $GRAPHOS_FACTORY_CORE_LIVE_ENV);
42 # the environment wins. See live-env.sh.
43 #
44 # Exit codes: 0 pass · 1 fail · 3 not_run (credential unset, no
45 # tests/live.yaml, or APOLLO_ELV2_LICENSE not `accept`, see elv2.sh) · 127
46 # tool missing. 3 is a first-class result: evidence
47 # records it as `not_run` with the reason, never as a pass.
48 set -euo pipefail
49
50 WORKSPACE = " ${1 :- . } "
51 ROUTER_VERSION = "${ ROUTER_VERSION :- 2 . 17 . 0 }"
52 ROUTER_PORT = "${ ROUTER_PORT :- 4000 }"
53 ROUTER_HEALTH_PORT = "${ ROUTER_HEALTH_PORT :- 8088 }"
54
55 for tool in curl jq ; do
56 command -v " $tool " > /dev/null 2>&1 || { echo "live: $tool is not installed" >&2 ; exit 127 ; }
57 done
58 command -v rover > /dev/null 2>&1 || { echo "live: rover is not installed" >&2 ; exit 127 ; }
59
60 # shellcheck source=SCRIPTDIR/cache.sh
61 . "$( dirname " $0 ")/cache.sh"
62 CACHE = " $GRAPHOS_FACTORY_CORE_CACHE_DIR "
63 # The product binary, and a refusal (exit 78) when it is older than
64 # the version these scripts pin — see resolve-bin.sh for the order and why.
65 # shellcheck source=SCRIPTDIR/resolve-bin.sh
66 . "$( dirname " $0 ")/resolve-bin.sh"
67
68
69 LIVE = " $WORKSPACE /tests/live.yaml"
70 if [ ! -f " $LIVE " ]; then
71 echo "live: no tests/live.yaml — nothing to run (not_run)"
72 exit 3
73 fi
74
75 OUT = "$( mktemp -d )"
76 ROUTER_PID = ""
77 cleanup () { if [ -n " $ROUTER_PID " ]; then kill " $ROUTER_PID " 2> /dev/null || true ; fi ; rm -rf " $OUT " ; }
78 trap cleanup EXIT
79
80 # The live layer's credentials file (smoke.env, or $GRAPHOS_FACTORY_CORE_LIVE_ENV),
81 # loaded into this process only — see live-env.sh for why no other layer does.
82 # shellcheck source=SCRIPTDIR/live-env.sh
83 . "$( dirname " $0 ")/live-env.sh"
84 live_env_load " $WORKSPACE " || exit 1
85
86 # Credential: the env var named by AUTH_EXPR's test_default must be set with a
87 # REAL value here. Unset means not_run, never a run with `test-token`.
88 AUTH_VAR = "$( " $RC " auth-env " $WORKSPACE ")"
89 if [ -n " $AUTH_VAR " ] && [ -z "${ ! AUTH_VAR :- }" ]; then
90 echo "live: $AUTH_VAR is not set — skipping live smoke (not_run)"
91 exit 3
92 fi
93
94 # shellcheck source=SCRIPTDIR/elv2.sh
95 . "$( dirname " $0 ")/elv2.sh"
96 elv2_require live
97
98 for port in " $ROUTER_PORT " " $ROUTER_HEALTH_PORT " ; do
99 if curl -s "http://localhost: $port /" > /dev/null 2>&1 ; then echo "live: port $port is busy" >&2 ; exit 1 ; fi
100 done
101
102 ROUTER_BIN = " $CACHE /router- $ROUTER_VERSION "
103 if [ ! -x " $ROUTER_BIN " ]; then
104 echo "live: Apollo Router $ROUTER_VERSION is not cached at $ROUTER_BIN — run e2e.sh once, or the SessionStart hook" >&2
105 exit 127
106 fi
107
108 eval "$( " $RC " render " $WORKSPACE " --out " $OUT ")"
109 NO_COLOR = 1 rover supergraph compose --config " $COMPOSE_CONFIG " > " $OUT /supergraph.graphql" 2> " $OUT /compose.err" \
110 || { cat " $OUT /compose.err" >&2 ; echo "live: compose failed" >&2 ; exit 1 ; }
111
112 # No override_url: connectors hit the real upstream. Subgraph errors stay
113 # visible so a mapped error is asserted, not redacted.
114 cat > " $OUT /router.yaml" << EOF
115 supergraph:
116 listen: 127.0.0.1:${ ROUTER_PORT }
117 introspection: false
118 health_check:
119 listen: 127.0.0.1:${ ROUTER_HEALTH_PORT }
120 include_subgraph_errors:
121 all: true
122 EOF
123 env -u OTEL_EXPORTER_OTLP_ENDPOINT " $ROUTER_BIN " --supergraph " $OUT /supergraph.graphql" --config " $OUT /router.yaml" > " $OUT /router.log" 2>&1 &
124 ROUTER_PID = $!
125 # The same wait as e2e.sh: E2E_ROUTER_START_SECONDS raises it for a large
126 # supergraph (Databricks: 152.6s to healthy), the default stays 30.
127 router_wait = ${E2E_ROUTER_START_SECONDS :- 30}
128 deadline = $(( $(date +%s ) + router_wait ))
129 until curl -sf "http://localhost:${ ROUTER_HEALTH_PORT }/health" > /dev/null 2>&1 ; do
130 kill -0 " $ROUTER_PID " 2> /dev/null || { echo "live: router exited before it reported healthy" ; tail -30 " $OUT /router.log" >&2 ; exit 1 ; }
131 [ "$( date +%s)" -ge " $deadline " ] && { echo "live: router did not start within ${ router_wait }s (set E2E_ROUTER_START_SECONDS to wait longer)" ; tail -30 " $OUT /router.log" >&2 ; exit 1 ; }
132 sleep 1
133 done
134
135 # Cases, as JSON so bash never parses YAML.
136 " $RC " yaml2json " $LIVE " | jq -c '.cases // []' > " $OUT /cases.json"
137 count = "$( jq 'length' " $OUT /cases.json")"
138 echo "live: Apollo Router $ROUTER_VERSION against the real API, $count case(s)"
139
140 PASS = 0 ; FAIL = 0
141 for ((i = 0 ; i < count; i ++ )); do
142 name = "$( jq -r ".[ $i ].name" " $OUT /cases.json")"
143 safety = "$( jq -r ".[ $i ].safety // \" read \" " " $OUT /cases.json")"
144 require = "$( jq -r ".[ $i ].require // \"\" " " $OUT /cases.json")"
145 query_file = " $WORKSPACE /tests/live/ $name .graphql"
146 if [ ! -f " $query_file " ]; then echo "FAIL: $name — missing $query_file " ; FAIL = $(( FAIL+1 )); continue ; fi
147
148 # Chained variables: each value comes from an earlier case's saved response.
149 vars = '{}' ; chain_error = ""
150 while IFS = $' \t ' read -r vname vfrom vpath ; do
151 [ -n " $vname " ] || continue
152 src = " $OUT /responses/ $vfrom .json"
153 if [ ! -f " $src " ]; then chain_error = "variable \$ $vname needs case ' $vfrom ', which has not passed before this one" ; break ; fi
154 if ! value = "$( jq -c " $vpath " " $src " 2> /dev/null)" ; then chain_error = "variable \$ $vname : jq path ' $vpath ' is invalid" ; break ; fi
155 if [ -z " $value " ] || [ " $value " = "null" ]; then chain_error = "variable \$ $vname resolved to null from $vfrom ( $vpath )" ; break ; fi
156 vars = "$( jq -c --arg k " $vname " --argjson v " $value " '. + {($k): $v}' <<< " $vars ")"
157 done < <( jq -r ".[ $i ].variables // {} | to_entries[] | [.key, .value.from, .value.path] | @tsv" " $OUT /cases.json")
158 if [ -n " $chain_error " ]; then echo "FAIL: $name ( $safety ) — $chain_error " ; FAIL = $(( FAIL+1 )); continue ; fi
159
160 response = "$( jq -n --rawfile q " $query_file " --argjson v " $vars " '{query: $q, variables: $v}' \
161 | curl -sf -X POST -H 'content-type: application/json' -d @- "http://localhost:${ ROUTER_PORT }/" \
162 || echo '{"errors":[{"message":"curl error"}]}')"
163 errors = "$( echo " $response " | jq -r '.errors // empty | length' 2> /dev/null || echo parse)"
164 has_data = "$( echo " $response " | jq -r '(.data // empty) | if . == null or . == {} then "no" else "yes" end' 2> /dev/null || echo no)"
165 if [ -n " $errors " ] && [ " $errors " != "0" ]; then
166 echo "FAIL: $name ( $safety ) — $errors error(s):" ; echo " $response " | jq -c '.errors' | head -3 | sed 's/^/ /'
167 FAIL = $(( FAIL+1 )); continue
168 fi
169 if [ " $has_data " != "yes" ]; then echo "FAIL: $name ( $safety ) — empty data" ; FAIL = $(( FAIL+1 )); continue ; fi
170 if [ -n " $require " ] && [ " $require " != "null" ]; then
171 # Type-strict: the expression must yield exactly one boolean true.
172 verdict = "$( echo " $response " | jq -r "[ ( ${ require }
173 ) ] | if length == 1 and .[0] == true then \" true \" else \" false \" end" 2> /dev/null || echo jq-error)"
174 if [ " $verdict " != "true" ]; then
175 echo "FAIL: $name ( $safety ) — require did not evaluate to true ( $verdict ): $require "
176 echo " $response " | jq -c '.data' | cut -c1-400 | sed 's/^/ /'
177 FAIL = $(( FAIL+1 )); continue
178 fi
179 fi
180 mkdir -p " $OUT /responses" ; printf '%s' " $response " > " $OUT /responses/ $name .json"
181 echo "PASS: $name ( $safety )" ; PASS = $(( PASS+1 ))
182 done
183 # Exclusions: named, reasoned, reported.
184 " $RC " yaml2json " $LIVE " | jq -c '.exclusions // []' > " $OUT /exclusions.json"
185 excluded = "$( jq 'length' " $OUT /exclusions.json")"
186 EXCL = 0
187 for ((i = 0 ; i < excluded; i ++ )); do
188 op = "$( jq -r ".[ $i ].operation // \"\" " " $OUT /exclusions.json")"
189 field = "$( jq -r ".[ $i ].field // \"\" " " $OUT /exclusions.json")"
190 reason = "$( jq -r ".[ $i ].reason // \"\" " " $OUT /exclusions.json")"
191 if [ -z " $op " ] && [ -z " $field " ]; then echo "FAIL: exclusion $i names no operation or field" ; FAIL = $(( FAIL+1 )); continue ; fi
192 if [ -n " $op " ] && [ -n " $field " ]; then echo "FAIL: exclusion $i names both an operation ( $op ) and a field ( $field ); give each its own entry" ; FAIL = $(( FAIL+1 )); continue ; fi
193 if [ -z " $reason " ]; then echo "FAIL: exclusion for ${ op :- $field } gives no reason" ; FAIL = $(( FAIL+1 )); continue ; fi
194 # A field's line has its own prefix, so evidence cannot read it as an
195 # operation row (a relationship field has none).
196 if [ -n " $field " ]; then echo "EXCLUDED FIELD: $field — $reason " ; else echo "EXCLUDED: $op — $reason " ; fi
197 EXCL = $(( EXCL+1 ))
198 done
199 echo "live: $PASS passed, $FAIL failed, $EXCL excluded"
200 [ " $FAIL " -eq 0 ]