Setting the file. One moment.
Run · Remotion To Hyperframes · heygen-com/hyperframes · Skills Docs
Back to the top of the page assets/test-corpus/ run.sh
Shell · 249 lines · 8 KB
16 # - npm (for Remotion installs)
17 # - HF CLI built at packages/cli/dist/cli.js (run `bun run --filter @hyperframes/cli build`
18 # in the repo root if missing)
19 #
20 # Output:
21 # <fixture>/diff/summary.json per-fixture SSIM summary
22 # <fixture>/strip/strip.png per-fixture comparison strip (only on fail)
23 # ./run-report.json aggregate report
24
25 set -euo pipefail
26
27 THIS_DIR = "$( cd "$( dirname "${ BASH_SOURCE [0]}")" && pwd )"
28 SKILL_DIR = "$( cd " $THIS_DIR /../.." && pwd )"
29 REPO_ROOT = "$( cd " $SKILL_DIR /../.." && pwd )"
30
31 LINT = " $SKILL_DIR /scripts/lint_source.py"
32 DIFF = " $SKILL_DIR /scripts/render_diff.sh"
33 STRIP = " $SKILL_DIR /scripts/frame_strip.sh"
34 HF_CLI = " $REPO_ROOT /packages/cli/dist/cli.js"
35 REPORT = " $THIS_DIR /run-report.json"
36
37 # Per-fixture results land here as one JSON file each, then the aggregator
38 # globs them. This is safer than building JSON via bash string concatenation
39 # (a fixture name containing a quote would break the previous approach).
40 RESULTS_DIR = "$( mktemp -d )"
41 trap 'rm -rf "$RESULTS_DIR"' EXIT
42
43 # T4 is lint-only — no ffmpeg or HF CLI needed. Defer the render-tier
44 # toolchain checks until run_render_tier() actually runs, so
45 # `./run.sh tier-4-escape-hatch` works on a clean checkout.
46 require_render_tier_tools () {
47 if [[ ! -f " $HF_CLI " ]]; then
48 echo "error: HF CLI not built at $HF_CLI " >&2
49 echo " Run 'bun run --filter @hyperframes/cli build' in $REPO_ROOT " >&2
50 return 2
51 fi
52 if ! command -v ffmpeg > /dev/null 2>&1 ; then
53 echo "error: ffmpeg not on PATH" >&2
54 return 2
55 fi
56 return 0
57 }
58
59 # Write one fixture's result as a JSON file. Values are passed via argv so
60 # bash string interpolation can't corrupt the JSON or inject Python source.
61 write_result () {
62 local fixture_name = " $1 "
63 local status = " $2 "
64 shift 2
65 python3 - " $RESULTS_DIR / $fixture_name .json" " $fixture_name " " $status " " $@ " << 'PY'
66 import json
67 import sys
68
69 out_path, fixture_name, status, *kvs = sys.argv[1:]
70 result = {"fixture": fixture_name, "status": status}
71 for i in range(0, len(kvs), 2):
72 k, v = kvs[i], kvs[i + 1]
73 try:
74 result[k] = float(v) if "." in v or v.lstrip("-").isdigit() else v
75 except ValueError:
76 result[k] = v
77 with open(out_path, "w") as f:
78 json.dump(result, f)
79 PY
80 }
81
82 # Read a top-level scalar value from a JSON file. Falls back to $3 if the
83 # key is missing (used to default composition_id for older fixtures).
84 read_json_value () {
85 local file = " $1 "
86 local key = " $2 "
87 local default = " ${3 :- } "
88 python3 - " $file " " $key " " $default " << 'PY'
89 import json
90 import sys
91
92 path, key, default = sys.argv[1], sys.argv[2], sys.argv[3]
93 with open(path) as f:
94 data = json.load(f)
95 val = data.get(key, default)
96 print(val if val is not None else "")
97 PY
98 }
99
100 run_render_tier () {
101 local fixture_dir = " $1 "
102 local fixture_name
103 fixture_name = $( basename " $fixture_dir " )
104 local expected = " $fixture_dir /expected.json"
105
106 if ! require_render_tier_tools ; then
107 echo " ⚠ $fixture_name : render toolchain unavailable, skipping"
108 write_result " $fixture_name " "skipped" reason "render toolchain unavailable"
109 return 0
110 fi
111
112 local threshold composition_id
113 threshold = $( read_json_value " $expected " "ssim_threshold" )
114 composition_id = $( read_json_value " $expected " "composition_id" "Composition" )
115
116 echo " ▶ $fixture_name (threshold $threshold , composition $composition_id )"
117
118 if [[ -x " $fixture_dir /setup.sh" ]]; then
119 " $fixture_dir /setup.sh" > /dev/null
120 fi
121
122 if ! python3 " $LINT " " $fixture_dir /remotion-src/src/" > /dev/null ; then
123 echo " ✗ lint failed (blockers in Remotion source)"
124 write_result " $fixture_name " "fail" stage "lint"
125 return 0
126 fi
127
128 if [[ ! -d " $fixture_dir /remotion-src/node_modules" ]]; then
129 echo " ⏳ npm install (first run)"
130 ( cd " $fixture_dir /remotion-src" && npm install --silent --no-progress > /dev/null 2>&1 )
131 fi
132
133 echo " ⏳ render Remotion baseline"
134 if ! ( cd " $fixture_dir /remotion-src" && \
135 npx --no-install remotion render " $composition_id " out/baseline.mp4 > /dev/null 2>&1 ); then
136 echo " ✗ Remotion render failed"
137 write_result " $fixture_name " "fail" stage "remotion-render"
138 return 0
139 fi
140
141 echo " ⏳ render HF translation"
142 if ! ( cd " $fixture_dir " && \
143 node " $HF_CLI " render hf-src/ --output hf.mp4 --quiet > /dev/null 2>&1 ); then
144 echo " ✗ HF render failed"
145 write_result " $fixture_name " "fail" stage "hf-render"
146 return 0
147 fi
148
149 if R2HF_SSIM_THRESHOLD = " $threshold " " $DIFF " \
150 " $fixture_dir /remotion-src/out/baseline.mp4" \
151 " $fixture_dir /hf.mp4" \
152 " $fixture_dir /diff" > /dev/null ; then
153 local mean
154 mean = $( read_json_value " $fixture_dir /diff/summary.json" "mean" )
155 echo " ✓ pass (mean SSIM $mean , threshold $threshold )"
156 write_result " $fixture_name " "pass" mean_ssim " $mean " threshold " $threshold "
157 else
158 local mean
159 mean = $( read_json_value " $fixture_dir /diff/summary.json" "mean" )
160 echo " ✗ fail (mean SSIM $mean , threshold $threshold )"
161 " $STRIP " \
162 " $fixture_dir /remotion-src/out/baseline.mp4" \
163 " $fixture_dir /hf.mp4" \
164 " $fixture_dir /strip" 8 > /dev/null
165 write_result " $fixture_name " "fail" stage "ssim" mean_ssim " $mean " threshold " $threshold "
166 fi
167 }
168
169 run_lint_tier () {
170 local fixture_dir = " $1 "
171 local fixture_name
172 fixture_name = $( basename " $fixture_dir " )
173
174 echo " ▶ $fixture_name (lint-only)"
175 if " $fixture_dir /validate.sh" > /dev/null 2>&1 ; then
176 echo " ✓ pass (8/8 cases)"
177 write_result " $fixture_name " "pass" mode "lint"
178 else
179 echo " ✗ fail (some cases mismatched expected.json)"
180 write_result " $fixture_name " "fail" mode "lint"
181 fi
182 }
183
184 echo "remotion-to-hyperframes corpus run"
185 echo "=================================="
186
187 for tier in tier-1-title-card tier-2-multi-scene tier-3-data-driven ; do
188 if [[ -n " ${1 :- } " && " $1 " != " $tier " ]]; then
189 continue
190 fi
191 if [[ -d " $THIS_DIR / $tier " ]]; then
192 run_render_tier " $THIS_DIR / $tier "
193 fi
194 done
195
196 if [[ -z " ${1 :- } " || " $1 " == "tier-4-escape-hatch" ]]; then
197 if [[ -d " $THIS_DIR /tier-4-escape-hatch" ]]; then
198 run_lint_tier " $THIS_DIR /tier-4-escape-hatch"
199 fi
200 fi
201
202 # Aggregate the per-fixture JSON files into one report.
203 #
204 # Skipped fixtures are *not* a pass — they mean a tier didn't run because
205 # tooling or fixtures were unavailable. The orchestrator exits non-zero on
206 # any skip so a clean checkout that lacks the HF CLI doesn't accidentally
207 # report "passed 1/4" (T4 alone) and look like the corpus is healthy.
208 #
209 # Single-tier mode (`./run.sh tier-N`) only writes a result file for the
210 # selected tier; tiers that weren't run aren't counted as skips.
211 python3 - " $RESULTS_DIR " " $REPORT " << 'PY'
212 import json
213 import sys
214 from pathlib import Path
215
216 results_dir, out_path = Path(sys.argv[1]), Path(sys.argv[2])
217 results = sorted(
218 (json.loads(p.read_text()) for p in results_dir.glob("*.json")),
219 key=lambda r: r["fixture"],
220 )
221
222 total = len(results)
223 passed = sum(1 for r in results if r["status"] == "pass")
224 failed = sum(1 for r in results if r["status"] == "fail")
225 skipped = sum(1 for r in results if r["status"] == "skipped")
226 report = {
227 "total": total,
228 "passed": passed,
229 "failed": failed,
230 "skipped": skipped,
231 "results": results,
232 }
233 out_path.write_text(json.dumps(report, indent=2))
234
235 print()
236 print("=" * 50)
237 print(f" passed {passed}/{total}, failed {failed}, skipped {skipped}")
238 print(f" report → {out_path}")
239 if skipped > 0:
240 skipped_fixtures = [r["fixture"] for r in results if r["status"] == "skipped"]
241 skipped_reasons = sorted({r.get("reason", "unknown") for r in results if r["status"] == "skipped"})
242 print()
243 print(f" ⚠ {skipped} skipped: {', '.join(skipped_fixtures)}")
244 for reason in skipped_reasons:
245 print(f" reason: {reason}")
246 print(" Skipped fixtures count as failures for the aggregate.")
247 print("=" * 50)
248 sys.exit(0 if failed == 0 and skipped == 0 else 1)
249 PY