Setting the file. One moment.
Extract Audio Data · Hyperframes Creative · heygen-com/hyperframes · Skills Docs
Back to the top of the page 22.10
Narration
scripts/ extract-audio-data.py
Python · 199 lines · 7 KB
17
18 import argparse
19 import json
20 import subprocess
21 import sys
22
23 import numpy as np
24
25 # Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
26 # every platform; say so rather than depending on the console's code page. Carry
27 # `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
28 # stderr "backslashreplace" so the diagnostic path can never itself raise.
29 for _stream in (sys.stdout, sys.stderr):
30 if hasattr (_stream, "reconfigure" ):
31 _stream.reconfigure( encoding = "utf-8" , errors = _stream.errors)
32
33 # ---------------------------------------------------------------------------
34 # FFT parameters
35 #
36 # A 4096-sample window gives ~10.8 Hz per bin at 44100Hz — enough to resolve
37 # low-frequency bands cleanly. The per-frame audio slice (44100/30 = 1470
38 # samples at 30fps) is too small and causes low bands to map to the same bins.
39 #
40 # Frequency range 30Hz–16kHz covers the useful range for music. Below 30Hz is
41 # sub-bass most speakers can't reproduce; above 16kHz is noise/harmonics that
42 # don't contribute to perceived rhythm or melody.
43 # ---------------------------------------------------------------------------
44
45 SAMPLE_RATE = 44100
46 FFT_SIZE = 4096
47 MIN_FREQ = 30.0
48 MAX_FREQ = 16000.0
49
50
51 def decode_audio (path: str ) -> np.ndarray:
52 """Decode audio to mono float32 samples via ffmpeg."""
53 cmd = [
54 "ffmpeg" , "-i" , path,
55 "-vn" , "-ac" , "1" , "-ar" , str ( SAMPLE_RATE ),
56 "-f" , "s16le" , "-acodec" , "pcm_s16le" ,
57 "-loglevel" , "error" ,
58 "pipe:1" ,
59 ]
60 result = subprocess.run(cmd, capture_output = True )
61 if result.returncode != 0 :
62 # Decoding ffmpeg's diagnostics strictly makes the reporter the thing that
63 # crashes: a Windows ffmpeg emits cp1252 bytes, and UnicodeDecodeError here
64 # would bury the actual failure it was trying to report.
65 print ( f "ffmpeg error: { result.stderr.decode( 'utf-8' , errors = 'replace' ) } " , file = sys.stderr)
66 sys.exit( 1 )
67 return np.frombuffer(result.stdout, dtype = np.int16).astype(np.float32) / 32768.0
68
69
70 def compute_band_edges (n_bands: int ) -> np.ndarray:
71 """Logarithmically-spaced frequency band edges from MIN_FREQ to MAX_FREQ."""
72 return np.array([
73 MIN_FREQ * ( MAX_FREQ / MIN_FREQ ) ** (i / n_bands)
74 for i in range (n_bands + 1 )
75 ])
76
77
78 def compute_fft_bands (
79 windowed: np.ndarray, freq_per_bin: float , n_bins: int ,
80 band_edges: np.ndarray, n_bands: int ,
81 ) -> np.ndarray:
82 """Compute peak magnitude in logarithmically-spaced frequency bands."""
83 magnitudes = np.abs(np.fft.rfft(windowed))
84
85 bands = np.zeros(n_bands)
86 for b in range (n_bands):
87 low_bin = max ( 0 , int (band_edges[b] / freq_per_bin))
88 high_bin = min (n_bins, int (band_edges[b + 1 ] / freq_per_bin))
89 if high_bin <= low_bin:
90 high_bin = low_bin + 1
91 # Clamp to valid range to avoid empty slices
92 low_bin = min (low_bin, n_bins - 1 )
93 high_bin = min (high_bin, n_bins)
94 bands[b] = np.max(magnitudes[low_bin:high_bin])
95
96 return bands
97
98
99 def extract (path: str , fps: int , n_bands: int ) -> dict :
100 """Extract per-frame audio data."""
101 print ( f "Decoding audio from { path } ..." , file = sys.stderr)
102 samples = decode_audio(path)
103 duration = len (samples) / SAMPLE_RATE
104 frame_step = SAMPLE_RATE // fps
105 total_frames = int (duration * fps)
106
107 print ( f "Duration: { duration :.1f} s, { total_frames } frames at { fps } fps" , file = sys.stderr)
108 print ( f "FFT window: { FFT_SIZE } samples ( { SAMPLE_RATE / FFT_SIZE :.1f} Hz/bin)" , file = sys.stderr)
109 print ( f "Frequency range: { MIN_FREQ :.0f} - { MAX_FREQ :.0f} Hz, { n_bands } bands" , file = sys.stderr)
110
111 # Precompute constants
112 hann = np.hanning( FFT_SIZE )
113 band_edges = compute_band_edges(n_bands)
114 freq_per_bin = SAMPLE_RATE / FFT_SIZE
115 n_bins = FFT_SIZE // 2 + 1
116 half_fft = FFT_SIZE // 2
117
118 # Pass 1: extract raw values
119 rms_values = np.zeros(total_frames)
120 band_values = np.zeros((total_frames, n_bands))
121
122 for f in range (total_frames):
123 # RMS from the frame's audio slice
124 rms_start = f * frame_step
125 rms_end = rms_start + frame_step
126 frame_slice = samples[rms_start: min (rms_end, len (samples))]
127 if len (frame_slice) > 0 :
128 rms_values[f] = np.sqrt(np.mean(frame_slice ** 2 ))
129
130 # FFT from a centered 4096-sample window
131 center = rms_start + frame_step // 2
132 win_start = center - half_fft
133 win_end = center + half_fft
134
135 if win_start >= 0 and win_end <= len (samples):
136 window = samples[win_start:win_end] * hann
137 else :
138 # Zero-pad at edges
139 padded = np.zeros( FFT_SIZE )
140 src_start = max ( 0 , win_start)
141 src_end = min ( len (samples), win_end)
142 dst_start = src_start - win_start
143 dst_end = dst_start + (src_end - src_start)
144 padded[dst_start:dst_end] = samples[src_start:src_end]
145 window = padded * hann
146
147 band_values[f] = compute_fft_bands(window, freq_per_bin, n_bins, band_edges, n_bands)
148
149 # Pass 2: normalize
150 peak_rms = rms_values.max() if total_frames > 0 else 1.0
151 if peak_rms > 0 :
152 rms_values /= peak_rms
153
154 # Per-band normalization so treble is visible alongside louder bass
155 band_peaks = band_values.max( axis = 0 )
156 band_peaks[band_peaks == 0 ] = 1.0
157 band_values /= band_peaks
158
159 # Build output
160 frames = []
161 for f in range (total_frames):
162 frames.append({
163 "time" : round (f / fps, 4 ),
164 "rms" : round ( float (rms_values[f]), 4 ),
165 "bands" : [ round ( float (b), 4 ) for b in band_values[f]],
166 })
167
168 return {
169 "duration" : round (duration, 4 ),
170 "fps" : fps,
171 "bands" : n_bands,
172 "totalFrames" : total_frames,
173 "frames" : frames,
174 }
175
176
177 def main ():
178 parser = argparse.ArgumentParser( description = "Extract per-frame audio visualization data" )
179 parser.add_argument( "input" , help = "Audio or video file" )
180 parser.add_argument( "-o" , "--output" , default = "audio-data.json" , help = "Output JSON path" )
181 parser.add_argument( "--fps" , type = int , default = 30 , help = "Frames per second (default: 30)" )
182 parser.add_argument( "--bands" , type = int , default = 16 , help = "Number of frequency bands (default: 16)" )
183 args = parser.parse_args()
184
185 if args.fps < 1 :
186 parser.error( "--fps must be at least 1" )
187 if args.bands < 1 :
188 parser.error( "--bands must be at least 1" )
189
190 data = extract(args.input, args.fps, args.bands)
191
192 with open (args.output, "w" , encoding = "utf-8" ) as f:
193 json.dump(data, f)
194
195 print ( f "Wrote { args.output } ( { data[ 'totalFrames' ] } frames, { data[ 'bands' ] } bands)" , file = sys.stderr)
196
197
198 if __name__ == "__main__" :
199 main()