Setting the file. One moment.
Lighteval Vllm Uv · Huggingface Community Evals · huggingface/skills · Skills Docs
Back to the top of the page scripts/lighteval_vllm_uv.py
scripts/ lighteval_vllm_uv.py
Python · 297 lines · 9 KB
17
18 Usage (standalone):
19 uv run scripts/lighteval_vllm_uv.py --model "meta-llama/Llama-3.2-1B" --tasks "leaderboard|mmlu|5"
20
21 """
22
23 from __future__ import annotations
24
25 import argparse
26 import os
27 import subprocess
28 import sys
29 from typing import Optional
30
31
32 def setup_environment () -> None :
33 """Configure environment variables for HuggingFace authentication."""
34 hf_token = os.getenv( "HF_TOKEN" )
35 if hf_token:
36 os.environ.setdefault( "HUGGING_FACE_HUB_TOKEN" , hf_token)
37 os.environ.setdefault( "HF_HUB_TOKEN" , hf_token)
38
39
40 def run_lighteval_vllm (
41 model_id: str ,
42 tasks: str ,
43 output_dir: Optional[ str ] = None ,
44 max_samples: Optional[ int ] = None ,
45 batch_size: int = 1 ,
46 tensor_parallel_size: int = 1 ,
47 gpu_memory_utilization: float = 0.8 ,
48 dtype: str = "auto" ,
49 trust_remote_code: bool = False ,
50 use_chat_template: bool = False ,
51 system_prompt: Optional[ str ] = None ,
52 ) -> None :
53 """
54 Run lighteval with vLLM backend for efficient GPU inference.
55
56 Args:
57 model_id: HuggingFace model ID (e.g., "meta-llama/Llama-3.2-1B")
58 tasks: Task specification (e.g., "leaderboard|mmlu|5" or "lighteval|hellaswag|0")
59 output_dir: Directory for evaluation results
60 max_samples: Limit number of samples per task
61 batch_size: Batch size for evaluation
62 tensor_parallel_size: Number of GPUs for tensor parallelism
63 gpu_memory_utilization: GPU memory fraction to use (0.0-1.0)
64 dtype: Data type for model weights (auto, float16, bfloat16)
65 trust_remote_code: Allow executing remote code from model repo
66 use_chat_template: Apply chat template for conversational models
67 system_prompt: System prompt for chat models
68 """
69 setup_environment()
70
71 # Build lighteval vllm command
72 cmd = [
73 "lighteval" ,
74 "vllm" ,
75 model_id,
76 tasks,
77 "--batch-size" , str (batch_size),
78 "--tensor-parallel-size" , str (tensor_parallel_size),
79 "--gpu-memory-utilization" , str (gpu_memory_utilization),
80 "--dtype" , dtype,
81 ]
82
83 if output_dir:
84 cmd.extend([ "--output-dir" , output_dir])
85
86 if max_samples:
87 cmd.extend([ "--max-samples" , str (max_samples)])
88
89 if trust_remote_code:
90 cmd.append( "--trust-remote-code" )
91
92 if use_chat_template:
93 cmd.append( "--use-chat-template" )
94
95 if system_prompt:
96 cmd.extend([ "--system-prompt" , system_prompt])
97
98 print ( f "Running: { ' ' .join(cmd) } " )
99
100 try :
101 subprocess.run(cmd, check = True )
102 print ( "Evaluation complete." )
103 except subprocess.CalledProcessError as exc:
104 print ( f "Evaluation failed with exit code { exc.returncode } " , file = sys.stderr)
105 sys.exit(exc.returncode)
106
107
108 def run_lighteval_accelerate (
109 model_id: str ,
110 tasks: str ,
111 output_dir: Optional[ str ] = None ,
112 max_samples: Optional[ int ] = None ,
113 batch_size: int = 1 ,
114 dtype: str = "bfloat16" ,
115 trust_remote_code: bool = False ,
116 use_chat_template: bool = False ,
117 system_prompt: Optional[ str ] = None ,
118 ) -> None :
119 """
120 Run lighteval with accelerate backend for multi-GPU distributed inference.
121
122 Use this backend when vLLM is not available or for models not supported by vLLM.
123
124 Args:
125 model_id: HuggingFace model ID
126 tasks: Task specification
127 output_dir: Directory for evaluation results
128 max_samples: Limit number of samples per task
129 batch_size: Batch size for evaluation
130 dtype: Data type for model weights
131 trust_remote_code: Allow executing remote code
132 use_chat_template: Apply chat template
133 system_prompt: System prompt for chat models
134 """
135 setup_environment()
136
137 # Build lighteval accelerate command
138 cmd = [
139 "lighteval" ,
140 "accelerate" ,
141 model_id,
142 tasks,
143 "--batch-size" , str (batch_size),
144 "--dtype" , dtype,
145 ]
146
147 if output_dir:
148 cmd.extend([ "--output-dir" , output_dir])
149
150 if max_samples:
151 cmd.extend([ "--max-samples" , str (max_samples)])
152
153 if trust_remote_code:
154 cmd.append( "--trust-remote-code" )
155
156 if use_chat_template:
157 cmd.append( "--use-chat-template" )
158
159 if system_prompt:
160 cmd.extend([ "--system-prompt" , system_prompt])
161
162 print ( f "Running: { ' ' .join(cmd) } " )
163
164 try :
165 subprocess.run(cmd, check = True )
166 print ( "Evaluation complete." )
167 except subprocess.CalledProcessError as exc:
168 print ( f "Evaluation failed with exit code { exc.returncode } " , file = sys.stderr)
169 sys.exit(exc.returncode)
170
171
172 def main () -> None :
173 parser = argparse.ArgumentParser(
174 description = "Run lighteval evaluations with vLLM or accelerate backend on custom HuggingFace models" ,
175 formatter_class = argparse.RawDescriptionHelpFormatter,
176 epilog = """
177 Examples:
178 # Run MMLU evaluation with vLLM
179 uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5"
180
181 # Run with accelerate backend instead of vLLM
182 uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --backend accelerate
183
184 # Run with chat template for instruction-tuned models
185 uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B-Instruct --tasks "leaderboard|mmlu|5" --use-chat-template
186
187 # Run with limited samples for testing
188 uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --max-samples 10
189
190 Task format:
191 Tasks use the format: "suite|task|num_fewshot"
192 - leaderboard|mmlu|5 (MMLU with 5-shot)
193 - lighteval|hellaswag|0 (HellaSwag zero-shot)
194 - leaderboard|gsm8k|5 (GSM8K with 5-shot)
195 - Multiple tasks: "leaderboard|mmlu|5,leaderboard|gsm8k|5"
196 """ ,
197 )
198
199 parser.add_argument(
200 "--model" ,
201 required = True ,
202 help = "HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)" ,
203 )
204 parser.add_argument(
205 "--tasks" ,
206 required = True ,
207 help = "Task specification (e.g., 'leaderboard|mmlu|5')" ,
208 )
209 parser.add_argument(
210 "--backend" ,
211 choices = [ "vllm" , "accelerate" ],
212 default = "vllm" ,
213 help = "Inference backend to use (default: vllm)" ,
214 )
215 parser.add_argument(
216 "--output-dir" ,
217 default = None ,
218 help = "Directory for evaluation results" ,
219 )
220 parser.add_argument(
221 "--max-samples" ,
222 type = int ,
223 default = None ,
224 help = "Limit number of samples per task (useful for testing)" ,
225 )
226 parser.add_argument(
227 "--batch-size" ,
228 type = int ,
229 default = 1 ,
230 help = "Batch size for evaluation (default: 1)" ,
231 )
232 parser.add_argument(
233 "--tensor-parallel-size" ,
234 type = int ,
235 default = 1 ,
236 help = "Number of GPUs for tensor parallelism (vLLM only, default: 1)" ,
237 )
238 parser.add_argument(
239 "--gpu-memory-utilization" ,
240 type = float ,
241 default = 0.8 ,
242 help = "GPU memory fraction to use (vLLM only, default: 0.8)" ,
243 )
244 parser.add_argument(
245 "--dtype" ,
246 default = "auto" ,
247 choices = [ "auto" , "float16" , "bfloat16" , "float32" ],
248 help = "Data type for model weights (default: auto)" ,
249 )
250 parser.add_argument(
251 "--trust-remote-code" ,
252 action = "store_true" ,
253 help = "Allow executing remote code from model repository" ,
254 )
255 parser.add_argument(
256 "--use-chat-template" ,
257 action = "store_true" ,
258 help = "Apply chat template for instruction-tuned/chat models" ,
259 )
260 parser.add_argument(
261 "--system-prompt" ,
262 default = None ,
263 help = "System prompt for chat models" ,
264 )
265
266 args = parser.parse_args()
267
268 if args.backend == "vllm" :
269 run_lighteval_vllm(
270 model_id = args.model,
271 tasks = args.tasks,
272 output_dir = args.output_dir,
273 max_samples = args.max_samples,
274 batch_size = args.batch_size,
275 tensor_parallel_size = args.tensor_parallel_size,
276 gpu_memory_utilization = args.gpu_memory_utilization,
277 dtype = args.dtype,
278 trust_remote_code = args.trust_remote_code,
279 use_chat_template = args.use_chat_template,
280 system_prompt = args.system_prompt,
281 )
282 else :
283 run_lighteval_accelerate(
284 model_id = args.model,
285 tasks = args.tasks,
286 output_dir = args.output_dir,
287 max_samples = args.max_samples,
288 batch_size = args.batch_size,
289 dtype = args.dtype if args.dtype != "auto" else "bfloat16" ,
290 trust_remote_code = args.trust_remote_code,
291 use_chat_template = args.use_chat_template,
292 system_prompt = args.system_prompt,
293 )
294
295
296 if __name__ == "__main__" :
297 main()