Setting the file. One moment. Train Sft Example · Huggingface LLM Trainer · huggingface/skills · Skills Docs14.10
Unsloth
(opens in a new tab)
scripts/train_sft_example.py
Python·122 lines·3 KB
- Trackio integration for real-time monitoring
18- LoRA/PEFT for efficient training
19- Proper Hub saving configuration
20- Train/eval split for monitoring
21- Checkpoint management
22- Optimized training parameters
23
24Usage with hf_jobs MCP tool:
25 hf_jobs("uv", {
26 "script": '''<paste this entire file>''',
27 "flavor": "a10g-large",
28 "timeout": "3h",
29 "secrets": {"HF_TOKEN": "$HF_TOKEN"},
30 })
31
32Or submit the script content directly inline without saving to a file.
33"""
34
35import trackio
36from datasets import load_dataset
37from peft import LoraConfig
38from trl import SFTTrainer, SFTConfig
39
40
41# Load dataset
42print("📦 Loading dataset...")
43dataset = load_dataset("trl-lib/Capybara", split="train")
44print(f"✅ Dataset loaded: {len(dataset)} examples")
45
46# Create train/eval split
47print("🔀 Creating train/eval split...")
48dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
49train_dataset = dataset_split["train"]
50eval_dataset = dataset_split["test"]
51print(f" Train: {len(train_dataset)} examples")
52print(f" Eval: {len(eval_dataset)} examples")
53
54# Note: For memory-constrained demos, skip eval by using full dataset as train_dataset
55# and removing eval_dataset, eval_strategy, and eval_steps from config below
56
57# Training configuration
58config = SFTConfig(
59 # CRITICAL: Hub settings
60 output_dir="qwen-capybara-sft",
61 push_to_hub=True,
62 hub_model_id="username/qwen-capybara-sft",
63 hub_strategy="every_save", # Push checkpoints
64
65 # Training parameters
66 num_train_epochs=3,
67 per_device_train_batch_size=4,
68 gradient_accumulation_steps=4,
69 learning_rate=2e-5,
70 # max_length=1024, # Default - only set if you need different sequence length
71
72 # Logging & checkpointing
73 logging_steps=10,
74 save_strategy="steps",
75 save_steps=100,
76 save_total_limit=2,
77
78 # Evaluation - IMPORTANT: Only enable if eval_dataset provided
79 eval_strategy="steps",
80 eval_steps=100,
81
82 # Optimization
83 warmup_ratio=0.1,
84 lr_scheduler_type="cosine",
85
86 # Monitoring
87 report_to="trackio", # Integrate with Trackio
88 project="meaningful_project_name", # project name for the training name (trackio)
89 run_name="baseline-run", #Descriptive name for this training run
90)
91
92# LoRA configuration
93peft_config = LoraConfig(
94 r=16,
95 lora_alpha=32,
96 lora_dropout=0.05,
97 bias="none",
98 task_type="CAUSAL_LM",
99 target_modules=["q_proj", "v_proj"],
100)
101
102# Initialize and train
103print("🎯 Initializing trainer...")
104trainer = SFTTrainer(
105 model="Qwen/Qwen2.5-0.5B",
106 train_dataset=train_dataset,
107 eval_dataset=eval_dataset, # CRITICAL: Must provide eval_dataset when eval_strategy is enabled
108 args=config,
109 peft_config=peft_config,
110)
111
112print("🚀 Starting training...")
113trainer.train()
114
115print("💾 Pushing to Hub...")
116trainer.push_to_hub()
117
118# Finish Trackio tracking
119trackio.finish()
120
121print("✅ Complete! Model at: https://huggingface.co/username/qwen-capybara-sft")
122print("📊 View metrics at: https://huggingface.co/spaces/username/trackio")