Alerts

trackio.alert(
    title="Loss divergence",                    # Short title (required)
    text="Loss 5.2 still high after 200 steps", # Detailed description (optional)
    level=trackio.AlertLevel.WARN,               # INFO, WARN, or ERROR (default: WARN)
    webhook_url="https://hooks.slack.com/...",   # Per-alert webhook override (optional)
)

trackio.init(
    project="my-project",
    webhook_url="https://hooks.slack.com/services/...",
    webhook_min_level=trackio.AlertLevel.WARN,  # Only send WARN+ to webhook
)

trackio.alert(
    title="Critical failure",
    level=trackio.AlertLevel.ERROR,
    webhook_url="https://hooks.slack.com/services/...",  # Overrides global URL
)

# List all alerts for a project
trackio list alerts --project my-project --json

# Filter by run or level
trackio list alerts --project my-project --run my-run --level error --json

# Poll for new alerts since a timestamp (efficient for agents)
trackio list alerts --project my-project --json --since "2025-06-01T12:00:00"

{
  "project": "my-project",
  "run": null,
  "level": null,
  "since": "2025-06-01T12:00:00",
  "alerts": [
    {
      "run": "run-name",
      "title": "Loss divergence",
      "text": "Loss 5.2 still high after 200 steps",
      "level": "warn",
      "step": 200,
      "timestamp": "2025-06-01T12:05:30"
    }
  ]
}

import trackio

trackio.init(project="hyperparam-sweep", config={"lr": lr, "batch_size": bs})

for step in range(num_steps):
    loss = train_step()
    trackio.log({"loss": loss, "step": step})

    if step > 200 and loss > 5.0:
        trackio.alert(
            title="Loss divergence",
            text=f"Loss {loss:.4f} still above 5.0 after {step} steps — learning rate may be too high",
            level=trackio.AlertLevel.ERROR,
        )

    if step > 500 and loss_delta < 0.001:
        trackio.alert(
            title="Training stall",
            text=f"Loss barely changed over last 100 steps (delta={loss_delta:.6f})",
            level=trackio.AlertLevel.WARN,
        )

    if math.isnan(loss):
        trackio.alert(
            title="NaN loss",
            text="Loss became NaN — training is broken",
            level=trackio.AlertLevel.ERROR,
        )
        break

trackio.finish()

# Poll for alerts (run periodically)
trackio list alerts --project hyperparam-sweep --json --since "2025-06-01T00:00:00"

# Alert fired at step 200 — get all metrics in a ±5 step window
trackio get snapshot --project hyperparam-sweep --run run-1 --around 200 --window 5 --json

# Or inspect a single metric around the alert's timestamp
trackio get metric --project hyperparam-sweep --run run-1 --metric loss --around 200 --window 10 --json

# Check metrics from previous runs
trackio get run --project hyperparam-sweep --run run-1 --json
trackio get metric --project hyperparam-sweep --run run-1 --metric loss --json

# Launch new run with adjusted config
python train.py --lr 5e-5

from transformers import TrainerCallback

class AlertCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        if "trackio" not in args.report_to:
            return
        if logs and "loss" in logs:
            if logs["loss"] > 5.0 and state.global_step > 100:
                trackio.alert(
                    title="High loss",
                    text=f"Loss {logs['loss']:.4f} at step {state.global_step}",
                    level=trackio.AlertLevel.ERROR,
                )

trainer = SFTTrainer(
    model=model,
    args=SFTConfig(output_dir="./out", report_to="trackio"),
    callbacks=[AlertCallback()],
    ...
)