┌─────────────────────────────────────┐
│  1. Agent writes training script    │
│     with trackio.init(), log(),     │
│     and alert() calls               │
├─────────────────────────────────────┤
│  2. Agent launches the script       │
├─────────────────────────────────────┤
│  3. Agent polls for alerts and      │
│     metrics while training runs     │
├─────────────────────────────────────┤
│  4. Agent reads results, decides    │
│     next hyperparameters            │
├─────────────────────────────────────┤
│  5. Agent writes a new script       │
│     and repeats from step 1         │
└─────────────────────────────────────┘

import trackio
import torch

trackio.init(
    project="cifar10-hparam-search",
    name="run-lr0.001-bs64",
    config={
        "learning_rate": 0.001,
        "batch_size": 64,
        "optimizer": "adamw",
        "epochs": 50,
        "weight_decay": 1e-4,
    },
)

model = build_model()
optimizer = torch.optim.AdamW(
    model.parameters(), lr=0.001, weight_decay=1e-4
)

best_val_loss = float("inf")
patience_counter = 0
patience_limit = 5

for epoch in range(50):
    train_loss = train_one_epoch(model, train_loader, optimizer)
    val_loss, val_acc = evaluate(model, val_loader)

    trackio.log({
        "train_loss": train_loss,
        "val_loss": val_loss,
        "val_accuracy": val_acc,
        "learning_rate": optimizer.param_groups[0]["lr"],
    })

    # Alert on loss divergence
    if train_loss > 10.0:
        trackio.alert(
            title="Training diverged",
            text=f"train_loss={train_loss:.4f} at epoch {epoch}. "
                 "Learning rate is likely too high.",
            level=trackio.AlertLevel.ERROR,
        )
        break

    # Alert when validation stops improving
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        patience_counter = 0
        torch.save(model.state_dict(), "best_model.pt")
    else:
        patience_counter += 1
        if patience_counter >= patience_limit:
            trackio.alert(
                title="Early stopping triggered",
                text=f"No improvement for {patience_limit} epochs. "
                     f"Best val_loss={best_val_loss:.4f} at epoch "
                     f"{epoch - patience_limit}.",
                level=trackio.AlertLevel.WARN,
            )
            break

    # Alert on overfitting
    if epoch > 10 and (train_loss < 0.1 and val_loss > 1.5):
        trackio.alert(
            title="Overfitting detected",
            text=f"train_loss={train_loss:.4f} vs val_loss={val_loss:.4f}. "
                 "Consider adding regularization or reducing model capacity.",
            level=trackio.AlertLevel.WARN,
        )

    # Alert on good results
    if val_acc > 0.93:
        trackio.alert(
            title="High accuracy reached",
            text=f"val_accuracy={val_acc:.4f} at epoch {epoch}.",
            level=trackio.AlertLevel.INFO,
        )

trackio.alert(
    title="Training complete",
    text=f"Final val_loss={val_loss:.4f}, val_accuracy={val_acc:.4f}. "
         f"Best val_loss={best_val_loss:.4f}.",
    level=trackio.AlertLevel.INFO,
)
trackio.finish()

python train.py

# First poll — get all alerts
trackio get alerts --project "cifar10-hparam-search" --run "run-lr0.001-bs64" --json

# Subsequent polls — only new alerts
trackio get alerts --project "cifar10-hparam-search" --since "2025-06-15T14:30:00" --json

import trackio
from datetime import datetime, timezone

api = trackio.Api()
last_check = None

while training_is_running():
    alerts = api.alerts(
        "cifar10-hparam-search",
        run="run-lr0.001-bs64",
        since=last_check,
    )
    if alerts:
        last_check = alerts[0]["timestamp"]  # alerts are sorted newest-first
        for a in alerts:
            print(f"[{a['level'].upper()}] {a['title']}: {a['text']}")

            if a["level"] == "error":
                # Stop the run and re-plan
                handle_error(a)

    time.sleep(30)

# Get run summary (metrics, config, step count)
trackio get run --project "cifar10-hparam-search" --run "run-lr0.001-bs64" --json

# Get the loss curve
trackio get metric --project "cifar10-hparam-search" --run "run-lr0.001-bs64" --metric "val_loss" --json

# Get all alerts (the most useful signal for decision-making)
trackio get alerts --project "cifar10-hparam-search" --run "run-lr0.001-bs64" --json

import json
import subprocess
import trackio

api = trackio.Api()

# Read alerts from the last run
alerts = api.alerts("cifar10-hparam-search", run="run-lr0.001-bs64")

# Read the config from the last run
runs = api.runs("cifar10-hparam-search")
last_run = [r for r in runs if r.name == "run-lr0.001-bs64"][0]
config = last_run.config

# Decision logic based on alerts
new_config = dict(config)
for alert in alerts:
    if "diverged" in alert["title"].lower():
        new_config["learning_rate"] = config["learning_rate"] * 0.1
    elif "overfitting" in alert["title"].lower():
        new_config["weight_decay"] = config.get("weight_decay", 1e-4) * 10
    elif "early stopping" in alert["title"].lower():
        new_config["learning_rate"] = config["learning_rate"] * 0.5
    elif "high accuracy" in alert["title"].lower():
        # This config is good — try fine-tuning
        new_config["learning_rate"] = config["learning_rate"] * 0.5
        new_config["epochs"] = config["epochs"] + 20

# List all runs
trackio list runs --project "cifar10-hparam-search" --json

# Get a summary of each run
trackio get run --project "cifar10-hparam-search" --run "run-lr0.001-bs64" --json
trackio get run --project "cifar10-hparam-search" --run "run-lr0.0001-bs64" --json

api = trackio.Api()
runs = api.runs("cifar10-hparam-search")

for run in runs:
    alerts = run.alerts()
    errors = [a for a in alerts if a["level"] == "error"]
    warns = [a for a in alerts if a["level"] == "warn"]
    infos = [a for a in alerts if a["level"] == "info"]
    print(f"{run.name}: {len(errors)} errors, {len(warns)} warnings, {len(infos)} info")

trackio.init(
    project="cifar10-hparam-search",
    webhook_url="https://hooks.slack.com/services/T.../B.../xxx",
)

import trackio
import random
import time

def run_experiment(config):
    """Simulate a training run. Replace with real training code."""
    run = trackio.init(
        project="auto-hparam-search",
        config=config,
    )

    lr = config["learning_rate"]
    best_val_loss = float("inf")
    patience = 0

    for epoch in range(config["epochs"]):
        # Simulated metrics — replace with real training
        noise = random.gauss(0, lr * 10)
        train_loss = max(0.01, 1.0 / (epoch + 1) + noise)
        val_loss = train_loss + random.uniform(0, 0.3)
        val_acc = max(0, min(1, 1.0 - val_loss + random.uniform(-0.05, 0.05)))

        trackio.log({
            "train_loss": train_loss,
            "val_loss": val_loss,
            "val_accuracy": val_acc,
        })

        if train_loss > 5.0:
            trackio.alert(
                title="Training diverged",
                text=f"train_loss={train_loss:.4f}, lr={lr}",
                level=trackio.AlertLevel.ERROR,
            )
            break

        if val_loss < best_val_loss:
            best_val_loss = val_loss
            patience = 0
        else:
            patience += 1
            if patience >= 5:
                trackio.alert(
                    title="Early stopping",
                    text=f"Best val_loss={best_val_loss:.4f}",
                    level=trackio.AlertLevel.WARN,
                )
                break

        time.sleep(0.05)

    trackio.alert(
        title="Run complete",
        text=f"Final val_loss={val_loss:.4f}, val_acc={val_acc:.4f}, "
             f"best_val_loss={best_val_loss:.4f}",
        level=trackio.AlertLevel.INFO,
    )
    trackio.finish()
    return run.name


def analyze_and_decide(api, project, run_name, config):
    """Read alerts from a run and suggest new hyperparameters."""
    alerts = api.alerts(project, run=run_name)

    new_config = dict(config)
    for alert in alerts:
        title = alert["title"].lower()
        if "diverged" in title:
            new_config["learning_rate"] *= 0.1
        elif "early stopping" in title:
            new_config["learning_rate"] *= 0.5
        elif "run complete" in title and "val_acc" in (alert.get("text") or ""):
            # Slight refinement
            new_config["learning_rate"] *= 0.8
            new_config["epochs"] += 10

    return new_config


api = trackio.Api()
project = "auto-hparam-search"

config = {
    "learning_rate": 0.01,
    "batch_size": 64,
    "epochs": 30,
}

for iteration in range(5):
    print(f"\n--- Iteration {iteration + 1} ---")
    print(f"Config: {config}")

    run_name = run_experiment(config)
    config = analyze_and_decide(api, project, run_name, config)

import trackio
from transformers import Trainer, TrainerCallback, TrainingArguments

class AlertCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        if "trackio" not in args.report_to or logs is None:
            return
        loss = logs.get("loss")
        if loss is not None and loss > 5.0:
            trackio.alert(
                title="Training loss spike",
                text=f"loss={loss:.4f} at step {state.global_step}. "
                     "Consider lowering the learning rate.",
                level=trackio.AlertLevel.ERROR,
            )

    def on_evaluate(self, args, state, control, metrics=None, **kwargs):
        if "trackio" not in args.report_to or metrics is None:
            return
        eval_loss = metrics.get("eval_loss")
        if eval_loss is not None and eval_loss > 2.0:
            trackio.alert(
                title="High eval loss",
                text=f"eval_loss={eval_loss:.4f} at step {state.global_step}.",
                level=trackio.AlertLevel.WARN,
            )

trainer = Trainer(
    model=model,
    args=TrainingArguments(
        report_to="trackio",
        project="my-project",
        output_dir="./output",
        num_train_epochs=3,
        eval_strategy="steps",
        eval_steps=100,
    ),
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    callbacks=[AlertCallback()],
)
trainer.train()

import trackio
from transformers import TrainerCallback

class RLAlertCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        if "trackio" not in args.report_to or logs is None:
            return

        reward = logs.get("train/reward")
        if reward is not None and reward < -1.0:
            trackio.alert(
                title="Reward collapse",
                text=f"reward={reward:.4f} at step {state.global_step}. "
                     "Consider lowering the learning rate or checking the "
                     "reward model.",
                level=trackio.AlertLevel.ERROR,
            )

        kl = logs.get("train/kl")
        if kl is not None and kl > 10.0:
            trackio.alert(
                title="KL divergence too high",
                text=f"kl={kl:.4f} at step {state.global_step}. "
                     "The policy is drifting too far from the reference model.",
                level=trackio.AlertLevel.WARN,
            )

        completion_length = logs.get("train/completion_length")
        if completion_length is not None and completion_length < 5.0:
            trackio.alert(
                title="Completions too short",
                text=f"Mean completion length={completion_length:.1f} tokens. "
                     "The model may be collapsing to short outputs.",
                level=trackio.AlertLevel.WARN,
            )

from trl import GRPOTrainer, GRPOConfig

trainer = GRPOTrainer(
    model=model,
    config=GRPOConfig(
        report_to="trackio",
        project="rl-experiment",
        output_dir="./output",
    ),
    train_dataset=dataset,
    callbacks=[RLAlertCallback()],
)
trainer.train()