Hyperparameter Tuning

import optuna
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import cross_val_score
import numpy as np

X, y = load_breast_cancer(return_X_y=True)

def objective(trial):
    params = {
        "n_estimators": trial.suggest_int("n_estimators", 50, 500, step=50),
        "max_depth": trial.suggest_int("max_depth", 3, 30),
        "min_samples_split": trial.suggest_int("min_samples_split", 2, 20),
        "min_samples_leaf": trial.suggest_int("min_samples_leaf", 1, 10),
        "max_features": trial.suggest_categorical("max_features", ["sqrt", "log2", None]),
        "criterion": trial.suggest_categorical("criterion", ["gini", "entropy"]),
    }
    clf = RandomForestClassifier(**params, random_state=42, n_jobs=-1)
    scores = cross_val_score(clf, X, y, cv=5, scoring="f1")
    return scores.mean()

study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=42))
study.optimize(objective, n_trials=100, show_progress_bar=True)

print(f"Best F1: {study.best_value:.4f}")
print(f"Best params: {study.best_params}")

# Visualization
fig_importance = optuna.visualization.plot_param_importances(study)
fig_history = optuna.visualization.plot_optimization_history(study)
fig_contour = optuna.visualization.plot_contour(study, params=["n_estimators", "max_depth"])

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset, random_split
from ray import tune
from ray.tune.schedulers import ASHAScheduler
from ray.air import session
import numpy as np

def train_nn(config):
    X = torch.randn(2000, 20)
    y = (X[:, 0] + X[:, 1] * 2 > 0).long()
    dataset = TensorDataset(X, y)
    train_set, val_set = random_split(dataset, [1600, 400])
    train_loader = DataLoader(train_set, batch_size=config["batch_size"], shuffle=True)
    val_loader = DataLoader(val_set, batch_size=256)

    model = nn.Sequential(
        nn.Linear(20, config["hidden_size"]),
        nn.ReLU(),
        nn.Dropout(config["dropout"]),
        nn.Linear(config["hidden_size"], config["hidden_size"] // 2),
        nn.ReLU(),
        nn.Linear(config["hidden_size"] // 2, 2),
    )
    optimizer = torch.optim.Adam(model.parameters(), lr=config["lr"], weight_decay=config["weight_decay"])
    criterion = nn.CrossEntropyLoss()

    for epoch in range(50):
        model.train()
        for xb, yb in train_loader:
            loss = criterion(model(xb), yb)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        model.eval()
        correct, total = 0, 0
        with torch.no_grad():
            for xb, yb in val_loader:
                correct += (model(xb).argmax(1) == yb).sum().item()
                total += yb.size(0)
        session.report({"val_accuracy": correct / total})

search_space = {
    "hidden_size": tune.choice([64, 128, 256]),
    "lr": tune.loguniform(1e-4, 1e-1),
    "dropout": tune.uniform(0.1, 0.5),
    "batch_size": tune.choice([32, 64, 128]),
    "weight_decay": tune.loguniform(1e-5, 1e-2),
}

scheduler = ASHAScheduler(max_t=50, grace_period=5, reduction_factor=3)
result = tune.run(
    train_nn,
    config=search_space,
    num_samples=50,
    scheduler=scheduler,
    metric="val_accuracy",
    mode="max",
    resources_per_trial={"cpu": 2},
)

print(f"Best config: {result.best_config}")
print(f"Best val accuracy: {result.best_result['val_accuracy']:.4f}")