超参数调优

Optuna是一个开源的超参数优化框架，专门用于机器学习

安装

pip install optuna

使用

import optuna

# 定义目标函数
def objective(trial: optuna.Trial):

    # 设定搜索区间
    hidden_size = trial.suggest_int('hidden_size', 1, 128)
    hidden_layers_num = trial.suggest_int('hidden_layers_num', 1, 5)
    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)
    optimizer_name = trial.suggest_categorical('optimizer', ['Adam', 'SGD'])
    layer_norm = trial.suggest_categorical('layer_norm', [True, False])
    nonlinearity = trial.suggest_categorical('nonlinearity', ['ReLU', 'LeakyReLU', 'Tanh'])

    model = Model(
        features_dim=1,
        output_dim=1,
        hidden_dim=hidden_size,
        hidden_layers_num=hidden_layers_num,
        nonlinearity=nonlinearity,
        layer_norm=layer_norm
    )
    optimizer = getattr(torch.optim, optimizer_name)(model.parameters(), lr=lr)
    loss_fn = nn.MSELoss()
    epochs = 1000
    test_loss_value = 0
    for epoch in range(epochs):
        y_pred = model(x_train_scaled)
        loss = loss_fn(y_pred, y_train_scaled)
        test_loss = loss_fn(model(x_test_scaled), y_test_scaled)
        test_loss_value = test_loss.item()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # 用于报告中间验证的最小测试损失
        trial.report(test_loss_value, epoch)

        # 添加剪枝(可选)
        if trial.should_prune():
            raise optuna.exceptions.TrialPruned()
    return test_loss_value

# 创建study
study = optuna.create_study(
    direction='minimize', # 最小化损失
    sampler=optuna.samplers.TPESampler(), # 采样器
    pruner=optuna.pruners.MedianPruner() # 剪枝器
)

# 优化
study.optimize(objective, n_trials=100)

# 输出结果
trail = study.best_trial
print(f'best trial: value={trail.value}, params={trail.params}')

可视化

import optuna.visualization as vis

# 绘制优化历史和参数重要性
vis.plot_optimization_history(study).show()
vis.plot_param_importances(study).show()