Python超参数优化的实战方法

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
param_grid = {'C': [0.1, 1, 10], 'kernel': ['rbf', 'linear']}
grid_search = GridSearchCV(SVC(), param_grid, cv=5, n_jobs=-1)
grid_search.fit(X_train, y_train)

from scipy.stats import randint
param_dist = {'n_estimators': randint(50, 300), 'max_depth': [None, 10, 20]}
random_search = RandomizedSearchCV(RandomForestClassifier(), param_dist, n_iter=50)

import optuna
def objective(trial):
    params = {
        'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 1e-1),
        'max_depth': trial.suggest_int('max_depth', 2, 10)
    }
    model = XGBClassifier(**params)
    return log_loss(y_test, model.predict_proba(X_test))
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=50)

def objective(trial):
    params = {
        'objective': 'binary:logistic',
        'n_estimators': trial.suggest_int('n_estimators', 50, 500),
        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.3),
        'subsample': trial.suggest_uniform('subsample', 0.7, 1.0)
    }
    model = XGBClassifier(**params)
    return log_loss(y_test, model.predict_proba(X_test))
study.optimize(objective, n_trials=100)

from catboost import CatBoostRegressor, CV
params = {
    'iterations': 1000,
    'learning_rate': 0.05,
    'depth': 6
}
cv_data = CV(CatBoostRegressor(**params), df_train, fold_count=5)

def objective(trial):
    max_depth = trial.suggest_int('max_depth', 2, 10)
    if max_depth > 5:
        learning_rate = trial.suggest_loguniform('learning_rate', 1e-3, 1e-1)
    else:
        learning_rate = 0.1
    # 后续模型构建

# Scikit-learn并行
RandomizedSearchCV(..., n_jobs=-1)
# Optuna分布式
study.optimize(..., n_jobs=8)

from sklearn.ensemble import StackingClassifier
estimators = [
    ('rf', RandomForestClassifier()),
    ('svm', SVC())
]
stack = StackingClassifier(estimators, final_estimator=LogisticRegression())
param_grid = {'rf__n_estimators': [50, 100], 'svm__C': [0.1, 1]}