# Source: content/notes/ml/hyperparameter-tuning.md
# Independent CPU example; use the curriculum environment.
# See /notes/ml/#example-environment or /notes/deep-learning/#example-environment.

import numpy as np
from scipy.stats import loguniform
from sklearn.datasets import make_moons
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.model_selection import (train_test_split, StratifiedKFold,
    RandomizedSearchCV, cross_validate)
from sklearn.metrics import roc_auc_score

X, y = make_moons(n_samples=420, noise=0.22, random_state=3)
dev, test, y_dev, y_test = train_test_split(
    X, y, test_size=0.25, stratify=y, random_state=19)
inner_cv = StratifiedKFold(3, shuffle=True, random_state=5)
outer_cv = StratifiedKFold(3, shuffle=True, random_state=6)
search = RandomizedSearchCV(
    make_pipeline(StandardScaler(), SVC()),
    {"svc__C": loguniform(0.1, 100), "svc__gamma": loguniform(0.03, 5)},
    n_iter=8, scoring="roc_auc", cv=inner_cv, random_state=4, n_jobs=1,
    error_score="raise", refit=True, return_train_score=True)
result = cross_validate(search, dev, y_dev, cv=outer_cv,
    scoring="roc_auc", return_estimator=True, n_jobs=1)
assert len(result["estimator"]) == 3
assert all(len(est.cv_results_["params"]) == 8 for est in result["estimator"])
search.fit(dev, y_dev)
holdout_auc = roc_auc_score(y_test, search.decision_function(test))
assert result["test_score"].mean() > 0.85
assert holdout_auc > 0.85
assert search.best_estimator_.named_steps["standardscaler"].n_samples_seen_ == len(dev)
print("outer scores", result["test_score"], "descriptive std", result["test_score"].std())
print("selected parameters", search.best_params_, "holdout AUC", holdout_auc)