# Source: content/notes/ml/trees-and-ensembles.md
# Independent CPU example; use the curriculum environment.
# See /notes/ml/#example-environment or /notes/deep-learning/#example-environment.

import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

X, y = make_classification(n_samples=400, n_features=8,
                           n_informative=5, random_state=41)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, stratify=y, random_state=41)
model = StackingClassifier(estimators=[
    ("forest", RandomForestClassifier(n_estimators=50, min_samples_leaf=3,
                                       random_state=41, n_jobs=1)),
    ("linear", make_pipeline(StandardScaler(), LogisticRegression(max_iter=500))),
    ("neighbors", make_pipeline(StandardScaler(), KNeighborsClassifier(11))),
], final_estimator=LogisticRegression(max_iter=500),
   cv=StratifiedKFold(3, shuffle=True, random_state=41), n_jobs=1)
model.fit(X_train, y_train)
pred = model.predict(X_test)
p = model.predict_proba(X_test)[:, 1]
assert pred.shape == y_test.shape and np.isfinite(p).all()
print("Test log loss:", log_loss(y_test, p))
print("Test ROC AUC:", roc_auc_score(y_test, p))