Chapter 14: What Doesn't Work

import numpy as np
from sklearn.cluster import KMeans
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import adjusted_rand_score, roc_auc_score
from sklearn.preprocessing import StandardScaler

from kaggle_companion.activities._common import clean

SEED = 14
REPLICATES = 16              # independent constructed datasets; every estimate is their mean
GROUPS, DIM = 6, 8
N_TRAIN, N_NEW = 800, 6000
EFFECT_SD = 1.5              # how strongly the true group moves the log odds


def generate(spread, seed):
    """Six hidden groups in eight dimensions. The label depends on three features and on the hidden group."""
    rng = np.random.default_rng(seed)
    centers = rng.normal(0.0, spread, (GROUPS, DIM))               # `spread` sets how far apart the groups sit
    effect = rng.normal(size=GROUPS)
    effect = (effect - effect.mean()) / effect.std() * EFFECT_SD   # group effect on the log odds
    weights = np.array([0.7, -0.5, 0.4])

    def rows(n):
        group = rng.integers(0, GROUPS, n)
        X = centers[group] + rng.normal(size=(n, DIM))
        y = (rng.random(n) < 1 / (1 + np.exp(-(X[:, :3] @ weights + effect[group])))).astype(int)
        return X, y, group

    return rows(N_TRAIN), rows(N_NEW)


def booster(**extra):
    return HistGradientBoostingClassifier(max_iter=60, learning_rate=0.1, max_leaf_nodes=8, min_samples_leaf=15,
                                          random_state=0, **extra)


def cluster_features(X_fit, *others):
    """Scale and cluster on training rows only; return the training columns and the same columns for other rows."""
    scaler = StandardScaler().fit(X_fit)
    km = KMeans(GROUPS, n_init=5, random_state=0).fit(scaler.transform(X_fit))
    def build(X):
        Z = scaler.transform(X)
        return np.column_stack([X, km.predict(Z), km.transform(Z)])   # raw columns, cluster id, distance to each centre
    return [build(X_fit)] + [build(X) for X in others], km, scaler


def run(spread):
    gain_learned, gain_known, recovery, stability, base_auc = [], [], [], [], []
    for rep in range(REPLICATES):
        (X, y, g), (X_new, y_new, g_new) = generate(spread, SEED * 100 + rep)
        score = lambda model, a, b: roc_auc_score(y_new, model.fit(a, y).predict_proba(b)[:, 1])
        base = score(booster(), X, X_new)
        (F, F_new), km, scaler = cluster_features(X, X_new)
        gain_learned.append(score(booster(categorical_features=[DIM]), F, F_new) - base)   # K-means id learned from training rows
        gain_known.append(score(booster(categorical_features=[DIM]),               # the true group id, supplied directly
                                np.column_stack([X, g]), np.column_stack([X_new, g_new])) - base)
        recovery.append(adjusted_rand_score(g_new, km.predict(scaler.transform(X_new))))   # known only because the data are constructed
        half = N_TRAIN // 2                                                         # observable: do two halves agree on new rows?
        labels = []
        for part in (slice(0, half), slice(half, N_TRAIN)):
            half_scaler = StandardScaler().fit(X[part])
            half_km = KMeans(GROUPS, n_init=5, random_state=0).fit(half_scaler.transform(X[part]))
            labels.append(half_km.predict(half_scaler.transform(X_new)))
        stability.append(adjusted_rand_score(labels[0], labels[1]))
        base_auc.append(base)
    return clean({
        "spread": spread, "baseline_auc": float(np.mean(base_auc)),
        "gain_learned": float(np.mean(gain_learned)), "gain_known": float(np.mean(gain_known)),
        "recovery": float(np.mean(recovery)), "stability": float(np.mean(stability)),
        "standard_error_learned": float(np.std(gain_learned) / np.sqrt(REPLICATES)),
        "per_replicate": {"gain_learned": gain_learned, "gain_known": gain_known},
        "replicates": REPLICATES, "train_rows": N_TRAIN, "new_rows": N_NEW,
    })

import pandas as pd
from kaggle_companion.activities.ch14 import explain

VALUES = [0.4, 0.8, 1.2, 1.8]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch14 import draw, NCOLS, HEIGHT

DEFAULT = 0.8
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()