Chapter 18: Final Submission Selection

import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier

from kaggle_companion.activities._common import clean

SEED = 18
WORLDS = 5          # independent training sets, each with its own twelve fitted candidates
DRAWS = 300         # random hidden splits per training set
N_TRAIN, N_POOL = 2500, 12000
N_TEST, N_LOCAL = 2000, 1000   # hidden test rows (public + private) and a separate local validation set
# Twelve gradient-boosting variants: learning rate x leaf count x L2 penalty. All are reasonable, so they sit close together.
CONFIGS = [(lr, leaves, l2) for lr in (0.04, 0.08) for leaves in (4, 6, 10) for l2 in (0.0, 8.0)]
RULES = ("public", "local", "pooled")


def candidate_losses(seed):
    """Fit every candidate once and return each pool row's log loss under each candidate (rows x candidates)."""
    rng = np.random.default_rng(seed)
    w = rng.normal(0, 0.45, 8)

    def rows(n):
        X = rng.normal(size=(n, 8))
        logit = X @ w + 0.6 * X[:, 0] * X[:, 1] + 0.5 * np.maximum(X[:, 2], 0) - 0.2
        return X, (rng.random(n) < 1 / (1 + np.exp(-logit))).astype(int)

    X_train, y_train = rows(N_TRAIN)
    X_pool, y_pool = rows(N_POOL)       # a large pool: its mean loss is the "true" loss of each candidate
    columns = []
    for lr, leaves, l2 in CONFIGS:
        model = HistGradientBoostingClassifier(learning_rate=lr, max_leaf_nodes=leaves, l2_regularization=l2,
                                               max_iter=50, early_stopping=False, random_state=0)
        columns.append(np.clip(model.fit(X_train, y_train).predict_proba(X_pool)[:, 1], 1e-3, 1 - 1e-3))
    p = np.column_stack(columns)
    return -(y_pool[:, None] * np.log(p) + (1 - y_pool[:, None]) * np.log(1 - p))


def run(public_fraction):
    n_public = int(public_fraction * N_TEST)
    stats = {rule: {"regret": [], "best": [], "rank": [], "optimism": []} for rule in RULES}
    gaps = []
    for w in range(WORLDS):
        loss = candidate_losses(SEED * 100 + w)
        truth = loss.mean(0)                                   # true log loss of each candidate
        best = int(np.argmin(truth))
        gaps.append(np.sort(truth)[1] - truth[best])
        rng = np.random.default_rng(SEED * 1000 + w)
        for _ in range(DRAWS):
            order = rng.permutation(len(loss))
            test, local = order[:N_TEST], order[N_TEST:N_TEST + N_LOCAL]
            public, private = test[:n_public], test[n_public:]
            private_loss = loss[private].mean(0)               # what the final standings will use
            scores = {"public": loss[public].mean(0), "local": loss[local].mean(0),
                      "pooled": (loss[public].sum(0) + loss[local].sum(0)) / (n_public + N_LOCAL)}
            for rule, score in scores.items():
                pick = int(np.argmin(score))                   # the candidate each evidence source would submit
                stats[rule]["regret"].append(truth[pick] - truth[best])
                stats[rule]["best"].append(pick == best)
                stats[rule]["rank"].append(1 + int((private_loss < private_loss[pick]).sum()))
                stats[rule]["optimism"].append(truth[pick] - score[pick])   # how much better the evidence said it was
    return clean({
        "public_fraction": public_fraction, "public_rows": n_public, "private_rows": N_TEST - n_public,
        "local_rows": N_LOCAL, "candidates": len(CONFIGS), "worlds": WORLDS, "draws": WORLDS * DRAWS,
        "gap_to_second": float(np.mean(gaps)),
        "rules": {rule: {"regret": float(np.mean(s["regret"])), "p_best": float(np.mean(s["best"])),
                         "private_rank": float(np.mean(s["rank"])), "optimism": float(np.mean(s["optimism"])),
                         "regret_se": float(np.std(s["regret"]) / np.sqrt(len(s["regret"])))}
                  for rule, s in stats.items()},
    }, digits=6)   # regrets are a few ten-thousandths of log loss: keep enough digits to tell the rules apart

import pandas as pd
from kaggle_companion.activities.ch18 import explain

VALUES = [0.05, 0.1, 0.25, 0.5]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch18 import draw, NCOLS, HEIGHT

DEFAULT = 0.1
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()