Chapter 3: Building a CV That Matches the Test Set

import numpy as np
from sklearn.linear_model import LogisticRegression

from kaggle_companion.activities._common import clean

SEED = 3
REPLICATES = 100      # independent constructed histories; every estimate is their mean
SEASONS, GAMES, FEATURES = 8, 400, 4
FIRST_SCORED = 4      # score seasons 5, 6 and 7 (0-based 4, 5, 6); season 8 is the future
FUTURE_GAMES = 4000


def generate(drift, seed):
    """Seven development seasons and a large season 8. The weights that turn features into win odds
    wander from season to season; `drift` is the size of the wander (their length stays constant)."""
    rng = np.random.default_rng(seed)
    weights = np.zeros((SEASONS, FEATURES))
    weights[0] = rng.normal(0, 1.0, FEATURES)
    for s in range(1, SEASONS):
        step = weights[s - 1] + rng.normal(0, drift, FEATURES)
        weights[s] = step * np.linalg.norm(weights[0]) / np.linalg.norm(step)

    def games(s, n):
        x = rng.normal(size=(n, FEATURES))
        win = rng.random(n) < 1 / (1 + np.exp(-x @ weights[s]))
        return x, win.astype(int)

    dev = [games(s, GAMES) for s in range(SEASONS - 1)]
    return dev, games(SEASONS - 1, FUTURE_GAMES)


def fit(seasons):
    X = np.vstack([s[0] for s in seasons])
    y = np.concatenate([s[1] for s in seasons])
    return LogisticRegression(max_iter=500).fit(X, y)


def brier(y, p):
    return float(np.mean((y - p) ** 2))


def one_history(drift, seed):
    dev, (X_next, y_next) = generate(drift, seed)
    held, loso, past = [], [], []
    for s in range(FIRST_SCORED, SEASONS - 1):
        X, y = dev[s]
        held.append(y)
        # Leave-one-season-out: train on every other development season, later ones included.
        loso.append(fit([d for k, d in enumerate(dev) if k != s]).predict_proba(X)[:, 1])
        # Past-only: train on the seasons before s, as a real forecast would.
        past.append(fit(dev[:s]).predict_proba(X)[:, 1])
    y = np.concatenate(held)
    # Truth: the final recipe (all seven development seasons) on the next season, 4,000 fresh games.
    truth = brier(y_next, fit(dev).predict_proba(X_next)[:, 1])
    return brier(y, np.concatenate(loso)), brier(y, np.concatenate(past)), truth


def run(drift):
    runs = np.array([one_history(drift, SEED * 1000 + i) for i in range(REPLICATES)])
    loso, past, truth = runs.T
    return clean({
        "drift": drift,
        "loso": float(loso.mean()), "past_only": float(past.mean()), "next_season": float(truth.mean()),
        # optimism = Brier on the next season minus the estimate (positive: the estimate looked better than reality)
        "optimism": {"loso": float((truth - loso).mean()), "past_only": float((truth - past).mean())},
        "per_history": {"loso": (truth - loso).tolist(), "past_only": (truth - past).tolist()},
        "loso_more_optimistic": float(np.mean((truth - loso) > (truth - past))),
        "replicates": REPLICATES, "scored_games": GAMES * (SEASONS - 1 - FIRST_SCORED), "future_games": FUTURE_GAMES,
    })

import pandas as pd
from kaggle_companion.activities.ch03 import explain

VALUES = [0, 0.25, 0.5, 1.0]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch03 import draw, NCOLS, HEIGHT

DEFAULT = 0.5
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()