Chapter 26: Advanced Target Encoding

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import log_loss
from sklearn.model_selection import KFold

from kaggle_companion.activities._common import clean

SEED = 26
REPLICATES = 30       # independent constructed datasets per setting; every number is their mean
K, N_TRAIN, N_NEW = 300, 6000, 20000
PSEUDOCOUNTS = [0, 1, 3, 10, 30, 100, 300]   # smoothing strength m; 0 means no smoothing
WOE_SMOOTH = 0.5      # the chapter's Weight of Evidence default
RARE = 5              # a category with fewer than 5 development rows is rare


def generate(spread, seed):
    """300 categories with Zipf-like frequencies, so a few are common and many are rare. True category effects ~ Normal(0, spread)."""
    rng = np.random.default_rng(seed)
    effect = rng.normal(0, spread, K)
    freq = 1 / np.arange(1, K + 1)
    freq /= freq.sum()
    cat, cat_new = rng.choice(K, N_TRAIN, p=freq), rng.choice(K, N_NEW, p=freq)

    def rows(c):
        x = rng.normal(size=(len(c), 2))
        logit = 0.6 * x[:, 0] - 0.4 * x[:, 1] + effect[c] - 0.3
        return x, (rng.random(len(c)) < 1 / (1 + np.exp(-logit))).astype(int)

    (x, y), (x_new, y_new) = rows(cat), rows(cat_new)
    return cat, x, y, cat_new, x_new, y_new


def smoothed_mean(cat, y, m):
    """The chapter 9 encoding e_c = (s_c + m*mu) / (n_c + m); with m = 0 it is each category's own mean (the global mean if unseen)."""
    mu = y.mean()
    s, n = np.bincount(cat, weights=y, minlength=K), np.bincount(cat, minlength=K)
    return (s + m * mu) / (n + m) if m > 0 else np.where(n > 0, s / np.maximum(n, 1), mu)


def weight_of_evidence(cat, y, smooth=WOE_SMOOTH):
    """The chapter's fit_woe and transform_woe: log of (smoothed share among positives) over (smoothed share among negatives).

    As in the chapter, the pseudocount goes to every category seen in fitting (k_seen*smooth in each denominator), and a
    category unseen in fitting gets the neutral value 0."""
    seen = np.bincount(cat, minlength=K) > 0
    k_seen = seen.sum()
    pos = np.bincount(cat, weights=y, minlength=K) + smooth
    neg = np.bincount(cat, weights=1 - y, minlength=K) + smooth
    woe = np.log((pos / (y.sum() + k_seen * smooth)) / (neg / ((1 - y).sum() + k_seen * smooth)))
    return np.where(seen, woe, 0.0)


def to_logit(p):
    p = np.clip(p, 1e-4, 1 - 1e-4)
    return np.log(p / (1 - p))


def evaluate(data, encoder, transform=lambda v: v):
    """The deployable pipeline: cross-fit the training column, fit the model, encode new rows with the map from all training rows."""
    cat, x, y, cat_new, x_new, y_new = data
    column = np.zeros(len(cat))
    for fit, out in KFold(5, shuffle=True, random_state=2).split(cat):
        column[out] = encoder(cat[fit], y[fit])[cat[out]]
    full_map = encoder(cat, y)
    model = LogisticRegression(C=10, max_iter=500).fit(np.column_stack([x, transform(column)]), y)
    p = model.predict_proba(np.column_stack([x_new, transform(full_map[cat_new])]))[:, 1]
    rare = np.bincount(cat, minlength=K)[cat_new] < RARE
    return [log_loss(y_new, p), log_loss(y_new[rare], p[rare], labels=[0, 1]), log_loss(y_new[~rare], p[~rare], labels=[0, 1]), rare.mean()]


def run(category_spread):
    sets = [generate(category_spread, SEED * 1000 + i) for i in range(REPLICATES)]
    by_m = {str(m): np.mean([evaluate(d, lambda c, y, m=m: smoothed_mean(c, y, m), to_logit) for d in sets], axis=0) for m in PSEUDOCOUNTS}
    woe = np.mean([evaluate(d, weight_of_evidence) for d in sets], axis=0)
    best = min(PSEUDOCOUNTS, key=lambda m: by_m[str(m)][0])
    return clean({
        "category_spread": category_spread, "replicates": REPLICATES, "categories": K, "training_rows": N_TRAIN, "new_rows": N_NEW,
        "rare_share_of_new_rows": float(woe[3]), "best_m": best,
        "overall": {str(m): by_m[str(m)][0] for m in PSEUDOCOUNTS}, "rare": {str(m): by_m[str(m)][1] for m in PSEUDOCOUNTS},
        "common": {str(m): by_m[str(m)][2] for m in PSEUDOCOUNTS},
        "woe": {"overall": woe[0], "rare": woe[1], "common": woe[2]},
    })

import pandas as pd
from kaggle_companion.activities.ch26 import explain

VALUES = [0.2, 0.5, 1.0, 1.5]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch26 import draw, NCOLS, HEIGHT

DEFAULT = 1.0
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()