Chapter 43: Image Classification Tricks

import numpy as np
from scipy.ndimage import zoom
from sklearn.datasets import load_digits
from sklearn.model_selection import GroupKFold, KFold
from sklearn.neighbors import KNeighborsClassifier

from kaggle_companion.activities._common import clean

SEED = 43
REPLICATES = 8          # independent source-image draws per control value; results are means over them
SOURCES, TEST_SOURCES = 300, 400
CROP = 14               # crops are 14 x 14 windows of a 16 x 16 image
OFFSETS = [(i, j) for i in range(3) for j in range(3)]    # nine possible, heavily overlapping, window positions


def load_images():
    """Real digit scans: scikit-learn's bundled 8 x 8 digits, enlarged to 16 x 16 by bilinear interpolation."""
    digits = load_digits()
    return np.stack([zoom(image / 16.0, 2, order=1) for image in digits.images]), digits.target


def make_crops(images, labels, crops_per_image, rng):
    """Cut `crops_per_image` windows out of every source image; remember which source each crop came from."""
    X, y, source = [], [], []
    for n, (image, label) in enumerate(zip(images, labels)):
        for position in rng.choice(len(OFFSETS), crops_per_image, replace=False):
            i, j = OFFSETS[position]
            X.append(image[i:i + CROP, j:j + CROP].ravel())
            y.append(label)
            source.append(n)
    return np.array(X), np.array(y), np.array(source)


def cross_validated_accuracy(splitter, X, y, groups=None):
    """Pool the held-out predictions of every fold and score them once."""
    pred = np.zeros(len(y), dtype=int)
    for fit, out in splitter.split(X, y, groups):
        pred[out] = KNeighborsClassifier(1).fit(X[fit], y[fit]).predict(X[out])
    return float((pred == y).mean())


def run(crops_per_image):
    images, labels = load_images()
    scores = {"random": [], "grouped": [], "heldout": []}
    for r in range(REPLICATES):
        rng = np.random.default_rng(SEED * 100 + r)
        order = rng.permutation(len(labels))
        train, test = order[:SOURCES], order[1000:1000 + TEST_SOURCES]       # test sources never appear in training
        X, y, source = make_crops(images[train], labels[train], crops_per_image, rng)
        X_test, y_test, _ = make_crops(images[test], labels[test], crops_per_image, rng)
        scores["random"].append(cross_validated_accuracy(KFold(5, shuffle=True, random_state=0), X, y))   # crops split at random
        scores["grouped"].append(cross_validated_accuracy(GroupKFold(5), X, y, source))                   # whole source images split
        scores["heldout"].append(float((KNeighborsClassifier(1).fit(X, y).predict(X_test) == y_test).mean()))
    mean = {k: float(np.mean(v)) for k, v in scores.items()}
    return clean({"crops_per_image": crops_per_image, "accuracy": mean, "training_crops": SOURCES * crops_per_image,
                  "random_overstates_in_share_of_worlds": float(np.mean(np.array(scores["random"]) > np.array(scores["heldout"]))),
                  "replicates": REPLICATES, "source_images": SOURCES})

import pandas as pd
from kaggle_companion.activities.ch43 import explain

VALUES = [1, 3, 9]
results = {value: run(value) for value in VALUES}
table = pd.DataFrame({str(v): explain(results[v], v)['metrics'] for v in VALUES})
print(table.to_string())

import matplotlib.pyplot as plt
from kaggle_companion.activities.ch43 import draw, NCOLS, HEIGHT

DEFAULT = 9
fig, axes = plt.subplots(1, NCOLS, figsize=(6.6 if NCOLS == 1 else 10.4, HEIGHT), layout='constrained')
draw(axes, results[DEFAULT], DEFAULT)
plt.show()