# ---- Part 0: Setup & Config ----
# Install once in your environment before running (Apple Silicon: tensorflow-macos + tensorflow-metal):
# %pip install "tensorflow>=2.16,<2.17" tensorflow-metal numpy pandas matplotlib seaborn scikit-learn pillow

import os, sys, gc, json, time, math, pickle, random, platform, importlib, warnings
warnings.filterwarnings("ignore")
os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from sklearn.model_selection import train_test_split
from IPython.display import display, Markdown
tf.autograph.set_verbosity(0)   # silence AutoGraph notes about tf.data lambdas

# ---- Reproducibility: every source of randomness is seeded here and re-seeded per run ----
SEED = 42
def reseed(seed=SEED):
    os.environ["PYTHONHASHSEED"] = str(seed)
    random.seed(seed); np.random.seed(seed); tf.random.set_seed(seed)
    keras.utils.set_random_seed(seed)
reseed()

# ---- Hardware detection & training-setup justification (rubric §1) ----
GPUS = tf.config.list_physical_devices("GPU")
for _g in GPUS:
    try: tf.config.experimental.set_memory_growth(_g, True)
    except Exception: pass
_is_metal = bool(GPUS) and platform.system() == "Darwin" and platform.machine() == "arm64"
DEVICE = "Apple Metal GPU" if _is_metal else ("CUDA GPU" if GPUS else "CPU")

# ---- Environment stamp: the versions behind every number below ----
_env = {"python": platform.python_version(), "os": platform.platform(), "device": DEVICE}
for _m in ("tensorflow", "keras", "numpy", "pandas", "sklearn", "PIL"):
    try: _env[_m] = importlib.import_module(_m).__version__
    except Exception as _e: _env[_m] = f"n/a ({_e.__class__.__name__})"
print("Environment:", " | ".join(f"{k}={v}" for k, v in _env.items()))

# ---- Run profile: one switch trades wall-clock for coverage; every knob below reads from it ----
#   FULL : the graded run (spec-compliant 40k/10k/10k split, 15 epochs with early stopping)
#   FAST : end-to-end smoke test of the pipeline on a tiny stratified subset (minutes, not hours)
RUN_PROFILE = os.environ.get("CAPSTONE_PROFILE", "FULL")
PROFILES = {
    "FULL": dict(TRAIN_N=None, TEST_N=None, EPOCHS=15, EXP_TRAIN_N=10000, EXP_VAL_N=2000, EXP_EPOCHS=5,
                 FEAT_N=2000, TSNE_N=2000, CV_FOLDS=3, QUANT_EVAL_N=1000),
    "FAST": dict(TRAIN_N=640,  TEST_N=256,  EPOCHS=1,  EXP_TRAIN_N=256,   EXP_VAL_N=64,   EXP_EPOCHS=1,
                 FEAT_N=128,  TSNE_N=128,  CV_FOLDS=2, QUANT_EVAL_N=64),
}
P = PROFILES[RUN_PROFILE]

# ---- Specification constants (Technical Requirements §2-§4) ----
DATA_DIR      = os.environ.get("CIFAR_DIR", "cifar-10-batches-py")   # extracted from cifar-10-python.tar.gz
IMG_SIZE      = 224            # model input; CIFAR-10 native resolution is 32
NUM_CLASSES   = 10
BATCH_SIZE    = 32
LEARNING_RATE = 1e-3
DROPOUT       = 0.5
VAL_FRACTION  = 0.20           # 50k training pool -> 40k train / 10k validation
EPOCHS        = P["EPOCHS"]
ES_PATIENCE, RLR_FACTOR, RLR_PATIENCE = 5, 0.2, 3
CLASS_NAMES = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
AUTOTUNE = tf.data.AUTOTUNE

RESULTS_DIR = "results"; os.makedirs(RESULTS_DIR, exist_ok=True)
FIG_DIR = os.path.join(RESULTS_DIR, "figures"); os.makedirs(FIG_DIR, exist_ok=True)
RESULTS = {"config": dict(profile=RUN_PROFILE, seed=SEED, img_size=IMG_SIZE, batch_size=BATCH_SIZE,
                          learning_rate=LEARNING_RATE, dropout=DROPOUT, epochs_max=EPOCHS,
                          val_fraction=VAL_FRACTION, **P),
           "environment": _env, "models": {}, "features": {}, "experiments": {}, "challenges": {}}

def hardware_note():
    if _is_metal:
        return ("Training on the Apple Silicon GPU through the Metal PluggableDevice (unified memory: the GPU "
                "shares system RAM, so the effective budget is the machine's RAM minus ~4 GB of OS overhead). "
                "This is why the ImageNet backbones are kept frozen and batch size 32 is retained.")
    if GPUS:
        return "Training on a CUDA GPU; memory growth enabled so four models can be built sequentially in one kernel."
    return ("No GPU visible: training on CPU. Per the specification's hardware note the epoch budget is the "
            "reduced 5-10 range and the transfer-learning backbones stay frozen (forward pass only).")
print(f"Profile: {RUN_PROFILE} | Device: {DEVICE} | GPUs: {len(GPUS)} | Seed: {SEED}")
print("Setup justification:", hardware_note())
RESULTS["environment"]["setup_justification"] = hardware_note()

Environment: python=3.11.13 | os=macOS-26.6.2-arm64-arm-64bit | device=Apple Metal GPU | tensorflow=2.16.1 | keras=3.15.1 | numpy=1.26.4 | pandas=2.3.2 | sklearn=1.7.2 | PIL=11.3.0
Profile: FULL | Device: Apple Metal GPU | GPUs: 1 | Seed: 42
Setup justification: Training on the Apple Silicon GPU through the Metal PluggableDevice (unified memory: the GPU shares system RAM, so the effective budget is the machine's RAM minus ~4 GB of OS overhead). This is why the ImageNet backbones are kept frozen and batch size 32 is retained.

# ---- Part 1a: Load CIFAR-10 pickles and write train.csv / test.csv (spec "Instructions for Learners") ----
def _unpickle(path):
    with open(path, "rb") as fh:
        return pickle.load(fh, encoding="bytes")

def load_cifar10_batches(data_dir):
    '''Loads the 5 training batches + test batch. Raises a clear error if the extraction is incomplete.'''
    if not os.path.isdir(data_dir):
        raise FileNotFoundError(f"'{data_dir}' not found. Extract cifar-10-python.tar.gz next to this notebook "
                                f"or set the CIFAR_DIR environment variable.")
    required = [f"data_batch_{i}" for i in range(1, 6)] + ["test_batch", "batches.meta"]
    missing = [f for f in required if not os.path.exists(os.path.join(data_dir, f))]
    if missing:
        raise FileNotFoundError(f"Missing files in '{data_dir}': {missing}. Do NOT skip the extraction step.")
    Xs, ys = [], []
    for i in range(1, 6):
        b = _unpickle(os.path.join(data_dir, f"data_batch_{i}"))
        Xs.append(b[b"data"]); ys.extend(b[b"labels"])
    X_train_flat = np.concatenate(Xs).astype(np.uint8)          # (50000, 3072) in R,G,B plane order
    y_train_all  = np.asarray(ys, dtype=np.int64)
    t = _unpickle(os.path.join(data_dir, "test_batch"))
    X_test_flat  = t[b"data"].astype(np.uint8); y_test = np.asarray(t[b"labels"], dtype=np.int64)
    meta = [n.decode() for n in _unpickle(os.path.join(data_dir, "batches.meta"))[b"label_names"]]
    assert meta == CLASS_NAMES, f"Unexpected label names: {meta}"
    return X_train_flat, y_train_all, X_test_flat, y_test

def to_images(flat):
    '''(N, 3072) plane-major -> (N, 32, 32, 3) uint8.'''
    return flat.reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1)

def write_csv(flat, y, path):
    '''Spec format: first column 'label', then pixel_0..pixel_3071. Skipped if already present.'''
    if os.path.exists(path):
        print(f"{path} already exists — skipped"); return
    cols = ["label"] + [f"pixel_{i}" for i in range(flat.shape[1])]
    pd.DataFrame(np.column_stack([y, flat]), columns=cols).to_csv(path, index=False)
    print(f"wrote {path}")

t0 = time.time()
X_train_flat, y_train_all, X_test_flat, y_test = load_cifar10_batches(DATA_DIR)
print(f"Loaded pickles in {time.time()-t0:.1f}s | train pool {X_train_flat.shape} | test {X_test_flat.shape}")

WRITE_CSV = RUN_PROFILE == "FULL"       # ~300 MB; the smoke test skips the write
if WRITE_CSV:
    write_csv(X_train_flat, y_train_all, "train.csv"); write_csv(X_test_flat, y_test, "test.csv")
    # Verify the expected output without loading 300 MB into RAM: row count + column count from the header.
    for f, n in (("train.csv", 50000), ("test.csv", 10000)):
        with open(f) as fh:
            ncols = len(fh.readline().split(",")); nrows = sum(1 for _ in fh)
        print(f"{f}: {nrows} rows x {ncols} columns (expected {n} x 3073) -> {'OK' if (nrows, ncols)==(n, 3073) else 'MISMATCH'}")

X_train_all = to_images(X_train_flat); X_test = to_images(X_test_flat)
del X_train_flat, X_test_flat; gc.collect()
print("Images:", X_train_all.shape, X_test.shape, X_train_all.dtype)

Loaded pickles in 0.1s | train pool (50000, 3072) | test (10000, 3072)
train.csv already exists — skipped
test.csv already exists — skipped
train.csv: 50000 rows x 3073 columns (expected 50000 x 3073) -> OK
test.csv: 10000 rows x 3073 columns (expected 10000 x 3073) -> OK
Images: (50000, 32, 32, 3) (10000, 32, 32, 3) uint8

# ---- Part 1b: Stratified 80/20 split of the 50k pool; optional stratified subsample for FAST profile ----
reseed()
X_train, X_val, y_train, y_val = train_test_split(
    X_train_all, y_train_all, test_size=VAL_FRACTION, stratify=y_train_all, random_state=SEED)

if P["TRAIN_N"]:   # FAST profile only: shrink every split, keeping class balance
    X_train, _, y_train, _ = train_test_split(X_train, y_train, train_size=P["TRAIN_N"], stratify=y_train, random_state=SEED)
    X_val,   _, y_val,   _ = train_test_split(X_val,   y_val,   train_size=P["TRAIN_N"]//4, stratify=y_val, random_state=SEED)
    X_test,  _, y_test,  _ = train_test_split(X_test,  y_test,  train_size=P["TEST_N"],  stratify=y_test, random_state=SEED)

# Per-channel statistics of the TRAINING split only (used by the Simple CNN's Normalization layer).
CIFAR_MEAN = (X_train.astype(np.float32) / 255.0).mean(axis=(0, 1, 2))
CIFAR_STD  = (X_train.astype(np.float32) / 255.0).std(axis=(0, 1, 2))
RESULTS["config"].update(n_train=int(len(X_train)), n_val=int(len(X_val)), n_test=int(len(X_test)),
                         channel_mean=CIFAR_MEAN.round(4).tolist(), channel_std=CIFAR_STD.round(4).tolist())
print(f"train {X_train.shape} | val {X_val.shape} | test {X_test.shape}")
print("channel mean:", CIFAR_MEAN.round(4), "| channel std:", CIFAR_STD.round(4))

# Class distribution per split (balance check) + sample grid
dist = pd.DataFrame({"train": np.bincount(y_train, minlength=10), "val": np.bincount(y_val, minlength=10),
                     "test": np.bincount(y_test, minlength=10)}, index=CLASS_NAMES)
display(dist.T)
fig, axes = plt.subplots(2, 10, figsize=(15, 3.4))
for c in range(10):
    idx = np.where(y_train == c)[0][:2]
    for r in range(2):
        axes[r, c].imshow(X_train[idx[r]]); axes[r, c].axis("off")
    axes[0, c].set_title(CLASS_NAMES[c], fontsize=9)
plt.suptitle("CIFAR-10 training samples (native 32x32)"); plt.tight_layout()
plt.savefig(os.path.join(FIG_DIR, "samples.png"), dpi=120); plt.show()

train (40000, 32, 32, 3) | val (10000, 32, 32, 3) | test (10000, 32, 32, 3)
channel mean: [0.4911 0.4821 0.4466] | channel std: [0.247  0.2435 0.2616]

# ---- Part 1c: tf.data input pipeline — augmentation (train only) -> resize -> float32 [0,255] ----
# Augmentation runs at native 32x32 (cheap) and the upscale to IMG_SIZE happens last, so the
# 224x224 tensors only ever exist one batch at a time in GPU memory (50k x 224 x 224 x 3 float32
# would be ~30 GB if materialised).
PAD = 4
def _augment_image(img, label):
    img = tf.image.random_flip_left_right(img)                                   # horizontal flip
    img = tf.pad(img, [[PAD, PAD], [PAD, PAD], [0, 0]], mode="REFLECT")          # random crop with padding
    img = tf.image.random_crop(img, [32, 32, 3])
    return img, label

_batch_augment = keras.Sequential([
    layers.RandomRotation(factor=15/360, fill_mode="reflect"),                  # small angles (+/-15 deg)
    layers.RandomTranslation(0.1, 0.1, fill_mode="reflect"),                    # width/height shift 10%
], name="batch_augmentation")

def _resize(img, label, size):
    return tf.image.resize(img, [size, size], method="bilinear"), label

def make_dataset(X, y, training=False, batch_size=BATCH_SIZE, img_size=IMG_SIZE, augment=True, seed=SEED, shuffle=True):
    y_1h = keras.utils.to_categorical(y, NUM_CLASSES).astype(np.float32)       # categorical cross-entropy
    ds = tf.data.Dataset.from_tensor_slices((X, y_1h))
    if training and shuffle:
        ds = ds.shuffle(min(len(X), 20000), seed=seed, reshuffle_each_iteration=True)
        if augment:
            ds = ds.map(_augment_image, num_parallel_calls=AUTOTUNE)
    ds = ds.map(lambda i, l: (tf.cast(i, tf.float32), l), num_parallel_calls=AUTOTUNE).batch(batch_size)
    if training and augment:
        ds = ds.map(lambda i, l: (_batch_augment(i, training=True), l), num_parallel_calls=AUTOTUNE)
    ds = ds.map(lambda i, l: _resize(i, l, img_size), num_parallel_calls=AUTOTUNE)
    return ds.prefetch(AUTOTUNE)

train_ds = make_dataset(X_train, y_train, training=True)
val_ds   = make_dataset(X_val,   y_val)
test_ds  = make_dataset(X_test,  y_test)          # deterministic order -> y_test aligns with predictions
STEPS_PER_EPOCH = math.ceil(len(X_train) / BATCH_SIZE)
print(f"steps/epoch: {STEPS_PER_EPOCH} | batch: {next(iter(train_ds))[0].shape}")

# Visual check: the same 8 training images before and after augmentation (+ the resize to IMG_SIZE)
raw = X_train[:8]
aug = make_dataset(raw, y_train[:8], training=True, batch_size=8, shuffle=False)   # same order as `raw`
aug_imgs = next(iter(aug))[0].numpy().astype(np.uint8)
fig, axes = plt.subplots(2, 8, figsize=(14, 3.6))
for i in range(8):
    axes[0, i].imshow(raw[i]); axes[0, i].axis("off")
    axes[1, i].imshow(aug_imgs[i]); axes[1, i].axis("off")
axes[0, 0].set_title("original 32x32", fontsize=9, loc="left"); axes[1, 0].set_title(f"augmented + resized {IMG_SIZE}x{IMG_SIZE}", fontsize=9, loc="left")
plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "augmentation.png"), dpi=120); plt.show()

steps/epoch: 1250 | batch: (32, 224, 224, 3)

# ---- Part 2: Model builders ----
def build_simple_cnn(img_size=IMG_SIZE, dropout=DROPOUT, extra_block=False, name="SimpleCNN"):
    '''Model 1. Three Conv->MaxPool blocks (32, 64, 128), Flatten, Dense(128), Dropout, Dense(10).
    extra_block=True adds Conv2D(256)->MaxPool before Flatten (Part 6, Experiment 5).'''
    pools = [4, 4, 2] if img_size >= 128 else [2, 2, 2]
    inputs = keras.Input(shape=(img_size, img_size, 3), name="input")
    x = layers.Rescaling(1.0 / 255.0, name="rescale_0_1")(inputs)                        # [0,255] -> [0,1]
    x = layers.Normalization(mean=CIFAR_MEAN.tolist(), variance=(CIFAR_STD ** 2).tolist(), name="standardize")(x)
    x = layers.Conv2D(32,  3, padding="same", activation="relu", name="conv1")(x); x = layers.MaxPooling2D(pools[0], name="pool1")(x)
    x = layers.Conv2D(64,  3, padding="same", activation="relu", name="conv2")(x); x = layers.MaxPooling2D(pools[1], name="pool2")(x)
    x = layers.Conv2D(128, 3, padding="same", activation="relu", name="conv3" if extra_block else "last_conv")(x)
    x = layers.MaxPooling2D(pools[2], name="pool3")(x)
    if extra_block:
        x = layers.Conv2D(256, 3, padding="same", activation="relu", name="last_conv")(x); x = layers.MaxPooling2D(2, name="pool4")(x)
    x = layers.Flatten(name="flatten")(x)
    x = layers.Dense(128, activation="relu", name="fc1")(x)
    x = layers.Dropout(dropout, name="dropout")(x)
    outputs = layers.Dense(NUM_CLASSES, activation="softmax", name="predictions")(x)
    model = keras.Model(inputs, outputs, name=name)
    model._feature_layer = "last_conv"
    model._base = None
    return model

# Each ImageNet backbone expects its OWN input scaling; applied in-graph so train/test/deploy agree.
BACKBONES = {
    "VGG16":          (keras.applications.VGG16,          keras.applications.vgg16.preprocess_input,        "block5_pool"),
    "ResNet50":       (keras.applications.ResNet50,       keras.applications.resnet.preprocess_input,       "conv5_block3_out"),
    "EfficientNetB0": (keras.applications.EfficientNetB0, keras.applications.efficientnet.preprocess_input, "top_activation"),
}

# Same-family fallback if EfficientNetB0's legacy .h5 weights refuse to load in this environment
# (known conda+pip Keras 3 issue: the stem is mis-built and the ImageNet weights fail on a shape check —
# it already occurred on this machine in Case Study 3). Keeps the notebook spec-compliant (an EfficientNet
# backbone, same head, same 'top_activation' feature layer) and lets it finish end-to-end.
FALLBACK = {
    "EfficientNetB0": ("EfficientNetV2B0", keras.applications.EfficientNetV2B0, keras.applications.efficientnet_v2.preprocess_input, "top_activation"),
}

def _load_backbone(name, img_size):
    '''ImageNet weights with one retry after clearing a possibly corrupt cache, then the same-family
    fallback; in the FAST smoke test (offline sandbox) a failed download falls back to random init.'''
    import glob
    ctor, pre, feat = BACKBONES[name]
    candidates = [(name, ctor, pre, feat)] + ([FALLBACK[name]] if name in FALLBACK else [])
    last = None
    for label, c, p, f in candidates:
        for attempt in (1, 2):
            try:
                base = c(include_top=False, weights="imagenet", input_shape=(img_size, img_size, 3))
                if label != name:
                    print(f"[info] {name} ImageNet weights unavailable in this environment -> using {label} (same family).")
                return base, p, f, ("imagenet" if label == name else f"imagenet ({label})")
            except Exception as e:
                last = e
                print(f"[{label}] ImageNet weight load failed (attempt {attempt}): {type(e).__name__}: {str(e)[:120]}")
                for fpath in glob.glob(os.path.expanduser(f"~/.keras/models/*{label.lower()}*")):
                    try: os.remove(fpath)
                    except OSError: pass
    if RUN_PROFILE == "FAST":
        print(f"[{name}] FAST profile: continuing with random weights (smoke test only).")
        return ctor(include_top=False, weights=None, input_shape=(img_size, img_size, 3)), pre, feat, "random"
    raise RuntimeError(f"Could not load ImageNet weights for {name} (nor its fallback): {last}")

def build_transfer_model(name, img_size=IMG_SIZE, dropout=DROPOUT, trainable_base=False):
    '''Models 2-4. Frozen ImageNet backbone + GAP -> Dense(128, relu) -> Dropout -> Dense(10, softmax).'''
    base, preprocess, feat_layer, weights_used = _load_backbone(name, img_size)
    base.trainable = trainable_base
    inputs = keras.Input(shape=(img_size, img_size, 3), name="input")
    x = layers.Lambda(preprocess, name="preprocess_input")(inputs)
    feat = base(x, training=False)                     # training=False keeps BatchNorm statistics frozen
    x = layers.GlobalAveragePooling2D(name="gap")(feat)
    x = layers.Dense(128, activation="relu", name="fc1")(x)
    x = layers.Dropout(dropout, name="dropout")(x)
    outputs = layers.Dense(NUM_CLASSES, activation="softmax", name="predictions")(x)
    model = keras.Model(inputs, outputs, name=name)
    model._feature_layer = feat_layer
    model._base = base
    model._feature_tensor = feat
    model._weights_used = weights_used
    return model

def count_params(model):
    total = int(model.count_params())
    trainable = int(sum(int(np.prod(w.shape)) for w in model.trainable_weights))
    return total, trainable

def compile_model(model, lr=LEARNING_RATE):
    # Adam (default 1e-3) + categorical cross-entropy for a 10-class softmax with one-hot targets (spec §4).
    model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr),
                  loss="categorical_crossentropy", metrics=["accuracy"])
    return model

MODEL_BUILDERS = {
    "SimpleCNN":      lambda **kw: build_simple_cnn(**kw),
    "VGG16":          lambda **kw: build_transfer_model("VGG16", **kw),
    "ResNet50":       lambda **kw: build_transfer_model("ResNet50", **kw),
    "EfficientNetB0": lambda **kw: build_transfer_model("EfficientNetB0", **kw),
}

# Build once to document the architectures and parameter counts (rubric §4). Models are rebuilt fresh in Part 3.
arch_rows = []
for name, builder in MODEL_BUILDERS.items():
    m = builder(); total, trainable = count_params(m)
    arch_rows.append(dict(model=name, total_params=total, trainable_params=trainable,
                          frozen_params=total - trainable, feature_layer=m._feature_layer,
                          weights=getattr(m, "_weights_used", "scratch")))
    if name == "SimpleCNN":
        m.summary()
    else:
        print(f"{name}: backbone layers={len(m._base.layers)} | feature layer={m._feature_layer} "
              f"| output={m._base.output.shape[1:]} | weights={m._weights_used}")
    del m
keras.backend.clear_session(); gc.collect()
arch_df = pd.DataFrame(arch_rows); display(arch_df)

Model: "SimpleCNN"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ input (InputLayer)              │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ rescale_0_1 (Rescaling)         │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ standardize (Normalization)     │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv1 (Conv2D)                  │ (None, 224, 224, 32)   │           896 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool1 (MaxPooling2D)            │ (None, 56, 56, 32)     │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv2 (Conv2D)                  │ (None, 56, 56, 64)     │        18,496 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool2 (MaxPooling2D)            │ (None, 14, 14, 64)     │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ last_conv (Conv2D)              │ (None, 14, 14, 128)    │        73,856 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool3 (MaxPooling2D)            │ (None, 7, 7, 128)      │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ flatten (Flatten)               │ (None, 6272)           │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ fc1 (Dense)                     │ (None, 128)            │       802,944 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout (Dropout)               │ (None, 128)            │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ predictions (Dense)             │ (None, 10)             │         1,290 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 897,482 (3.42 MB)

 Trainable params: 897,482 (3.42 MB)

 Non-trainable params: 0 (0.00 B)

VGG16: backbone layers=19 | feature layer=block5_pool | output=(7, 7, 512) | weights=imagenet
ResNet50: backbone layers=175 | feature layer=conv5_block3_out | output=(7, 7, 2048) | weights=imagenet
EfficientNetB0: backbone layers=238 | feature layer=top_activation | output=(7, 7, 1280) | weights=imagenet

# ---- Part 3: Unified training driver (Parts 3, 6 and 7 all go through this function) ----
HISTORIES, MODELS = {}, {}

class EpochTimer(keras.callbacks.Callback):
    def on_train_begin(self, logs=None): self.epoch_times = []
    def on_epoch_begin(self, epoch, logs=None): self._t = time.time()
    def on_epoch_end(self, epoch, logs=None): self.epoch_times.append(time.time() - self._t)

def make_callbacks(es_patience=ES_PATIENCE, rlr_patience=RLR_PATIENCE, rlr_factor=RLR_FACTOR):
    return [keras.callbacks.EarlyStopping(monitor="val_accuracy", patience=es_patience, restore_best_weights=True, verbose=1),
            keras.callbacks.ReduceLROnPlateau(monitor="val_loss", factor=rlr_factor, patience=rlr_patience, min_lr=1e-6, verbose=1),
            EpochTimer()]

def gpu_mem_mb():
    '''Peak device memory when the runtime exposes it (CUDA); Metal/CPU report -1 (measured via RSS instead).'''
    try:
        return round(tf.config.experimental.get_memory_info("GPU:0")["peak"] / 2**20, 1)
    except Exception:
        return -1.0

def rss_mb():
    try:
        import resource
        r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        return round(r / 2**20 if platform.system() == "Darwin" else r / 2**10, 1)   # bytes on macOS, KB on Linux
    except Exception:
        return -1.0

def train_model(name, train_ds, val_ds, epochs=EPOCHS, lr=LEARNING_RATE, build_kwargs=None,
                callbacks=None, verbose=2, tag=None):
    '''Fresh build -> compile -> fit with callbacks. Returns (model, record).'''
    reseed()                                      # identical init + shuffling order for every run
    keras.backend.clear_session(); gc.collect()
    model = compile_model(MODEL_BUILDERS[name](**(build_kwargs or {})), lr=lr)
    total, trainable = count_params(model)
    cbs = callbacks if callbacks is not None else make_callbacks()
    timer = next((c for c in cbs if isinstance(c, EpochTimer)), None)
    try: tf.config.experimental.reset_memory_stats("GPU:0")
    except Exception: pass
    rss_before = rss_mb()
    t0 = time.time()
    hist = model.fit(train_ds, validation_data=val_ds, epochs=epochs, callbacks=cbs, verbose=verbose)
    train_time = time.time() - t0
    h = hist.history
    ran = len(h["loss"]); best_ep = int(np.argmax(h["val_accuracy"])) + 1
    es = next((c for c in cbs if isinstance(c, keras.callbacks.EarlyStopping)), None)
    stopped_early = bool(es is not None and getattr(es, "stopped_epoch", 0) > 0)
    reason = (f"EarlyStopping: val_accuracy did not improve for {ES_PATIENCE} epochs after epoch {best_ep}; best weights restored"
              if stopped_early else f"Reached the epoch ceiling ({epochs}); best epoch {best_ep} weights restored")
    lr_final = float(keras.backend.get_value(model.optimizer.learning_rate)) if hasattr(model.optimizer, "learning_rate") else lr
    rec = dict(model=name, tag=tag or name, epochs_run=ran, epochs_max=epochs, best_epoch=best_ep,
               stopped_early=stopped_early, stop_reason=reason,
               best_val_accuracy=float(max(h["val_accuracy"])), final_train_accuracy=float(h["accuracy"][-1]),
               final_val_loss=float(h["val_loss"][-1]), train_time_s=round(train_time, 1),
               mean_epoch_time_s=round(float(np.mean(timer.epoch_times)), 1) if timer else None,
               lr_start=lr, lr_final=lr_final, lr_reductions=int(round(math.log(max(lr_final, 1e-12) / lr, RLR_FACTOR))) if lr_final < lr else 0,
               total_params=total, trainable_params=trainable,
               peak_gpu_mem_mb=gpu_mem_mb(), rss_delta_mb=round(rss_mb() - rss_before, 1), history={k: [float(v) for v in vals] for k, vals in h.items()})
    print(f"\n[{rec['tag']}] {ran}/{epochs} epochs | best val acc {rec['best_val_accuracy']:.4f} @ epoch {best_ep} "
          f"| {train_time/60:.1f} min ({rec['mean_epoch_time_s']} s/epoch) | {reason}")
    return model, rec

# ---- Train the four required models ----
for name in MODEL_BUILDERS:
    print(f"\n{'='*30} {name} {'='*30}")
    model, rec = train_model(name, train_ds, val_ds)
    MODELS[name], HISTORIES[name] = model, rec
    os.makedirs(os.path.join(RESULTS_DIR, "models"), exist_ok=True)
    model.save_weights(os.path.join(RESULTS_DIR, "models", f"{name}.weights.h5"))   # reload: MODEL_BUILDERS[name]().load_weights(...)
    RESULTS["models"][name] = {k: v for k, v in rec.items() if k != "history"}
    RESULTS["models"][name]["backbone_weights"] = getattr(model, "_weights_used", "scratch")
    RESULTS["models"][name]["history"] = rec["history"]

============================== SimpleCNN ==============================
Epoch 1/15
1250/1250 - 42s - 33ms/step - accuracy: 0.3014 - loss: 1.9397 - val_accuracy: 0.4171 - val_loss: 1.6513 - learning_rate: 0.0010
Epoch 2/15
1250/1250 - 40s - 32ms/step - accuracy: 0.3673 - loss: 1.7783 - val_accuracy: 0.4455 - val_loss: 1.5541 - learning_rate: 0.0010
Epoch 3/15
1250/1250 - 40s - 32ms/step - accuracy: 0.3824 - loss: 1.8142 - val_accuracy: 0.4281 - val_loss: 1.8687 - learning_rate: 0.0010
Epoch 4/15
1250/1250 - 40s - 32ms/step - accuracy: 0.3891 - loss: 1.9292 - val_accuracy: 0.4718 - val_loss: 1.6850 - learning_rate: 0.0010
Epoch 5/15

Epoch 5: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.
1250/1250 - 41s - 33ms/step - accuracy: 0.3864 - loss: 2.1665 - val_accuracy: 0.4660 - val_loss: 1.9022 - learning_rate: 0.0010
Epoch 6/15
1250/1250 - 40s - 32ms/step - accuracy: 0.4731 - loss: 1.5381 - val_accuracy: 0.5412 - val_loss: 1.3038 - learning_rate: 2.0000e-04
Epoch 7/15
1250/1250 - 40s - 32ms/step - accuracy: 0.4739 - loss: 1.5363 - val_accuracy: 0.5815 - val_loss: 1.1980 - learning_rate: 2.0000e-04
Epoch 8/15
1250/1250 - 42s - 34ms/step - accuracy: 0.4643 - loss: 1.5842 - val_accuracy: 0.5239 - val_loss: 1.4534 - learning_rate: 2.0000e-04
Epoch 9/15
1250/1250 - 43s - 34ms/step - accuracy: 0.4656 - loss: 1.6000 - val_accuracy: 0.5732 - val_loss: 1.2878 - learning_rate: 2.0000e-04
Epoch 10/15

Epoch 10: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05.
1250/1250 - 41s - 33ms/step - accuracy: 0.4678 - loss: 1.6282 - val_accuracy: 0.5883 - val_loss: 1.2091 - learning_rate: 2.0000e-04
Epoch 11/15
1250/1250 - 41s - 33ms/step - accuracy: 0.5157 - loss: 1.3797 - val_accuracy: 0.5992 - val_loss: 1.1500 - learning_rate: 4.0000e-05
Epoch 12/15
1250/1250 - 40s - 32ms/step - accuracy: 0.5265 - loss: 1.3498 - val_accuracy: 0.6006 - val_loss: 1.1467 - learning_rate: 4.0000e-05
Epoch 13/15
1250/1250 - 40s - 32ms/step - accuracy: 0.5289 - loss: 1.3461 - val_accuracy: 0.6122 - val_loss: 1.1256 - learning_rate: 4.0000e-05
Epoch 14/15
1250/1250 - 40s - 32ms/step - accuracy: 0.5244 - loss: 1.3490 - val_accuracy: 0.6042 - val_loss: 1.1337 - learning_rate: 4.0000e-05
Epoch 15/15
1250/1250 - 41s - 32ms/step - accuracy: 0.5242 - loss: 1.3511 - val_accuracy: 0.5937 - val_loss: 1.1742 - learning_rate: 4.0000e-05
Restoring model weights from the end of the best epoch: 13.

[SimpleCNN] 15/15 epochs | best val acc 0.6122 @ epoch 13 | 10.2 min (40.8 s/epoch) | Reached the epoch ceiling (15); best epoch 13 weights restored

============================== VGG16 ==============================
Epoch 1/15
1250/1250 - 303s - 243ms/step - accuracy: 0.5003 - loss: 2.6068 - val_accuracy: 0.7671 - val_loss: 0.7991 - learning_rate: 0.0010
Epoch 2/15
1250/1250 - 329s - 264ms/step - accuracy: 0.6378 - loss: 1.0890 - val_accuracy: 0.8052 - val_loss: 0.6000 - learning_rate: 0.0010
Epoch 3/15
1250/1250 - 334s - 267ms/step - accuracy: 0.6646 - loss: 0.9949 - val_accuracy: 0.7858 - val_loss: 0.6903 - learning_rate: 0.0010
Epoch 4/15
1250/1250 - 333s - 267ms/step - accuracy: 0.6745 - loss: 0.9567 - val_accuracy: 0.7904 - val_loss: 0.6666 - learning_rate: 0.0010
Epoch 5/15

Epoch 5: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.
1250/1250 - 336s - 269ms/step - accuracy: 0.6809 - loss: 0.9561 - val_accuracy: 0.7985 - val_loss: 0.6588 - learning_rate: 0.0010
Epoch 6/15
1250/1250 - 338s - 270ms/step - accuracy: 0.7158 - loss: 0.8346 - val_accuracy: 0.8118 - val_loss: 0.5886 - learning_rate: 2.0000e-04
Epoch 7/15
1250/1250 - 337s - 270ms/step - accuracy: 0.7248 - loss: 0.8105 - val_accuracy: 0.8237 - val_loss: 0.5478 - learning_rate: 2.0000e-04
Epoch 8/15
1250/1250 - 343s - 274ms/step - accuracy: 0.7269 - loss: 0.8085 - val_accuracy: 0.8077 - val_loss: 0.6026 - learning_rate: 2.0000e-04
Epoch 9/15
1250/1250 - 343s - 274ms/step - accuracy: 0.7283 - loss: 0.7970 - val_accuracy: 0.8211 - val_loss: 0.5621 - learning_rate: 2.0000e-04
Epoch 10/15

Epoch 10: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05.
1250/1250 - 343s - 274ms/step - accuracy: 0.7265 - loss: 0.8007 - val_accuracy: 0.8186 - val_loss: 0.5683 - learning_rate: 2.0000e-04
Epoch 11/15
1250/1250 - 341s - 273ms/step - accuracy: 0.7350 - loss: 0.7783 - val_accuracy: 0.8226 - val_loss: 0.5569 - learning_rate: 4.0000e-05
Epoch 12/15
1250/1250 - 337s - 270ms/step - accuracy: 0.7344 - loss: 0.7836 - val_accuracy: 0.8238 - val_loss: 0.5498 - learning_rate: 4.0000e-05
Epoch 13/15

Epoch 13: ReduceLROnPlateau reducing learning rate to 8.000000525498762e-06.
1250/1250 - 340s - 272ms/step - accuracy: 0.7359 - loss: 0.7771 - val_accuracy: 0.8196 - val_loss: 0.5580 - learning_rate: 4.0000e-05
Epoch 14/15
1250/1250 - 339s - 271ms/step - accuracy: 0.7363 - loss: 0.7758 - val_accuracy: 0.8241 - val_loss: 0.5496 - learning_rate: 8.0000e-06
Epoch 15/15
1250/1250 - 339s - 271ms/step - accuracy: 0.7375 - loss: 0.7695 - val_accuracy: 0.8219 - val_loss: 0.5548 - learning_rate: 8.0000e-06
Restoring model weights from the end of the best epoch: 14.

[VGG16] 15/15 epochs | best val acc 0.8241 @ epoch 14 | 83.9 min (335.7 s/epoch) | Reached the epoch ceiling (15); best epoch 14 weights restored

============================== ResNet50 ==============================
Epoch 1/15
1250/1250 - 217s - 174ms/step - accuracy: 0.5931 - loss: 2.2186 - val_accuracy: 0.8412 - val_loss: 0.5234 - learning_rate: 0.0010
Epoch 2/15
1250/1250 - 217s - 174ms/step - accuracy: 0.7046 - loss: 0.9514 - val_accuracy: 0.8757 - val_loss: 0.3575 - learning_rate: 0.0010
Epoch 3/15
1250/1250 - 216s - 173ms/step - accuracy: 0.7279 - loss: 0.8102 - val_accuracy: 0.8659 - val_loss: 0.3954 - learning_rate: 0.0010
Epoch 4/15
1250/1250 - 214s - 171ms/step - accuracy: 0.7404 - loss: 0.7608 - val_accuracy: 0.8785 - val_loss: 0.3550 - learning_rate: 0.0010
Epoch 5/15
1250/1250 - 220s - 176ms/step - accuracy: 0.7494 - loss: 0.7444 - val_accuracy: 0.8831 - val_loss: 0.3511 - learning_rate: 0.0010
Epoch 6/15
1250/1250 - 225s - 180ms/step - accuracy: 0.7510 - loss: 0.7411 - val_accuracy: 0.8720 - val_loss: 0.3740 - learning_rate: 0.0010
Epoch 7/15
1250/1250 - 204s - 164ms/step - accuracy: 0.7531 - loss: 0.7385 - val_accuracy: 0.8728 - val_loss: 0.3777 - learning_rate: 0.0010
Epoch 8/15
1250/1250 - 194s - 155ms/step - accuracy: 0.7588 - loss: 0.7276 - val_accuracy: 0.8908 - val_loss: 0.3458 - learning_rate: 0.0010
Epoch 9/15
1250/1250 - 189s - 151ms/step - accuracy: 0.7623 - loss: 0.7193 - val_accuracy: 0.8896 - val_loss: 0.3292 - learning_rate: 0.0010
Epoch 10/15
1250/1250 - 185s - 148ms/step - accuracy: 0.7595 - loss: 0.7247 - val_accuracy: 0.8858 - val_loss: 0.3504 - learning_rate: 0.0010
Epoch 11/15
1250/1250 - 185s - 148ms/step - accuracy: 0.7658 - loss: 0.7088 - val_accuracy: 0.8783 - val_loss: 0.3813 - learning_rate: 0.0010
Epoch 12/15

Epoch 12: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.
1250/1250 - 182s - 145ms/step - accuracy: 0.7648 - loss: 0.7142 - val_accuracy: 0.8907 - val_loss: 0.3396 - learning_rate: 0.0010
Epoch 13/15
1250/1250 - 181s - 145ms/step - accuracy: 0.7983 - loss: 0.5944 - val_accuracy: 0.8900 - val_loss: 0.3215 - learning_rate: 2.0000e-04
Epoch 13: early stopping
Restoring model weights from the end of the best epoch: 8.

[ResNet50] 13/15 epochs | best val acc 0.8908 @ epoch 8 | 43.8 min (202.3 s/epoch) | EarlyStopping: val_accuracy did not improve for 5 epochs after epoch 8; best weights restored

============================== EfficientNetB0 ==============================
Epoch 1/15
1250/1250 - 125s - 100ms/step - accuracy: 0.6532 - loss: 1.0071 - val_accuracy: 0.8654 - val_loss: 0.4099 - learning_rate: 0.0010
Epoch 2/15
1250/1250 - 115s - 92ms/step - accuracy: 0.7219 - loss: 0.8079 - val_accuracy: 0.8699 - val_loss: 0.3941 - learning_rate: 0.0010
Epoch 3/15
1250/1250 - 117s - 93ms/step - accuracy: 0.7348 - loss: 0.7614 - val_accuracy: 0.8736 - val_loss: 0.3821 - learning_rate: 0.0010
Epoch 4/15
1250/1250 - 115s - 92ms/step - accuracy: 0.7414 - loss: 0.7514 - val_accuracy: 0.8760 - val_loss: 0.3777 - learning_rate: 0.0010
Epoch 5/15
1250/1250 - 114s - 91ms/step - accuracy: 0.7469 - loss: 0.7332 - val_accuracy: 0.8765 - val_loss: 0.3733 - learning_rate: 0.0010
Epoch 6/15
1250/1250 - 116s - 93ms/step - accuracy: 0.7506 - loss: 0.7230 - val_accuracy: 0.8801 - val_loss: 0.3632 - learning_rate: 0.0010
Epoch 7/15
1250/1250 - 113s - 91ms/step - accuracy: 0.7519 - loss: 0.7179 - val_accuracy: 0.8771 - val_loss: 0.3661 - learning_rate: 0.0010
Epoch 8/15
1250/1250 - 114s - 91ms/step - accuracy: 0.7580 - loss: 0.7109 - val_accuracy: 0.8829 - val_loss: 0.3624 - learning_rate: 0.0010
Epoch 9/15
1250/1250 - 113s - 90ms/step - accuracy: 0.7546 - loss: 0.7111 - val_accuracy: 0.8798 - val_loss: 0.3706 - learning_rate: 0.0010
Epoch 10/15
1250/1250 - 114s - 91ms/step - accuracy: 0.7581 - loss: 0.7048 - val_accuracy: 0.8842 - val_loss: 0.3586 - learning_rate: 0.0010
Epoch 11/15
1250/1250 - 116s - 92ms/step - accuracy: 0.7603 - loss: 0.7015 - val_accuracy: 0.8867 - val_loss: 0.3451 - learning_rate: 0.0010
Epoch 12/15
1250/1250 - 113s - 90ms/step - accuracy: 0.7610 - loss: 0.6980 - val_accuracy: 0.8871 - val_loss: 0.3470 - learning_rate: 0.0010
Epoch 13/15
1250/1250 - 114s - 91ms/step - accuracy: 0.7608 - loss: 0.6915 - val_accuracy: 0.8870 - val_loss: 0.3568 - learning_rate: 0.0010
Epoch 14/15

Epoch 14: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.
1250/1250 - 112s - 89ms/step - accuracy: 0.7632 - loss: 0.6948 - val_accuracy: 0.8799 - val_loss: 0.3698 - learning_rate: 0.0010
Epoch 15/15
1250/1250 - 113s - 91ms/step - accuracy: 0.7766 - loss: 0.6512 - val_accuracy: 0.8849 - val_loss: 0.3572 - learning_rate: 2.0000e-04
Restoring model weights from the end of the best epoch: 12.

[EfficientNetB0] 15/15 epochs | best val acc 0.8871 @ epoch 12 | 28.7 min (114.8 s/epoch) | Reached the epoch ceiling (15); best epoch 12 weights restored

# ---- Training curves (accuracy + loss, train vs validation) for the four models ----
def plot_histories(hists, title, fname):
    fig, axes = plt.subplots(1, 2, figsize=(13, 4.2))
    for name, rec in hists.items():
        h = rec["history"]; ep = range(1, len(h["loss"]) + 1)
        axes[0].plot(ep, h["accuracy"], "--", alpha=.6, label=f"{name} train"); axes[0].plot(ep, h["val_accuracy"], "-", label=f"{name} val")
        axes[1].plot(ep, h["loss"], "--", alpha=.6, label=f"{name} train");     axes[1].plot(ep, h["val_loss"], "-", label=f"{name} val")
    axes[0].set_title("Accuracy"); axes[1].set_title("Loss (categorical cross-entropy)")
    for ax in axes: ax.set_xlabel("epoch"); ax.grid(alpha=.3); ax.legend(fontsize=7, ncol=2)
    plt.suptitle(title); plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, fname), dpi=120); plt.show()

plot_histories(HISTORIES, "Training curves — four required models", "training_curves.png")

train_df = pd.DataFrame([{k: v for k, v in r.items() if k not in ("history",)} for r in HISTORIES.values()])
display(train_df[["model", "epochs_run", "epochs_max", "best_epoch", "stopped_early", "best_val_accuracy",
                  "final_train_accuracy", "train_time_s", "mean_epoch_time_s", "lr_final", "lr_reductions"]])
for r in HISTORIES.values():
    print(f"- {r['model']}: {r['stop_reason']}")

- SimpleCNN: Reached the epoch ceiling (15); best epoch 13 weights restored
- VGG16: Reached the epoch ceiling (15); best epoch 14 weights restored
- ResNet50: EarlyStopping: val_accuracy did not improve for 5 epochs after epoch 8; best weights restored
- EfficientNetB0: Reached the epoch ceiling (15); best epoch 12 weights restored

# ---- Part 4: Common evaluation ----
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, classification_report, precision_score, recall_score
PROBS = {}

def evaluate_model(model, name, test_ds, y_true, tag=None):
    _ = model.predict(test_ds.take(1), verbose=0)          # warm-up: exclude graph tracing from the timing
    t0 = time.time(); probs = model.predict(test_ds, verbose=0); infer = time.time() - t0
    y_pred = probs.argmax(1)
    total, trainable = count_params(model)
    rec = dict(model=name, tag=tag or name,
               test_accuracy=float(accuracy_score(y_true, y_pred)),
               f1_weighted=float(f1_score(y_true, y_pred, average="weighted")),
               f1_macro=float(f1_score(y_true, y_pred, average="macro")),
               precision_weighted=float(precision_score(y_true, y_pred, average="weighted")),
               recall_weighted=float(recall_score(y_true, y_pred, average="weighted")),
               test_loss=float(keras.losses.categorical_crossentropy(keras.utils.to_categorical(y_true, NUM_CLASSES), probs).numpy().mean()),
               inference_time_s=round(infer, 2), inference_ms_per_image=round(1000 * infer / len(y_true), 3),
               n_test=int(len(y_true)), total_params=total, trainable_params=trainable,
               per_class_f1={CLASS_NAMES[i]: float(v) for i, v in enumerate(f1_score(y_true, y_pred, average=None))},
               confusion_matrix=confusion_matrix(y_true, y_pred).tolist())
    return rec, probs

EVAL = {}
for name, model in MODELS.items():
    rec, probs = evaluate_model(model, name, test_ds, y_test)
    PROBS[name] = probs; EVAL[name] = rec
    RESULTS["models"][name].update({k: v for k, v in rec.items() if k not in ("model", "tag")})
    print(f"{name:15s} acc={rec['test_accuracy']:.4f} | F1w={rec['f1_weighted']:.4f} | loss={rec['test_loss']:.4f} "
          f"| inference {rec['inference_time_s']}s ({rec['inference_ms_per_image']} ms/img)")

# ---- Confusion matrices ----
fig, axes = plt.subplots(1, 4, figsize=(22, 5))
for ax, (name, rec) in zip(axes, EVAL.items()):
    sns.heatmap(np.array(rec["confusion_matrix"]), annot=True, fmt="d", cmap="Blues", cbar=False, ax=ax,
                xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES if ax is axes[0] else False, annot_kws={"size": 7})
    ax.set_title(f"{name}\nacc={rec['test_accuracy']:.3f}  F1w={rec['f1_weighted']:.3f}"); ax.set_xlabel("predicted")
    ax.tick_params(axis="x", rotation=60, labelsize=8); ax.tick_params(axis="y", rotation=0, labelsize=8)
axes[0].set_ylabel("true"); plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "confusion_matrices.png"), dpi=120); plt.show()

# ---- Comparison table (deliverable: cnn_lab_comparison.csv) ----
comp = pd.DataFrame([{
    "model": n, "test_accuracy": round(e["test_accuracy"], 4), "f1_weighted": round(e["f1_weighted"], 4),
    "test_loss": round(e["test_loss"], 4), "best_val_accuracy": round(HISTORIES[n]["best_val_accuracy"], 4),
    "epochs_run": HISTORIES[n]["epochs_run"], "best_epoch": HISTORIES[n]["best_epoch"], "stopped_early": HISTORIES[n]["stopped_early"],
    "train_time_s": HISTORIES[n]["train_time_s"], "inference_time_s": e["inference_time_s"], "inference_ms_per_image": e["inference_ms_per_image"],
    "total_params": e["total_params"], "trainable_params": e["trainable_params"], "feature_layer": arch_df.set_index("model").loc[n, "feature_layer"],
} for n, e in EVAL.items()])
# Efficiency views: accuracy per million parameters and per training minute
comp["acc_per_Mparam"] = (comp["test_accuracy"] / (comp["total_params"] / 1e6)).round(4)
comp["acc_per_train_min"] = (comp["test_accuracy"] / (comp["train_time_s"] / 60)).round(4)
display(comp.set_index("model"))

# ---- Best model: primary criterion test accuracy, F1 as tiebreak; efficiency reported alongside ----
best = comp.sort_values(["test_accuracy", "f1_weighted"], ascending=False).iloc[0]["model"]
fastest = comp.sort_values("inference_ms_per_image").iloc[0]["model"]
RESULTS["best_model"] = dict(name=best, criterion="highest test accuracy (weighted F1 tiebreak)",
                             fastest_inference=fastest)
print(f"\nBest-performing model: {best} | fastest inference: {fastest}")

# Per-class report for the best model
print(classification_report(y_test, PROBS[best].argmax(1), target_names=CLASS_NAMES, digits=3))

# Persist a first version of the deliverables now; Part 9 rewrites them with every section included.
comp.to_csv(os.path.join(RESULTS_DIR, "cnn_lab_comparison.csv"), index=False)
with open(os.path.join(RESULTS_DIR, "cnn_lab_results.json"), "w") as fh: json.dump(RESULTS, fh, indent=2, default=str)

SimpleCNN       acc=0.6158 | F1w=0.5927 | loss=1.1152 | inference 3.02s (0.302 ms/img)
VGG16           acc=0.8165 | F1w=0.8144 | loss=0.5643 | inference 51.11s (5.111 ms/img)
ResNet50        acc=0.8856 | F1w=0.8849 | loss=0.3546 | inference 35.49s (3.549 ms/img)
EfficientNetB0  acc=0.8891 | F1w=0.8888 | loss=0.3400 | inference 22.7s (2.27 ms/img)

Best-performing model: EfficientNetB0 | fastest inference: SimpleCNN
              precision    recall  f1-score   support

    airplane      0.895     0.924     0.909      1000
  automobile      0.927     0.959     0.943      1000
        bird      0.923     0.808     0.862      1000
         cat      0.794     0.822     0.808      1000
        deer      0.893     0.819     0.854      1000
         dog      0.901     0.826     0.862      1000
        frog      0.820     0.952     0.881      1000
       horse      0.885     0.930     0.907      1000
        ship      0.958     0.911     0.934      1000
       truck      0.916     0.940     0.928      1000

    accuracy                          0.889     10000
   macro avg      0.891     0.889     0.889     10000
weighted avg      0.891     0.889     0.889     10000

# ---- Trade-off views: accuracy vs parameters, accuracy vs training time, accuracy vs inference latency ----
fig, axes = plt.subplots(1, 3, figsize=(16, 4.3))
for ax, xcol, xlabel, logx in [(axes[0], "total_params", "total parameters", True),
                               (axes[1], "train_time_s", "training time (s)", False),
                               (axes[2], "inference_ms_per_image", "inference latency (ms / image)", False)]:
    ax.scatter(comp[xcol], comp["test_accuracy"], s=70)
    for _, r in comp.iterrows(): ax.annotate(r["model"], (r[xcol], r["test_accuracy"]), textcoords="offset points", xytext=(5, 4), fontsize=8)
    ax.set_xlabel(xlabel); ax.set_ylabel("test accuracy"); ax.grid(alpha=.3)
    if logx: ax.set_xscale("log")
plt.suptitle("Accuracy vs. efficiency trade-offs"); plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "tradeoffs.png"), dpi=120); plt.show()

# Per-class F1 across models: where the architectures disagree
pcf = pd.DataFrame({n: e["per_class_f1"] for n, e in EVAL.items()})
ax = pcf.plot(kind="bar", figsize=(13, 4), width=.8); ax.set_ylabel("F1"); ax.set_title("Per-class F1 by model"); ax.grid(axis="y", alpha=.3)
plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "per_class_f1.png"), dpi=120); plt.show()
display(pcf.round(3))

# ---- Part 5: Feature extraction + statistics + t-SNE ----
from sklearn.manifold import TSNE
from sklearn.metrics import silhouette_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score

def feature_extractor(model):
    '''Sub-model ending at the designated last convolutional layer.'''
    if model._base is None:                                    # Simple CNN: named layer in the outer graph
        return keras.Model(model.input, model.get_layer(model._feature_layer).output, name=f"{model.name}_features")
    return keras.Model(model.input, model._feature_tensor, name=f"{model.name}_features")   # TL: backbone output

reseed()
feat_idx, _ = train_test_split(np.arange(len(X_test)), train_size=min(P["FEAT_N"], len(X_test) - 10), stratify=y_test, random_state=SEED)
feat_ds = make_dataset(X_test[feat_idx], y_test[feat_idx]); y_feat = y_test[feat_idx]

FEATS, feat_rows = {}, []
for name, model in MODELS.items():
    ext = feature_extractor(model)
    t0 = time.time(); fmap = ext.predict(feat_ds, verbose=0); t_ext = time.time() - t0
    pooled = fmap.mean(axis=(1, 2))                            # GAP -> (N, C) vectors for t-SNE
    FEATS[name] = pooled
    flat = fmap.reshape(len(fmap), -1)
    dead = float((fmap.reshape(-1, fmap.shape[-1]).max(axis=0) <= 1e-6).mean())
    rec = dict(model=name, layer=model._feature_layer, spatial_shape=list(fmap.shape[1:]), feature_dim=int(flat.shape[1]),
               pooled_dim=int(pooled.shape[1]), sparsity_pct=round(100 * float((fmap == 0).mean()), 2),
               near_zero_pct=round(100 * float((np.abs(fmap) < 1e-3).mean()), 2),
               mean=round(float(fmap.mean()), 4), std=round(float(fmap.std()), 4), max=round(float(fmap.max()), 3),
               min=round(float(fmap.min()), 3), dead_channels_pct=round(100 * dead, 2), extraction_s=round(t_ext, 2))
    feat_rows.append(rec); del fmap, flat, ext; gc.collect()

feat_df = pd.DataFrame(feat_rows); display(feat_df.set_index("model"))

# ---- t-SNE + objective separability ----
tsne_n = min(P["TSNE_N"], len(y_feat))
fig, axes = plt.subplots(1, 4, figsize=(22, 5.2))
for ax, (name, pooled) in zip(axes, FEATS.items()):
    Z = pooled[:tsne_n]; yz = y_feat[:tsne_n]
    perplexity = min(30, max(5, (len(Z) - 1) // 4))
    emb = TSNE(n_components=2, perplexity=perplexity, init="pca", learning_rate="auto", random_state=SEED).fit_transform(Z)
    sil = float(silhouette_score(emb, yz))
    knn = float(cross_val_score(KNeighborsClassifier(5), emb, yz, cv=min(5, np.bincount(yz).min())).mean())
    sil_raw = float(silhouette_score(Z, yz))
    sc = ax.scatter(emb[:, 0], emb[:, 1], c=yz, cmap="tab10", s=6, alpha=.8)
    ax.set_title(f"{name} ({feat_rows[[r['model'] for r in feat_rows].index(name)]['layer']})\nsilhouette={sil:.3f}  5-NN acc={knn:.3f}")
    ax.set_xticks([]); ax.set_yticks([])
    for r in feat_rows:
        if r["model"] == name: r.update(tsne_silhouette=round(sil, 4), tsne_knn5_accuracy=round(knn, 4), raw_silhouette=round(sil_raw, 4), tsne_n=int(len(Z)), tsne_perplexity=perplexity)
handles = [plt.Line2D([], [], marker="o", ls="", color=plt.cm.tab10(i / 9), label=c) for i, c in enumerate(CLASS_NAMES)]
fig.legend(handles=handles, loc="lower center", ncol=10, fontsize=8, bbox_to_anchor=(0.5, -0.02))
plt.suptitle("t-SNE of last-conv-layer features (GAP-pooled), stratified test sample"); plt.tight_layout()
plt.savefig(os.path.join(FIG_DIR, "tsne_features.png"), dpi=120, bbox_inches="tight"); plt.show()

feat_df = pd.DataFrame(feat_rows); display(feat_df.set_index("model")[["layer", "feature_dim", "sparsity_pct", "mean", "std", "dead_channels_pct", "tsne_silhouette", "tsne_knn5_accuracy", "raw_silhouette"]])
RESULTS["features"] = {r["model"]: r for r in feat_rows}

# ---- Part 6: Experiment runner ----
reseed()
exp_tr_idx, _ = train_test_split(np.arange(len(X_train)), train_size=min(P["EXP_TRAIN_N"], len(X_train) - 10), stratify=y_train, random_state=SEED)
exp_va_idx, _ = train_test_split(np.arange(len(X_val)),   train_size=min(P["EXP_VAL_N"],   len(X_val) - 10),   stratify=y_val,   random_state=SEED)
Xe_tr, ye_tr, Xe_va, ye_va = X_train[exp_tr_idx], y_train[exp_tr_idx], X_val[exp_va_idx], y_val[exp_va_idx]
EXP_EPOCHS = P["EXP_EPOCHS"]
EXPERIMENTS = []
print(f"Experiment subset: train {len(Xe_tr)} | val {len(Xe_va)} | test {len(X_test)} | epochs {EXP_EPOCHS}")

def run_experiment(exp_id, variable, value, model_name, batch_size=BATCH_SIZE, lr=LEARNING_RATE, img_size=IMG_SIZE,
                   build_kwargs=None, epochs=EXP_EPOCHS, keep_model=False):
    tag = f"E{exp_id} {model_name} {variable}={value}"
    tr = make_dataset(Xe_tr, ye_tr, training=True, batch_size=batch_size, img_size=img_size)
    va = make_dataset(Xe_va, ye_va, batch_size=batch_size, img_size=img_size)
    te = make_dataset(X_test, y_test, batch_size=batch_size, img_size=img_size)
    bk = dict(build_kwargs or {}); bk.setdefault("img_size", img_size)
    model, trec = train_model(model_name, tr, va, epochs=epochs, lr=lr, build_kwargs=bk, verbose=0, tag=tag)
    erec, probs = evaluate_model(model, model_name, te, y_test, tag=tag)
    row = dict(experiment=exp_id, variable=variable, value=value, model=model_name, batch_size=batch_size, lr=lr, img_size=img_size,
               epochs_run=trec["epochs_run"], best_epoch=trec["best_epoch"], train_time_s=trec["train_time_s"], mean_epoch_time_s=trec["mean_epoch_time_s"],
               best_val_accuracy=round(trec["best_val_accuracy"], 4), final_train_accuracy=round(trec["final_train_accuracy"], 4),
               train_val_gap=round(trec["final_train_accuracy"] - trec["history"]["val_accuracy"][-1], 4),
               test_accuracy=round(erec["test_accuracy"], 4), f1_weighted=round(erec["f1_weighted"], 4), test_loss=round(erec["test_loss"], 4),
               inference_ms_per_image=erec["inference_ms_per_image"], total_params=erec["total_params"], trainable_params=erec["trainable_params"],
               peak_gpu_mem_mb=trec["peak_gpu_mem_mb"], rss_delta_mb=trec["rss_delta_mb"], history=trec["history"])
    EXPERIMENTS.append(row)
    print(f"  -> test acc {row['test_accuracy']:.4f} | F1w {row['f1_weighted']:.4f} | {row['train_time_s']}s | params {row['total_params']:,}")
    if keep_model: return model, row
    del model; keras.backend.clear_session(); gc.collect()
    return None, row

Experiment subset: train 10000 | val 2000 | test 10000 | epochs 5

# ---- Experiment 1: BATCH_SIZE 16 / 32 / 64 (Simple CNN) ----
for bs in (16, 32, 64):
    run_experiment(1, "batch_size", bs, "SimpleCNN", batch_size=bs)

Restoring model weights from the end of the best epoch: 4.

[E1 SimpleCNN batch_size=16] 5/5 epochs | best val acc 0.4915 @ epoch 4 | 1.4 min (17.1 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.4792 | F1w 0.4634 | 85.6s | params 897,482
Restoring model weights from the end of the best epoch: 4.

[E1 SimpleCNN batch_size=32] 5/5 epochs | best val acc 0.4675 @ epoch 4 | 0.9 min (10.7 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.4751 | F1w 0.4501 | 53.5s | params 897,482
Restoring model weights from the end of the best epoch: 5.

[E1 SimpleCNN batch_size=64] 5/5 epochs | best val acc 0.4805 @ epoch 5 | 0.7 min (8.6 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4601 | F1w 0.4526 | 42.9s | params 897,482

# ---- Experiment 2: Adam learning rate 1e-4 / 1e-3 / 1e-2 (Simple CNN) ----
E2_HIST = {}
for lr in (1e-4, 1e-3, 1e-2):
    _, row = run_experiment(2, "learning_rate", lr, "SimpleCNN", lr=lr)
    E2_HIST[f"lr={lr:g}"] = row
plot_histories(E2_HIST, "Experiment 2 — learning-rate sweep (Simple CNN, experiment subset)", "exp2_lr_curves.png")

Restoring model weights from the end of the best epoch: 5.

[E2 SimpleCNN learning_rate=0.0001] 5/5 epochs | best val acc 0.4825 @ epoch 5 | 0.9 min (10.6 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4691 | F1w 0.4561 | 53.2s | params 897,482
Restoring model weights from the end of the best epoch: 5.

[E2 SimpleCNN learning_rate=0.001] 5/5 epochs | best val acc 0.4785 @ epoch 5 | 0.9 min (10.4 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4829 | F1w 0.4633 | 52.0s | params 897,482

Epoch 5: ReduceLROnPlateau reducing learning rate to 0.0019999999552965165.
Restoring model weights from the end of the best epoch: 1.

[E2 SimpleCNN learning_rate=0.01] 5/5 epochs | best val acc 0.1000 @ epoch 1 | 0.9 min (10.7 s/epoch) | Reached the epoch ceiling (5); best epoch 1 weights restored
  -> test acc 0.1000 | F1w 0.0182 | 53.5s | params 897,482

# ---- Experiment 3: head dropout 0.3 / 0.5 / 0.8 on the three transfer-learning models ----
for tl_name in ("VGG16", "ResNet50", "EfficientNetB0"):
    for dr in (0.3, 0.5, 0.8):
        run_experiment(3, "dropout", dr, tl_name, build_kwargs=dict(dropout=dr))

Restoring model weights from the end of the best epoch: 4.

[E3 VGG16 dropout=0.3] 5/5 epochs | best val acc 0.7830 @ epoch 4 | 5.4 min (65.4 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.7710 | F1w 0.7688 | 326.9s | params 14,781,642
Restoring model weights from the end of the best epoch: 3.

[E3 VGG16 dropout=0.5] 5/5 epochs | best val acc 0.7850 @ epoch 3 | 5.7 min (68.6 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.7754 | F1w 0.7734 | 342.9s | params 14,781,642
Restoring model weights from the end of the best epoch: 5.

[E3 VGG16 dropout=0.8] 5/5 epochs | best val acc 0.7290 @ epoch 5 | 5.7 min (68.2 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.7196 | F1w 0.7102 | 341.1s | params 14,781,642
Restoring model weights from the end of the best epoch: 4.

[E3 ResNet50 dropout=0.3] 5/5 epochs | best val acc 0.8675 @ epoch 4 | 3.7 min (44.0 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.8657 | F1w 0.8650 | 220.3s | params 23,851,274
Restoring model weights from the end of the best epoch: 4.

[E3 ResNet50 dropout=0.5] 5/5 epochs | best val acc 0.8600 @ epoch 4 | 3.7 min (43.8 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.8604 | F1w 0.8594 | 219.1s | params 23,851,274
Restoring model weights from the end of the best epoch: 3.

[E3 ResNet50 dropout=0.8] 5/5 epochs | best val acc 0.8315 @ epoch 3 | 3.6 min (43.4 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.8324 | F1w 0.8310 | 217.4s | params 23,851,274
Restoring model weights from the end of the best epoch: 5.

[E3 EfficientNetB0 dropout=0.3] 5/5 epochs | best val acc 0.8570 @ epoch 5 | 2.4 min (29.4 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.8540 | F1w 0.8530 | 146.9s | params 4,214,829
Restoring model weights from the end of the best epoch: 5.

[E3 EfficientNetB0 dropout=0.5] 5/5 epochs | best val acc 0.8560 @ epoch 5 | 2.4 min (28.7 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.8602 | F1w 0.8596 | 143.5s | params 4,214,829
Restoring model weights from the end of the best epoch: 5.

[E3 EfficientNetB0 dropout=0.8] 5/5 epochs | best val acc 0.8510 @ epoch 5 | 2.4 min (29.1 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.8505 | F1w 0.8501 | 145.7s | params 4,214,829

# ---- Experiment 4: IMG_SIZE 128 / 224 / 384 on the Simple CNN and the three transfer-learning models ----
# The 384 px runs are the heaviest cells in the notebook (~3x the 224 px cost, ~18 min for VGG16); it is what the question asks for.
for exp_model in ("SimpleCNN", "VGG16", "ResNet50", "EfficientNetB0"):
    for size in (128, 224, 384):
        run_experiment(4, "img_size", size, exp_model, img_size=size)

Restoring model weights from the end of the best epoch: 4.

[E4 SimpleCNN img_size=128] 5/5 epochs | best val acc 0.4820 @ epoch 4 | 0.8 min (9.1 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.4883 | F1w 0.4610 | 45.6s | params 356,810
Restoring model weights from the end of the best epoch: 5.

[E4 SimpleCNN img_size=224] 5/5 epochs | best val acc 0.4875 @ epoch 5 | 0.9 min (10.9 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4757 | F1w 0.4637 | 54.6s | params 897,482
Restoring model weights from the end of the best epoch: 4.

[E4 SimpleCNN img_size=384] 5/5 epochs | best val acc 0.4405 @ epoch 4 | 1.9 min (23.3 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.4450 | F1w 0.4331 | 116.4s | params 2,453,962
Restoring model weights from the end of the best epoch: 3.

[E4 VGG16 img_size=128] 5/5 epochs | best val acc 0.7790 @ epoch 3 | 1.8 min (22.0 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.7905 | F1w 0.7888 | 110.3s | params 14,781,642
Restoring model weights from the end of the best epoch: 3.

[E4 VGG16 img_size=224] 5/5 epochs | best val acc 0.7870 @ epoch 3 | 5.8 min (69.4 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.7840 | F1w 0.7825 | 347.3s | params 14,781,642
Restoring model weights from the end of the best epoch: 3.

[E4 VGG16 img_size=384] 5/5 epochs | best val acc 0.6880 @ epoch 3 | 16.5 min (198.5 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.6660 | F1w 0.6550 | 992.4s | params 14,781,642
Restoring model weights from the end of the best epoch: 4.

[E4 ResNet50 img_size=128] 5/5 epochs | best val acc 0.8350 @ epoch 4 | 1.5 min (18.3 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.8424 | F1w 0.8420 | 91.4s | params 23,851,274
Restoring model weights from the end of the best epoch: 3.

[E4 ResNet50 img_size=224] 5/5 epochs | best val acc 0.8585 @ epoch 3 | 3.8 min (45.4 s/epoch) | Reached the epoch ceiling (5); best epoch 3 weights restored
  -> test acc 0.8486 | F1w 0.8467 | 227.2s | params 23,851,274
Restoring model weights from the end of the best epoch: 4.

[E4 ResNet50 img_size=384] 5/5 epochs | best val acc 0.7805 @ epoch 4 | 10.3 min (124.0 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.7805 | F1w 0.7763 | 620.2s | params 23,851,274
Restoring model weights from the end of the best epoch: 5.

[E4 EfficientNetB0 img_size=128] 5/5 epochs | best val acc 0.8665 @ epoch 5 | 1.3 min (15.4 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.8659 | F1w 0.8651 | 77.3s | params 4,214,829
Restoring model weights from the end of the best epoch: 4.

[E4 EfficientNetB0 img_size=224] 5/5 epochs | best val acc 0.8580 @ epoch 4 | 2.6 min (31.0 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.8539 | F1w 0.8535 | 155.0s | params 4,214,829
Restoring model weights from the end of the best epoch: 4.

[E4 EfficientNetB0 img_size=384] 5/5 epochs | best val acc 0.7280 @ epoch 4 | 6.6 min (79.0 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
  -> test acc 0.7278 | F1w 0.7222 | 395.0s | params 4,214,829

# ---- Experiment 5: Simple CNN + one extra Conv2D(256)->MaxPool block, with t-SNE comparison ----
m_base,  r_base  = run_experiment(5, "depth", "3 blocks", "SimpleCNN", keep_model=True)
m_deep,  r_deep  = run_experiment(5, "depth", "4 blocks (+Conv256)", "SimpleCNN", build_kwargs=dict(extra_block=True), keep_model=True)
print(f"params: 3 blocks {r_base['total_params']:,} -> 4 blocks {r_deep['total_params']:,} "
      f"({100*(r_deep['total_params']/r_base['total_params']-1):+.1f}%) | time {r_base['train_time_s']}s -> {r_deep['train_time_s']}s")
m_deep.summary()

fig, axes = plt.subplots(1, 2, figsize=(11, 5))
E5 = {}
for ax, (label, m) in zip(axes, [("3 blocks (baseline)", m_base), ("4 blocks (+Conv256)", m_deep)]):
    pooled = feature_extractor(m).predict(feat_ds, verbose=0).mean(axis=(1, 2))[:tsne_n]; yz = y_feat[:tsne_n]
    emb = TSNE(n_components=2, perplexity=min(30, max(5, (len(pooled)-1)//4)), init="pca", learning_rate="auto", random_state=SEED).fit_transform(pooled)
    sil = float(silhouette_score(emb, yz)); knn = float(cross_val_score(KNeighborsClassifier(5), emb, yz, cv=min(5, np.bincount(yz).min())).mean())
    ax.scatter(emb[:, 0], emb[:, 1], c=yz, cmap="tab10", s=6); ax.set_xticks([]); ax.set_yticks([])
    ax.set_title(f"Simple CNN {label}\nlast_conv dim={pooled.shape[1]}  silhouette={sil:.3f}  5-NN={knn:.3f}")
    E5[label] = dict(pooled_dim=int(pooled.shape[1]), tsne_silhouette=round(sil, 4), tsne_knn5_accuracy=round(knn, 4))
plt.suptitle("Experiment 5 — feature separability, original vs deeper Simple CNN"); plt.tight_layout()
plt.savefig(os.path.join(FIG_DIR, "exp5_tsne_depth.png"), dpi=120); plt.show()
RESULTS["experiments"]["E5_tsne"] = E5
del m_base, m_deep; keras.backend.clear_session(); gc.collect()

Restoring model weights from the end of the best epoch: 5.

[E5 SimpleCNN depth=3 blocks] 5/5 epochs | best val acc 0.4670 @ epoch 5 | 1.0 min (11.5 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4717 | F1w 0.4558 | 57.4s | params 897,482
Restoring model weights from the end of the best epoch: 5.

[E5 SimpleCNN depth=4 blocks (+Conv256)] 5/5 epochs | best val acc 0.4840 @ epoch 5 | 1.0 min (11.8 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
  -> test acc 0.4847 | F1w 0.4601 | 58.9s | params 684,746
params: 3 blocks 897,482 -> 4 blocks 684,746 (-23.7%) | time 57.4s -> 58.9s

Model: "SimpleCNN"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ input (InputLayer)              │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ rescale_0_1 (Rescaling)         │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ standardize (Normalization)     │ (None, 224, 224, 3)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv1 (Conv2D)                  │ (None, 224, 224, 32)   │           896 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool1 (MaxPooling2D)            │ (None, 56, 56, 32)     │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv2 (Conv2D)                  │ (None, 56, 56, 64)     │        18,496 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool2 (MaxPooling2D)            │ (None, 14, 14, 64)     │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv3 (Conv2D)                  │ (None, 14, 14, 128)    │        73,856 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool3 (MaxPooling2D)            │ (None, 7, 7, 128)      │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ last_conv (Conv2D)              │ (None, 7, 7, 256)      │       295,168 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ pool4 (MaxPooling2D)            │ (None, 3, 3, 256)      │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ flatten (Flatten)               │ (None, 2304)           │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ fc1 (Dense)                     │ (None, 128)            │       295,040 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout (Dropout)               │ (None, 128)            │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ predictions (Dense)             │ (None, 10)             │         1,290 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 2,054,240 (7.84 MB)

 Trainable params: 684,746 (2.61 MB)

 Non-trainable params: 0 (0.00 B)

 Optimizer params: 1,369,494 (5.22 MB)

0

# ---- Experiments summary: table + deltas against the in-experiment baseline of each sweep ----
exp_df = pd.DataFrame([{k: v for k, v in r.items() if k != "history"} for r in EXPERIMENTS])
BASELINE_VALUE = {1: 32, 2: 1e-3, 3: 0.5, 4: 224, 5: "3 blocks"}
exp_df[["d_test_acc", "d_train_time_s", "d_params"]] = np.nan
for (eid, mname), g in exp_df.groupby(["experiment", "model"]):
    base = g[g["value"] == BASELINE_VALUE[eid]]
    if len(base):
        b = base.iloc[0]
        exp_df.loc[g.index, "d_test_acc"] = (g["test_accuracy"] - b["test_accuracy"]).round(4)
        exp_df.loc[g.index, "d_train_time_s"] = (g["train_time_s"] - b["train_time_s"]).round(1)
        exp_df.loc[g.index, "d_params"] = g["total_params"] - b["total_params"]
cols = ["experiment", "variable", "value", "model", "test_accuracy", "d_test_acc", "f1_weighted", "best_val_accuracy", "train_val_gap",
        "train_time_s", "d_train_time_s", "mean_epoch_time_s", "inference_ms_per_image", "total_params", "d_params", "peak_gpu_mem_mb", "rss_delta_mb", "epochs_run"]
display(exp_df[[c for c in cols if c in exp_df.columns]].set_index(["experiment", "model", "value"]))
exp_df.to_csv(os.path.join(RESULTS_DIR, "parameter_experiments.csv"), index=False)
RESULTS["experiments"]["table"] = exp_df.to_dict(orient="records")
RESULTS["experiments"]["subset"] = dict(train=int(len(Xe_tr)), val=int(len(Xe_va)), test=int(len(X_test)), epochs=EXP_EPOCHS)

fig, axes = plt.subplots(1, 4, figsize=(20, 4))
for ax, (eid, title) in zip(axes, [(1, "E1 batch size"), (2, "E2 learning rate"), (3, "E3 dropout (TL heads)"), (4, "E4 image size")]):
    g = exp_df[exp_df["experiment"] == eid]
    for m, gm in g.groupby("model"):
        ax.plot(gm["value"].astype(str), gm["test_accuracy"], "o-", label=f"{m} test acc")
        ax.plot(gm["value"].astype(str), gm["best_val_accuracy"], "s--", alpha=.6, label=f"{m} best val acc")
    ax.set_title(title); ax.grid(alpha=.3); ax.legend(fontsize=7)
    ax2 = ax.twinx(); ax2.bar(g["value"].astype(str).unique(), g.groupby("value")["train_time_s"].mean().reindex(g["value"].unique()).values, alpha=.15, color="gray"); ax2.set_ylabel("train time (s)", color="gray")
plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "experiments_summary.png"), dpi=120); plt.show()

# ---- Challenge 1: Fine-tuning the best transfer-learning model ----
CH = RESULTS["challenges"]
tl_names = [n for n in EVAL if n != "SimpleCNN"]
best_tl = max(tl_names, key=lambda n: EVAL[n]["test_accuracy"])
UNFREEZE = {"VGG16": 4, "ResNet50": 12, "EfficientNetB0": 16}[best_tl]   # ~ the last convolutional block of each backbone
FT_LR, FT_EPOCHS = LEARNING_RATE / 10, max(1, min(5, EPOCHS // 3))

# Start from the trained head (Part 3 weights), unfreeze the top of the backbone, keep BatchNorm frozen.
model_ft = MODELS[best_tl]
base = model_ft._base; base.trainable = True
for layer in base.layers[:-UNFREEZE]: layer.trainable = False
for layer in base.layers:
    if isinstance(layer, layers.BatchNormalization): layer.trainable = False
tot, trn = count_params(model_ft)
print(f"Fine-tuning {best_tl}: unfroze last {UNFREEZE} backbone layers -> trainable params {trn:,} / {tot:,} | lr {FT_LR:g} | epochs {FT_EPOCHS}")
model_ft.compile(optimizer=keras.optimizers.Adam(FT_LR), loss="categorical_crossentropy", metrics=["accuracy"])
cbs = make_callbacks(es_patience=3, rlr_patience=2)
t0 = time.time(); h_ft = model_ft.fit(train_ds, validation_data=val_ds, epochs=FT_EPOCHS, callbacks=cbs, verbose=2); ft_time = time.time() - t0
rec_ft, probs_ft = evaluate_model(model_ft, best_tl, test_ds, y_test, tag=f"{best_tl} fine-tuned")
PROBS[f"{best_tl}_finetuned"] = probs_ft
CH["fine_tuning"] = dict(model=best_tl, unfrozen_layers=UNFREEZE, lr=FT_LR, epochs_run=len(h_ft.history["loss"]), train_time_s=round(ft_time, 1),
                         trainable_params=trn, before_test_accuracy=EVAL[best_tl]["test_accuracy"], after_test_accuracy=rec_ft["test_accuracy"],
                         before_f1=EVAL[best_tl]["f1_weighted"], after_f1=rec_ft["f1_weighted"], delta_accuracy=round(rec_ft["test_accuracy"] - EVAL[best_tl]["test_accuracy"], 4),
                         history={k: [float(x) for x in v] for k, v in h_ft.history.items()})
print(f"{best_tl}: frozen {EVAL[best_tl]['test_accuracy']:.4f} -> fine-tuned {rec_ft['test_accuracy']:.4f} ({CH['fine_tuning']['delta_accuracy']:+.4f}) in {ft_time/60:.1f} min")

Fine-tuning EfficientNetB0: unfroze last 16 backbone layers -> trainable params 1,286,842 / 4,214,829 | lr 0.0001 | epochs 5
Epoch 1/5
1250/1250 - 138s - 110ms/step - accuracy: 0.7814 - loss: 0.6602 - val_accuracy: 0.9019 - val_loss: 0.3154 - learning_rate: 1.0000e-04
Epoch 2/5
1250/1250 - 128s - 102ms/step - accuracy: 0.8109 - loss: 0.5665 - val_accuracy: 0.9170 - val_loss: 0.2602 - learning_rate: 1.0000e-04
Epoch 3/5
1250/1250 - 131s - 105ms/step - accuracy: 0.8295 - loss: 0.5074 - val_accuracy: 0.9166 - val_loss: 0.2538 - learning_rate: 1.0000e-04
Epoch 4/5
1250/1250 - 131s - 105ms/step - accuracy: 0.8446 - loss: 0.4597 - val_accuracy: 0.9181 - val_loss: 0.2544 - learning_rate: 1.0000e-04
Epoch 5/5
1250/1250 - 132s - 105ms/step - accuracy: 0.8541 - loss: 0.4302 - val_accuracy: 0.9292 - val_loss: 0.2186 - learning_rate: 1.0000e-04
Restoring model weights from the end of the best epoch: 5.
EfficientNetB0: frozen 0.8891 -> fine-tuned 0.9282 (+0.0391) in 11.0 min

# ---- Challenge 2: k-fold cross-validation of the Simple CNN (experiment subset, EXP_EPOCHS) ----
from sklearn.model_selection import StratifiedKFold
K = P["CV_FOLDS"]
X_cv = np.concatenate([Xe_tr, Xe_va]); y_cv = np.concatenate([ye_tr, ye_va])
fold_acc, fold_test = [], []
for k, (tr_i, va_i) in enumerate(StratifiedKFold(K, shuffle=True, random_state=SEED).split(X_cv, y_cv), 1):
    tr = make_dataset(X_cv[tr_i], y_cv[tr_i], training=True); va = make_dataset(X_cv[va_i], y_cv[va_i])
    m, r = train_model("SimpleCNN", tr, va, epochs=EXP_EPOCHS, verbose=0, tag=f"CV fold {k}/{K}")
    e, _ = evaluate_model(m, "SimpleCNN", test_ds, y_test, tag=f"CV fold {k}")
    fold_acc.append(r["best_val_accuracy"]); fold_test.append(e["test_accuracy"])
    del m; keras.backend.clear_session(); gc.collect()
ci = 1.96 * np.std(fold_test, ddof=1) / math.sqrt(K) if K > 1 else float("nan")
CH["cross_validation"] = dict(model="SimpleCNN", folds=K, n_samples=int(len(X_cv)), epochs=EXP_EPOCHS, fold_val_accuracy=[round(a, 4) for a in fold_acc],
                              fold_test_accuracy=[round(a, 4) for a in fold_test], mean_test_accuracy=round(float(np.mean(fold_test)), 4),
                              std_test_accuracy=round(float(np.std(fold_test, ddof=1)), 4) if K > 1 else 0.0, ci95_halfwidth=round(float(ci), 4),
                              single_split_reference=EVAL["SimpleCNN"]["test_accuracy"])
print(f"{K}-fold CV Simple CNN: val acc per fold {np.round(fold_acc, 4)} | test acc {np.round(fold_test, 4)} "
      f"| mean {np.mean(fold_test):.4f} +/- {ci:.4f} (95% CI half-width)")

Restoring model weights from the end of the best epoch: 5.

[CV fold 1/3] 5/5 epochs | best val acc 0.4552 @ epoch 5 | 0.8 min (10.1 s/epoch) | Reached the epoch ceiling (5); best epoch 5 weights restored
Restoring model weights from the end of the best epoch: 4.

[CV fold 2/3] 5/5 epochs | best val acc 0.4550 @ epoch 4 | 0.8 min (10.1 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
Restoring model weights from the end of the best epoch: 4.

[CV fold 3/3] 5/5 epochs | best val acc 0.4502 @ epoch 4 | 0.9 min (10.4 s/epoch) | Reached the epoch ceiling (5); best epoch 4 weights restored
3-fold CV Simple CNN: val acc per fold [0.4552 0.455  0.4502] | test acc [0.4427 0.4531 0.4594] | mean 0.4517 +/- 0.0095 (95% CI half-width)

# ---- Challenge 3: Ensemble of the top-2 models (softmax averaging vs majority vote) ----
ranked = sorted(EVAL, key=lambda n: EVAL[n]["test_accuracy"], reverse=True)
top2 = ranked[:2]
p_avg = np.mean([PROBS[n] for n in top2], axis=0); y_avg = p_avg.argmax(1)
votes = np.stack([PROBS[n].argmax(1) for n in ranked[:3]], axis=1)             # 3-model hard vote (odd count)
y_vote = np.apply_along_axis(lambda v: np.bincount(v, minlength=NUM_CLASSES).argmax(), 1, votes)
acc_avg, f1_avg = float(accuracy_score(y_test, y_avg)), float(f1_score(y_test, y_avg, average="weighted"))
acc_vote = float(accuracy_score(y_test, y_vote))
agree = float((PROBS[top2[0]].argmax(1) == PROBS[top2[1]].argmax(1)).mean())
CH["ensemble"] = dict(members_avg=top2, members_vote=ranked[:3], soft_avg_accuracy=round(acc_avg, 4), soft_avg_f1=round(f1_avg, 4),
                      hard_vote_accuracy=round(acc_vote, 4), best_single=top2[0], best_single_accuracy=EVAL[top2[0]]["test_accuracy"],
                      delta_vs_best_single=round(acc_avg - EVAL[top2[0]]["test_accuracy"], 4), top2_agreement=round(agree, 4),
                      inference_ms_per_image=round(sum(EVAL[n]["inference_ms_per_image"] for n in top2), 3))
print(f"Top-2 {top2}: soft-avg acc {acc_avg:.4f} (F1w {f1_avg:.4f}) vs best single {EVAL[top2[0]]['test_accuracy']:.4f} "
      f"({CH['ensemble']['delta_vs_best_single']:+.4f}) | 3-model hard vote {acc_vote:.4f} | top-2 agreement {agree:.3f}")

Top-2 ['EfficientNetB0', 'ResNet50']: soft-avg acc 0.9059 (F1w 0.9054) vs best single 0.8891 (+0.0168) | 3-model hard vote 0.8946 | top-2 agreement 0.881

# ---- Challenge 4: Model compression — TensorFlow Lite float16 and dynamic-range int8 quantisation ----
# The TFLite converter runs in a SUBPROCESS: with Keras 3 / TF 2.16 a conversion failure can abort the
# whole interpreter (LLVM error) instead of raising, which would kill the kernel and lose Parts 0-7.
import subprocess, textwrap
best_name = RESULTS["best_model"]["name"]; best_model = MODELS[best_name]
q_idx, _ = train_test_split(np.arange(len(X_test)), train_size=min(P["QUANT_EVAL_N"], len(X_test) - 10), stratify=y_test, random_state=SEED)
Xq = tf.image.resize(tf.cast(X_test[q_idx], tf.float32), [IMG_SIZE, IMG_SIZE]).numpy(); yq = y_test[q_idx]
np.savez(os.path.join(RESULTS_DIR, "quant_eval.npz"), X=Xq, y=yq)

def keras_size_mb(model):
    return round(sum(int(np.prod(w.shape)) for w in model.weights) * 4 / 2**20, 2)     # float32 weights

export_dir = os.path.join(RESULTS_DIR, f"{best_name}_savedmodel")
best_model.export(export_dir, verbose=0)                                              # SavedModel with serving signature
fp32_acc = float(accuracy_score(yq, best_model.predict(Xq, batch_size=32, verbose=0).argmax(1)))
CH["compression"] = dict(model=best_name, eval_n=int(len(yq)), fp32_size_mb=keras_size_mb(best_model), fp32_accuracy_subset=round(fp32_acc, 4), variants={})

CONVERT_SCRIPT = textwrap.dedent('''
    import sys, json, time, os, numpy as np
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
    import tensorflow as tf
    export_dir, mode, npz, out_tfl = sys.argv[1:5]
    conv = tf.lite.TFLiteConverter.from_saved_model(export_dir)
    conv.optimizations = [tf.lite.Optimize.DEFAULT]
    if mode == "float16": conv.target_spec.supported_types = [tf.float16]
    t0 = time.time(); tfl = conv.convert(); conv_s = time.time() - t0
    open(out_tfl, "wb").write(tfl)
    d = np.load(npz); X, y = d["X"], d["y"]
    it = tf.lite.Interpreter(model_path=out_tfl, num_threads=4); it.allocate_tensors()
    inp, out = it.get_input_details()[0], it.get_output_details()[0]
    preds, times = [], []
    for i in range(len(X)):
        it.set_tensor(inp["index"], X[i:i+1].astype(np.float32)); t = time.perf_counter(); it.invoke(); times.append(time.perf_counter() - t)
        preds.append(int(it.get_tensor(out["index"]).argmax()))
    print(json.dumps(dict(size_mb=round(len(tfl) / 2**20, 2), accuracy=float(np.mean(np.array(preds) == y)),
                          cpu_latency_ms_batch1=round(1000 * float(np.mean(times[5:] or times)), 2), convert_s=round(conv_s, 1))))
''')
script_path = os.path.join(RESULTS_DIR, "tflite_convert.py"); open(script_path, "w").write(CONVERT_SCRIPT)

for mode in ("float16", "dynamic_int8"):
    out_tfl = os.path.join(RESULTS_DIR, f"{best_name}_{mode}.tflite")
    proc = subprocess.run([sys.executable, script_path, export_dir, mode, os.path.join(RESULTS_DIR, "quant_eval.npz"), out_tfl],
                          capture_output=True, text=True, timeout=1800)
    try:
        r = json.loads(proc.stdout.strip().splitlines()[-1])
        r.update(compression_ratio=round(CH["compression"]["fp32_size_mb"] / max(r["size_mb"], 1e-6), 2),
                 accuracy_subset=round(r.pop("accuracy"), 4))
        r["accuracy_delta"] = round(r["accuracy_subset"] - fp32_acc, 4)
        CH["compression"]["variants"][mode] = r
        print(f"{best_name} {mode:13s}: {r['size_mb']:6.2f} MB ({r['compression_ratio']}x smaller) | acc {r['accuracy_subset']:.4f} "
              f"({r['accuracy_delta']:+.4f} vs fp32 {fp32_acc:.4f}) | TFLite CPU {r['cpu_latency_ms_batch1']} ms/img | convert {r['convert_s']}s")
    except Exception as e:
        err = (proc.stderr.strip().splitlines() or ["no stderr"])[-1][:200]
        CH["compression"]["variants"][mode] = dict(error=f"converter exit {proc.returncode}: {err}")
        print(f"{mode}: conversion failed in subprocess (exit {proc.returncode}) -> {err}")

INFO:tensorflow:Assets written to: results/EfficientNetB0_savedmodel/assets

INFO:tensorflow:Assets written to: results/EfficientNetB0_savedmodel/assets

EfficientNetB0 float16      :   8.02 MB (2.0x smaller) | acc 0.9230 (+0.0040 vs fp32 0.9190) | TFLite CPU 9.1 ms/img | convert 1.6s
EfficientNetB0 dynamic_int8 :   4.50 MB (3.57x smaller) | acc 0.9180 (-0.0010 vs fp32 0.9190) | TFLite CPU 7.83 ms/img | convert 1.6s

# ---- Challenge 5: Real-time inference profiling (batch 1 vs 32) for every model ----
def bench(model, img_size=IMG_SIZE, batch=1, reps=30):
    '''Latency of a compiled (graph-mode) forward pass; intermediates are freed per op, unlike eager calls.'''
    x = tf.random.uniform((batch, img_size, img_size, 3), 0, 255)
    fwd = tf.function(lambda t: model(t, training=False), reduce_retracing=True)
    for _ in range(3): fwd(x).numpy()                                   # warm-up incl. tracing
    t0 = time.perf_counter()
    for _ in range(reps): fwd(x).numpy()
    return 1000 * (time.perf_counter() - t0) / reps

rows = []
gc.collect()
for name, m in MODELS.items():
    b1, b32 = bench(m, batch=1, reps=30), bench(m, batch=32, reps=10)
    rows.append(dict(model=name, latency_ms_batch1=round(b1, 2), latency_ms_batch32=round(b32, 2), ms_per_image_batch32=round(b32 / 32, 3),
                     throughput_img_s_batch32=round(32000 / b32, 1), params_M=round(EVAL[name]["total_params"] / 1e6, 2), size_fp32_mb=keras_size_mb(m),
                     test_accuracy=EVAL[name]["test_accuracy"], realtime_30fps_batch1=bool(b1 < 33.3)))
lat_df = pd.DataFrame(rows); display(lat_df.set_index("model"))
CH["realtime_inference"] = dict(device=DEVICE, rows=rows,
                                recommendation=min(rows, key=lambda r: r["latency_ms_batch1"] / max(r["test_accuracy"], 1e-6))["model"])
fig, ax = plt.subplots(figsize=(7, 4.2))
ax.scatter(lat_df["latency_ms_batch1"], lat_df["test_accuracy"], s=lat_df["params_M"] * 6 + 20)
for _, r in lat_df.iterrows(): ax.annotate(f"{r['model']} ({r['params_M']} M)", (r["latency_ms_batch1"], r["test_accuracy"]), textcoords="offset points", xytext=(6, 3), fontsize=8)
ax.axvline(33.3, ls="--", color="gray"); ax.text(33.3, ax.get_ylim()[0], " 30 fps budget", color="gray", fontsize=8, va="bottom")
ax.set_xlabel(f"latency, batch 1 (ms) on {DEVICE}"); ax.set_ylabel("test accuracy"); ax.set_title("Real-time suitability (bubble = parameters)"); ax.grid(alpha=.3)
plt.tight_layout(); plt.savefig(os.path.join(FIG_DIR, "realtime_latency.png"), dpi=120); plt.show()

# ---- Part 9: Final export — cnn_lab_results.json and cnn_lab_comparison.csv (every section) ----
RESULTS["timestamp"] = time.strftime("%Y-%m-%d %H:%M:%S")
RESULTS["comparison_table"] = comp.to_dict(orient="records")
RESULTS["deliverables"] = ["Gonzalez_Jose_CNN_Capstone_Lab.ipynb", "results/cnn_lab_results.json", "results/cnn_lab_comparison.csv",
                           "results/parameter_experiments.csv", "results/figures/*.png", "Gonzalez_Jose_CNN_Capstone_Report.docx", "train.csv", "test.csv"]
json_path, csv_path = os.path.join(RESULTS_DIR, "cnn_lab_results.json"), os.path.join(RESULTS_DIR, "cnn_lab_comparison.csv")
with open(json_path, "w") as fh: json.dump(RESULTS, fh, indent=2, default=str)
comp.to_csv(csv_path, index=False)
print(f"wrote {json_path} ({os.path.getsize(json_path)/1024:.0f} KB) and {csv_path}")

# ---- Executive summary rendered from RESULTS ----
b = RESULTS["best_model"]["name"]; e = EVAL[b]
runner = sorted(EVAL, key=lambda n: EVAL[n]["test_accuracy"], reverse=True)[1]; s = EVAL[runner]   # second-best model
ft = CH.get("fine_tuning", {}); en = CH.get("ensemble", {}); cv = CH.get("cross_validation", {}); rt = CH.get("realtime_inference", {}); cp = CH.get("compression", {})
lines = [f"### Executive summary ({RUN_PROFILE} profile, {DEVICE}, seed {SEED})", "",
         f"| Model | Test acc | F1 (w) | Params | Train (s) | Inference (ms/img) |", "|---|---|---|---|---|---|"]
for n, r in EVAL.items():
    lines.append(f"| {n} | {r['test_accuracy']:.4f} | {r['f1_weighted']:.4f} | {r['total_params']/1e6:.2f} M | {HISTORIES[n]['train_time_s']:.0f} | {r['inference_ms_per_image']:.2f} |")
lines += ["",
  f"- **Best-performing model: {b}** — test accuracy {e['test_accuracy']:.4f}, weighted F1 {e['f1_weighted']:.4f}, "
  f"{e['total_params']/1e6:.1f} M parameters, {e['inference_ms_per_image']:.2f} ms/image. It beats the runner-up ({runner}) by "
  f"{e['test_accuracy']-s['test_accuracy']:+.4f} accuracy at {e['total_params']/max(s['total_params'],1):.1f}x the parameters and "
  f"{e['inference_ms_per_image']/max(s['inference_ms_per_image'],1e-9):.1f}x the inference latency.",
  f"- **Efficiency leader:** {comp.sort_values('acc_per_Mparam', ascending=False).iloc[0]['model']} on accuracy per parameter; "
  f"{RESULTS['best_model']['fastest_inference']} on inference latency.",
]
if ft: lines.append(f"- **Fine-tuning {ft['model']}** (last {ft['unfrozen_layers']} layers, lr {ft['lr']:g}): {ft['before_test_accuracy']:.4f} -> {ft['after_test_accuracy']:.4f} ({ft['delta_accuracy']:+.4f}).")
if en: lines.append(f"- **Ensemble** of {en['members_avg']}: {en['soft_avg_accuracy']:.4f} ({en['delta_vs_best_single']:+.4f} vs best single).")
if cv: lines.append(f"- **{cv['folds']}-fold CV (Simple CNN, subset):** {cv['mean_test_accuracy']:.4f} +/- {cv['ci95_halfwidth']:.4f} (95% CI) — the resolution floor for model comparisons.")
if cp and cp.get("variants"):
    for k, v in cp["variants"].items():
        if "size_mb" in v: lines.append(f"- **Compression ({k}) of {cp['model']}:** {cp['fp32_size_mb']} MB -> {v['size_mb']} MB ({v['compression_ratio']}x), accuracy {v['accuracy_delta']:+.4f}.")
if rt: lines.append(f"- **Real-time recommendation:** {rt['recommendation']} (best accuracy per ms at batch 1 on {DEVICE}).")
display(Markdown("\n".join(lines)))
with open(os.path.join(RESULTS_DIR, "executive_summary.md"), "w") as fh: fh.write("\n".join(lines))

wrote results/cnn_lab_results.json (54 KB) and results/cnn_lab_comparison.csv

Layer	Implementation
Data	CIFAR-10 python pickles → `train.csv` / `test.csv` (spec format: 1 label + 3072 pixels) → stratified 40k / 10k / 10k split
Input pipeline	`tf.data`: augmentation (flip, pad-and-crop, rotation, shift) on the training split only → resize to `IMG_SIZE` → per-model normalisation inside the graph
Model 1	Simple CNN — Conv(32)→Pool, Conv(64)→Pool, Conv(128, `last_conv`)→Pool, Flatten, Dense(128), Dropout(0.5), Dense(10, softmax)
Models 2–4	VGG16 / ResNet50 / EfficientNetB0 (`include_top=False`, ImageNet weights, frozen) + GAP → Dense(128, ReLU) → Dropout(0.5) → Dense(10, softmax)
Training	Adam (1e-3), categorical cross-entropy, batch 32, EarlyStopping(`val_accuracy`, patience 5), ReduceLROnPlateau(0.2, patience 3)
Evaluation	Test accuracy, weighted F1, confusion matrix, training time, inference time, total / trainable parameters
Features	Last convolutional layer of each model → dimensionality, sparsity, mean / std, t-SNE, silhouette + kNN separability

	layer	feature_dim	sparsity_pct	mean	std	dead_channels_pct	tsne_silhouette	tsne_knn5_accuracy	raw_silhouette
model
SimpleCNN	last_conv	25088	86.86	0.7565	2.8669	32.03	-0.1314	0.3405	-0.0455
VGG16	block5_pool	25088	87.61	1.2207	4.8658	0.00	-0.0238	0.6180	-0.0044
ResNet50	conv5_block3_out	100352	83.33	0.4716	1.7646	0.00	0.1059	0.7405	0.0239
EfficientNetB0	top_activation	62720	0.00	0.0899	0.8225	0.00	0.1012	0.7685	0.0298

#	Variable	Values	Model(s)
1	`BATCH_SIZE`	16 / 32 / 64	Simple CNN
2	Adam learning rate	1e-4 / 1e-3 / 1e-2	Simple CNN
3	Head dropout	0.3 / 0.5 / 0.8	VGG16, ResNet50, EfficientNetB0
4	`IMG_SIZE`	128 / 224 / 384	Simple CNN, VGG16, ResNet50, EfficientNetB0
5	Depth	3 blocks / + Conv2D(256)→MaxPool	Simple CNN (+ t-SNE comparison)

CNN Capstone Lab — CIFAR-10 Image Classification with Deep Learning¶

Part 1 — Data Loading, `train.csv` / `test.csv`, Split and Preprocessing¶

Observations & insights: Part 1¶

Part 2 — Model Architectures (Model 1 custom CNN; Models 2–4 transfer learning)¶

Justification of optimizer and loss (rubric §4)¶

Part 3 — Training with Callbacks¶

Observations & insights: Part 3¶

Part 4 — Evaluation & Results¶

Observations & insights: Part 4 — identifying and justifying the best model¶

Part 5 — Feature Analysis (last convolutional layer, t-SNE, activation statistics)¶

Discussion: feature activation patterns (rubric §7)¶

Part 6 — Parameter Modifications (Experiments 1–5)¶

Section B — Parameter Modification Questions (answers grounded in the tables above)¶

Part 7 — Challenges (advanced analysis and production considerations)¶

Discussion: challenges¶

Part 8 — Section A: Project Execution Questions (Q1–Q10)¶

Part 9 — Results Export and Executive Summary¶

Executive summary (FULL profile, Apple Metal GPU, seed 42)¶

Reproducibility Notes¶

Reflection¶

	airplane	automobile	bird	cat	deer	dog	frog	horse	ship	truck
train	4000	4000	4000	4000	4000	4000	4000	4000	4000	4000
val	1000	1000	1000	1000	1000	1000	1000	1000	1000	1000
test	1000	1000	1000	1000	1000	1000	1000	1000	1000	1000

	model	total_params	trainable_params	frozen_params	feature_layer	weights
0	SimpleCNN	897482	897482	0	last_conv	scratch
1	VGG16	14781642	66954	14714688	block5_pool	imagenet
2	ResNet50	23851274	263562	23587712	conv5_block3_out	imagenet
3	EfficientNetB0	4214829	165258	4049571	top_activation	imagenet

	model	epochs_run	epochs_max	best_epoch	stopped_early	best_val_accuracy	final_train_accuracy	train_time_s	mean_epoch_time_s	lr_final	lr_reductions
0	SimpleCNN	15	15	13	False	0.6122	0.524200	611.3	40.8	0.000040	2
1	VGG16	15	15	14	False	0.8241	0.737525	5035.5	335.7	0.000008	3
2	ResNet50	13	15	8	True	0.8908	0.798275	2630.4	202.3	0.000200	1
3	EfficientNetB0	15	15	12	False	0.8871	0.776600	1722.7	114.8	0.000200	1

	test_accuracy	f1_weighted	test_loss	best_val_accuracy	epochs_run	best_epoch	stopped_early	train_time_s	inference_time_s	inference_ms_per_image	total_params	trainable_params	feature_layer	acc_per_Mparam	acc_per_train_min
model
SimpleCNN	0.6158	0.5927	1.1152	0.6122	15	13	False	611.3	3.02	0.302	897482	897482	last_conv	0.6861	0.0604
VGG16	0.8165	0.8144	0.5643	0.8241	15	14	False	5035.5	51.11	5.111	14781642	66954	block5_pool	0.0552	0.0097
ResNet50	0.8856	0.8849	0.3546	0.8908	13	8	True	2630.4	35.49	3.549	23851274	263562	conv5_block3_out	0.0371	0.0202
EfficientNetB0	0.8891	0.8888	0.3400	0.8871	15	12	False	1722.7	22.70	2.270	4214829	165258	top_activation	0.2109	0.0310

	SimpleCNN	VGG16	ResNet50	EfficientNetB0
airplane	0.672	0.854	0.897	0.909
automobile	0.745	0.903	0.936	0.943
bird	0.283	0.755	0.865	0.862
cat	0.378	0.676	0.795	0.808
deer	0.508	0.757	0.850	0.854
dog	0.557	0.788	0.860	0.862
frog	0.682	0.782	0.914	0.881
horse	0.654	0.851	0.881	0.907
ship	0.766	0.898	0.933	0.934
truck	0.682	0.881	0.919	0.928

			variable	test_accuracy	d_test_acc	f1_weighted	best_val_accuracy	train_val_gap	train_time_s	d_train_time_s	mean_epoch_time_s	inference_ms_per_image	total_params	d_params	peak_gpu_mem_mb	rss_delta_mb	epochs_run
experiment	model	value
1	SimpleCNN	16	batch_size	0.4792	0.0041	0.4634	0.4915	-0.0475	85.6	32.1	17.1	0.331	897482	0.0	0.0	0.0	5
		32	batch_size	0.4751	0.0000	0.4501	0.4675	-0.0766	53.5	0.0	10.7	0.262	897482	0.0	0.0	0.0	5
		64	batch_size	0.4601	-0.0150	0.4526	0.4805	-0.1086	42.9	-10.6	8.6	0.195	897482	0.0	0.0	0.0	5
2	SimpleCNN	0.0001	learning_rate	0.4691	-0.0138	0.4561	0.4825	-0.1294	53.2	1.2	10.6	0.268	897482	0.0	0.0	0.0	5
		0.001	learning_rate	0.4829	0.0000	0.4633	0.4785	-0.1020	52.0	0.0	10.4	0.276	897482	0.0	0.0	0.0	5
		0.01	learning_rate	0.1000	-0.3829	0.0182	0.1000	0.0000	53.5	1.5	10.7	0.221	897482	0.0	0.0	0.0	5
3	VGG16	0.3	dropout	0.7710	-0.0044	0.7688	0.7830	-0.1078	326.9	-16.0	65.4	5.450	14781642	0.0	0.0	0.0	5
		0.5	dropout	0.7754	0.0000	0.7734	0.7850	-0.1261	342.9	0.0	68.6	5.435	14781642	0.0	0.0	0.0	5
		0.8	dropout	0.7196	-0.0558	0.7102	0.7290	-0.2198	341.1	-1.8	68.2	5.483	14781642	0.0	0.0	0.0	5
	ResNet50	0.3	dropout	0.8657	0.0053	0.8650	0.8675	-0.1118	220.3	1.2	44.0	3.621	23851274	0.0	0.0	0.0	5
		0.5	dropout	0.8604	0.0000	0.8594	0.8600	-0.1278	219.1	0.0	43.8	3.625	23851274	0.0	0.0	0.0	5
		0.8	dropout	0.8324	-0.0280	0.8310	0.8315	-0.2823	217.4	-1.7	43.4	3.652	23851274	0.0	0.0	0.0	5
	EfficientNetB0	0.3	dropout	0.8540	-0.0062	0.8530	0.8570	-0.1121	146.9	3.4	29.4	2.214	4214829	0.0	0.0	0.0	5
		0.5	dropout	0.8602	0.0000	0.8596	0.8560	-0.1236	143.5	0.0	28.7	2.286	4214829	0.0	0.0	0.0	5
		0.8	dropout	0.8505	-0.0097	0.8501	0.8510	-0.2025	145.7	2.2	29.1	2.342	4214829	0.0	0.0	30.9	5
4	SimpleCNN	128	img_size	0.4883	0.0126	0.4610	0.4820	-0.0704	45.6	-9.0	9.1	0.169	356810	-540672.0	0.0	165.5	5
		224	img_size	0.4757	0.0000	0.4637	0.4875	-0.1039	54.6	0.0	10.9	0.255	897482	0.0	0.0	0.0	5
		384	img_size	0.4450	-0.0307	0.4331	0.4405	-0.0835	116.4	61.8	23.3	0.510	2453962	1556480.0	0.0	0.0	5
	VGG16	128	img_size	0.7905	0.0065	0.7888	0.7790	-0.1200	110.3	-237.0	22.0	1.895	14781642	0.0	0.0	0.0	5
		224	img_size	0.7840	0.0000	0.7825	0.7870	-0.1278	347.3	0.0	69.4	5.586	14781642	0.0	0.0	0.0	5
		384	img_size	0.6660	-0.1180	0.6550	0.6880	-0.0600	992.4	645.1	198.5	15.991	14781642	0.0	0.0	0.0	5
	ResNet50	128	img_size	0.8424	-0.0062	0.8420	0.8350	-0.1340	91.4	-135.8	18.3	1.396	23851274	0.0	0.0	0.0	5
		224	img_size	0.8486	0.0000	0.8467	0.8585	-0.1485	227.2	0.0	45.4	3.635	23851274	0.0	0.0	0.0	5
		384	img_size	0.7805	-0.0681	0.7763	0.7805	-0.1290	620.2	393.0	124.0	10.366	23851274	0.0	0.0	0.0	5
	EfficientNetB0	128	img_size	0.8659	0.0120	0.8651	0.8665	-0.1373	77.3	-77.7	15.4	1.077	4214829	0.0	0.0	0.0	5
		224	img_size	0.8539	0.0000	0.8535	0.8580	-0.1295	155.0	0.0	31.0	2.381	4214829	0.0	0.0	0.0	5
		384	img_size	0.7278	-0.1261	0.7222	0.7280	-0.1553	395.0	240.0	79.0	6.352	4214829	0.0	0.0	0.0	5
5	SimpleCNN	3 blocks	depth	0.4717	0.0000	0.4558	0.4670	-0.0899	57.4	0.0	11.5	0.286	897482	0.0	0.0	0.0	5
5	SimpleCNN	4 blocks (+Conv256)	depth	0.4847	0.0130	0.4601	0.4840	-0.0692	58.9	1.5	11.8	0.258	684746	-212736.0	0.0	0.0	5

	latency_ms_batch1	latency_ms_batch32	ms_per_image_batch32	throughput_img_s_batch32	params_M	size_fp32_mb	test_accuracy	realtime_30fps_batch1
model
SimpleCNN	1.32	4.22	0.132	7574.3	0.90	3.42	0.6158	True
VGG16	7.59	145.79	4.556	219.5	14.78	56.39	0.8165	True
ResNet50	17.41	98.10	3.066	326.2	23.85	90.99	0.8856	True
EfficientNetB0	17.25	60.84	1.901	525.9	4.21	16.08	0.8891	True

CNN Capstone Lab — CIFAR-10 Image Classification with Deep Learning¶

Part 1 — Data Loading, train.csv / test.csv, Split and Preprocessing¶

Observations & insights: Part 1¶

Part 2 — Model Architectures (Model 1 custom CNN; Models 2–4 transfer learning)¶

Justification of optimizer and loss (rubric §4)¶

Part 3 — Training with Callbacks¶

Observations & insights: Part 3¶

Part 4 — Evaluation & Results¶

Observations & insights: Part 4 — identifying and justifying the best model¶

Part 5 — Feature Analysis (last convolutional layer, t-SNE, activation statistics)¶

Discussion: feature activation patterns (rubric §7)¶

Part 6 — Parameter Modifications (Experiments 1–5)¶

Section B — Parameter Modification Questions (answers grounded in the tables above)¶

Part 7 — Challenges (advanced analysis and production considerations)¶

Discussion: challenges¶

Part 8 — Section A: Project Execution Questions (Q1–Q10)¶

Part 9 — Results Export and Executive Summary¶

Executive summary (FULL profile, Apple Metal GPU, seed 42)¶

Reproducibility Notes¶

Reflection¶

Part 1 — Data Loading, `train.csv` / `test.csv`, Split and Preprocessing¶