{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "a9cea997",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.patches import Patch\n",
    "from matplotlib.lines import Line2D\n",
    "import torch\n",
    "import torchvision\n",
    "from skimage.feature import hog\n",
    "from skimage import exposure\n",
    "\n",
    "\n",
    "import os\n",
    "import random\n",
    "import math\n",
    "\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "plt.rcParams['font.family'] = 'Charter'\n",
    "plt.rcParams['text.latex.preamble'] = r'\\usepackage[charter]{mathdesign}'\n",
    "\n",
    "plt.rcParams['font.size'] = 12\n",
    "plt.rcParams['axes.labelsize'] = 12\n",
    "plt.rcParams['axes.titlesize'] = 12\n",
    "plt.rcParams['legend.fontsize'] = 12\n",
    "plt.rcParams['xtick.labelsize'] = 10\n",
    "plt.rcParams['ytick.labelsize'] = 10\n",
    "\n",
    "golden = (1 + 5 ** 0.5) / 2\n",
    "pt_to_inch = 1 / 72.27\n",
    "textwidth = 430 * pt_to_inch\n",
    "fig_width = textwidth\n",
    "fig_height = fig_width / golden\n",
    "\n",
    "from cycler import cycler  # Optional for custom cycling\n",
    "default_colors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n",
    "blue, orange, green, red, purple, brown, pink, gray, yellow, cyan = default_colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44e8e38d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 788\n",
      "Validation size: 100\n",
      "Test size: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['prohibitory-no-stopping', 'prohibitory-no-overtaking', 'mandatory-keep-left', 'warning-pedestrian-crossing', 'prohibitory-no-stopping', 'warning-series', 'mandatory-keep-left', 'misc-pedestrian-crossing', 'mandatory-cycle-path', 'misc-parking', 'warning-curve', 'warning-slippery-road', 'prohibitory-no-turn', 'misc-dead-end', 'warning-pedestrian-crossing', 'misc-bus-stop', 'mandatory-cycle-path', 'warning-slippery-road', 'mandatory-cycle-path', 'mandatory-chains', 'misc-pedestrian-crossing', 'prohibitory-no-turn', 'warning-slippery-road', 'misc-gas-station', 'prohibitory-no-turn', 'misc-bus-stop', 'prohibitory-no-trucks', 'misc-pedestrian-crossing', 'warning-series', 'warning-slippery-road', 'misc-parking', 'misc-gas-station']\n",
      "torch.Size([32, 3, 200, 200])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 935908 characters]",
      "text/plain": [
       "<Figure size 1280x960 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def create_loaders(train_transform, test_transform, folder=\"./road-signs\"):\n",
    "    \"\"\"\n",
    "    Utility function to create loaders from transforms.\n",
    "    Used mainly to speed up creation of custom loaders (in this project grayscale).\n",
    "    The same transform is applied to validation and test set.\n",
    "    \"\"\"\n",
    "\n",
    "    # From the docs is_valid_file is normally used to check for corrupt files, here a cutom version is used to quickly split train and test\n",
    "    # this is done because of the nested folder structure:\n",
    "    # Graphical representation of the folder structure:\n",
    "    # road-signs/\n",
    "    # ├── <sign name>/\n",
    "    # │   ├── train/\n",
    "    # │   └── test/\n",
    "\n",
    "    # on top of that, the same is_valid_file mechanics is leveraged to select the validation set (5 samples per class from train folder)\n",
    "    val_paths = set()\n",
    "\n",
    "    for class_folder in os.listdir(folder):\n",
    "        train_folder = os.path.join(folder, class_folder, \"train\")\n",
    "        if os.path.isdir(train_folder):\n",
    "            train_files = os.listdir(train_folder)\n",
    "            val_samples = random.sample(train_files, 5)\n",
    "            for sample in val_samples:\n",
    "                val_paths.add(os.path.normpath(os.path.join(train_folder, sample)))\n",
    "\n",
    "\n",
    "    is_train = lambda p: os.path.normpath(p).split(os.sep)[-2] == 'train' and os.path.normpath(p) not in val_paths\n",
    "    is_validation = lambda p: os.path.normpath(p) in val_paths\n",
    "    is_test  = lambda p: os.path.normpath(p).split(os.sep)[-2] == 'test'\n",
    "\n",
    "    train_set = torchvision.datasets.ImageFolder(folder, transform=train_transform, is_valid_file=is_train)\n",
    "    train_loader = torch.utils.data.DataLoader(train_set, batch_size=32, shuffle=True, drop_last=False, num_workers=2)\n",
    "\n",
    "    # no augmentation for validation and test set\n",
    "    val_set = torchvision.datasets.ImageFolder(folder, transform=test_transform, is_valid_file=is_validation)\n",
    "    val_loader = torch.utils.data.DataLoader(val_set, batch_size=32, shuffle=False, drop_last=False, num_workers=2)\n",
    "    test_set = torchvision.datasets.ImageFolder(folder, transform=test_transform, is_valid_file=is_test)\n",
    "    test_loader = torch.utils.data.DataLoader(test_set, batch_size=32, shuffle=False, drop_last=False, num_workers=2)\n",
    "\n",
    "    print(f\"Train size: {len(train_set)}\")\n",
    "    print(f\"Validation size: {len(val_set)}\")\n",
    "    print(f\"Test size: {len(test_set)}\")\n",
    "\n",
    "    return train_loader, val_loader, test_loader\n",
    "\n",
    "\n",
    "transform = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.RandomRotation(10),\n",
    "    torchvision.transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.05),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "train_loader, val_loader, test_loader = create_loaders(transform, torchvision.transforms.ToTensor())\n",
    "num_classes = len(train_loader.dataset.classes)\n",
    "\n",
    "images, labels = next(iter(train_loader))\n",
    "plt.figure(dpi=200)\n",
    "grid = torchvision.utils.make_grid(images, nrow=8, normalize=True).permute(1, 2, 0)\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(grid)\n",
    "print([train_loader.dataset.classes[i] for i in labels])\n",
    "print(images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "592f05e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 788\n",
      "Validation size: 100\n",
      "Test size: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['warning-slippery-road', 'misc-dead-end', 'warning-series', 'prohibitory-no-transit', 'misc-parking', 'warning-steep-ascent', 'misc-parking', 'misc-parking', 'misc-parking', 'warning-slippery-road', 'mandatory-go-straight', 'prohibitory-no-turn', 'warning-series', 'misc-bus-stop', 'misc-bus-stop', 'misc-parking', 'warning-steep-ascent', 'mandatory-keep-left', 'prohibitory-no-trucks', 'warning-slippery-road', 'prohibitory-no-transit', 'misc-gas-station', 'mandatory-keep-left', 'mandatory-roundabout', 'prohibitory-no-overtaking', 'prohibitory-no-stopping', 'misc-bus-stop', 'misc-parking', 'warning-slippery-road', 'mandatory-go-straight', 'mandatory-keep-left', 'mandatory-chains']\n",
      "torch.Size([32, 1, 200, 200])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 469376 characters]",
      "text/plain": [
       "<Figure size 1280x960 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# here let's create the grayscale dataset, same as before\n",
    "transform_gray = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.RandomRotation(10),\n",
    "    torchvision.transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.05),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    torchvision.transforms.Grayscale(num_output_channels=1)\n",
    "])\n",
    "\n",
    "# no augmentation for test set\n",
    "transform_gray_test = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    torchvision.transforms.Grayscale(num_output_channels=1)\n",
    "])\n",
    "\n",
    "train_loader_gray, val_loader_gray, test_loader_gray = create_loaders(transform_gray, transform_gray_test)\n",
    "\n",
    "images, labels = next(iter(train_loader_gray))\n",
    "plt.figure(dpi=200)\n",
    "grid = torchvision.utils.make_grid(images, nrow=8, normalize=True).permute(1, 2, 0)\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(grid)\n",
    "print([train_loader_gray.dataset.classes[i] for i in labels])\n",
    "print(images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "53deabdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Color histogram shape: torch.Size([7, 192])\n",
      "Edge direction histogram shape: torch.Size([7, 64])\n",
      "RGB Co-occurrence shape: torch.Size([7, 729])\n",
      "HOG shape: torch.Size([7, 900])\n"
     ]
    }
   ],
   "source": [
    "def color_histogram(img, bins=64):\n",
    "    n = img.shape[0]\n",
    "    c = img.shape[1]  # num channels, useful to compute both grayscales c=1 and rgb c=3\n",
    "    img = (img * bins).long().clamp(0, bins - 1)\n",
    "    img = img + torch.arange(0, c * n * bins, bins, device=img.device).view(n, c, 1, 1)\n",
    "    hist = torch.bincount(img.view(-1), minlength=(c * n * bins)).view(n, c * bins)\n",
    "    return hist / img[0, 0].numel()\n",
    "\n",
    "\n",
    "def edge_direction_histogram(img, bins=64):\n",
    "    n = img.shape[0]\n",
    "    SOBEL = torch.tensor([\n",
    "        [[[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]],\n",
    "        [[[-1, -2, -1], [0, 0, 0], [1, 2, 1]]]\n",
    "    ], dtype=img.dtype, device=img.device)\n",
    "    gray = img.mean(1, True)\n",
    "    grad = torch.nn.functional.conv2d(gray, SOBEL)\n",
    "    magnitude = grad.norm(dim=1)\n",
    "    angle = torch.atan2(grad[:, 0], grad[:, 1])\n",
    "    directions = (bins * angle / (2 * torch.pi)).long() % bins\n",
    "    directions = directions + torch.arange(0, n * bins, bins, device=img.device).view(n, 1, 1)\n",
    "    hist = torch.bincount(directions.view(-1), weights=magnitude.view(-1), minlength=n * bins).view(n, bins)\n",
    "    hist = hist / hist.sum(1, True).clamp(1e-6)\n",
    "    return hist\n",
    "\n",
    "\n",
    "def _cooccurrence_matrix_dir(values, bins, di, dj):\n",
    "    b, m, n = values.shape\n",
    "    codes = values[:, :m - di, :n - dj] + bins * values[:, di:, dj:]\n",
    "    codes += torch.arange(0, b * (bins ** 2), bins ** 2, device=values.device).view(b, 1, 1)\n",
    "    entries = torch.bincount(codes.view(-1), minlength=b * (bins ** 2)).view(b, bins, bins)\n",
    "    return entries\n",
    "\n",
    "\n",
    "def rgb_cooccurrence_matrix(img, quantization=3, distance=10):\n",
    "    n = img.shape[0]\n",
    "    c = img.shape[1]  # num channels, useful to compute both grayscales c=1 and rgb c=3, same as color histogram\n",
    "    img = (img * quantization).long().clamp(0, quantization - 1)\n",
    "    img = (img * (quantization ** torch.arange(c, device=img.device)).view(1, c, 1, 1)).sum(1)\n",
    "    bins = quantization ** c           # the num of features is actually the square of this cause i do 2 mats\n",
    "    mat = _cooccurrence_matrix_dir(img, bins, distance, 0)\n",
    "    mat += _cooccurrence_matrix_dir(img, bins, 0, distance)\n",
    "    mat = (mat + mat.permute(0, 2, 1)).view(n, -1)\n",
    "    mat = mat / mat.sum(1, True)\n",
    "    return mat\n",
    "\n",
    "\n",
    "def histogram_oriented_gradients(img, orientations=9, pixels_per_cell=(32, 32)):\n",
    "    # orientations: Number of orientation bins\n",
    "    # pixels_per_cell: Tuple of (cell_height, cell_width)\n",
    "    \n",
    "    n = img.shape[0]\n",
    "    features = []\n",
    "    \n",
    "    for i in range(n):\n",
    "        # Convert to grayscale numpy array [H, W] and move to CPU (scikin on cpu only)\n",
    "        gray = img[i].mean(0).cpu().numpy()\n",
    "        \n",
    "        hog_feat = hog(\n",
    "            gray,\n",
    "            orientations=orientations,\n",
    "            pixels_per_cell=pixels_per_cell,\n",
    "            cells_per_block=(2, 2),\n",
    "            feature_vector=True,\n",
    "            transform_sqrt=True       # power law compression\n",
    "        )\n",
    "        \n",
    "        features.append(torch.tensor(hog_feat, dtype=img.dtype, device=img.device))\n",
    "    \n",
    "    return torch.stack(features)\n",
    "\n",
    "\n",
    "# test on random tensors to se shapes\n",
    "x = torch.rand(7, 3, 200, 200)\n",
    "ch = color_histogram(x.clone())\n",
    "print(f\"Color histogram shape: {ch.shape}\")\n",
    "edh = edge_direction_histogram(x.clone())\n",
    "print(f\"Edge direction histogram shape: {edh.shape}\")\n",
    "h = rgb_cooccurrence_matrix(x.clone())\n",
    "print(f\"RGB Co-occurrence shape: {h.shape}\")\n",
    "hog_tensor = histogram_oriented_gradients(x.clone())\n",
    "print(f\"HOG shape: {hog_tensor.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b23ffdf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "def standardize(X):\n",
    "    \"\"\"\n",
    "    Standardize the input tensor X (zero mean, unit variance).\n",
    "    \"\"\"\n",
    "    mean = X.mean(dim=0, keepdim=True)\n",
    "    std = X.std(dim=0, keepdim=True).clamp(min=1e-6)\n",
    "    X_standardized = (X - mean) / std\n",
    "    return X_standardized, mean, std\n",
    "\n",
    "def extract_features(loader, features_to_use=[\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]):\n",
    "    \"\"\"\n",
    "    Utility function to extract selected features from the given image loader.\n",
    "    Returns a tensor of features and corresponding labels.\n",
    "    \"\"\"\n",
    "    all_features = []\n",
    "    all_labels = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for images, labels in loader:\n",
    "            images = images.to(DEVICE)\n",
    "            \n",
    "            color = color_histogram(images) if \"color_histo\" in features_to_use else None\n",
    "            edge = edge_direction_histogram(images) if \"edge_dir_histo\" in features_to_use else None\n",
    "            cooccur = rgb_cooccurrence_matrix(images) if \"rgb_cooc_mat\" in features_to_use else None\n",
    "            hog_feat = histogram_oriented_gradients(images) if \"hog\" in features_to_use else None\n",
    "            \n",
    "            features = torch.cat([f for f in [color, edge, cooccur, hog_feat] if f is not None], dim=1)\n",
    "            all_features.append(features.cpu())\n",
    "            all_labels.append(labels)\n",
    "    \n",
    "    return torch.cat(all_features), torch.cat(all_labels)\n",
    "\n",
    "\n",
    "def extract_and_standardize(train_img_loader, val_img_loader, test_img_loader, features_to_use=[\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]):\n",
    "    \"\"\"\n",
    "    From the given image loader, extract features and standardize them.\n",
    "    Return a new loader with standardized selected features.\n",
    "    \"\"\"\n",
    "    \n",
    "    train_features, train_labels = extract_features(train_img_loader, features_to_use)\n",
    "    val_features, val_labels = extract_features(val_img_loader, features_to_use)\n",
    "    test_features, test_labels = extract_features(test_img_loader, features_to_use)\n",
    "\n",
    "    # Standard scaling, good for SVM/Logistic Regression/MLP\n",
    "    train_features_norm, mean, std = standardize(train_features)\n",
    "    val_features_norm = (val_features - mean) / std\n",
    "    test_features_norm = (test_features - mean) / std\n",
    "\n",
    "    train_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(train_features_norm, train_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(val_features_norm, val_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(test_features_norm, test_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_features, val_loader_features, test_loader_features, train_features_norm.shape[1]  # also return num features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "9c4107b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, loader, loss_fun):\n",
    "    model.eval()\n",
    "    total_loss = 0.0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for imgs, labels in loader:\n",
    "            imgs = imgs.to(DEVICE)\n",
    "            labels = labels.to(DEVICE)\n",
    "            logits = model(imgs)\n",
    "            loss = loss_fun(logits, labels)\n",
    "            total_loss += loss.item() * imgs.size(0)\n",
    "            preds = logits.argmax(dim=1)\n",
    "            correct += (preds == labels).sum().item()\n",
    "            total += labels.size(0)\n",
    "    avg_loss = total_loss / total\n",
    "    avg_acc = correct / total\n",
    "    model.train()\n",
    "    return avg_acc, avg_loss\n",
    "\n",
    "def train(model, optimizer, loss_fun, train_loader, val_loader, epochs=20, verbose=0, plotting=False, patience=10, scheduler=None):\n",
    "    if verbose > 2:\n",
    "        verbose = 2  # cap verbosity to 2\n",
    "    \n",
    "    steps = []\n",
    "    accuracies = []\n",
    "    losses = []\n",
    "    val_accs = []\n",
    "    val_losses = []\n",
    "\n",
    "    step = 0\n",
    "    model.train()\n",
    "    best_val_loss = float('inf')\n",
    "    patience_counter = 0\n",
    "    early_stop = False\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        # early stopping check\n",
    "        if early_stop:\n",
    "            break\n",
    "        # note that here, if the loader passes low-level features, imgs variable contains images expressed as low-level features!!!\n",
    "        for imgs, labels in train_loader:\n",
    "            imgs = imgs.to(DEVICE)\n",
    "            labels = labels.to(DEVICE)\n",
    "            optimizer.zero_grad()\n",
    "            logits = model(imgs)\n",
    "            loss = loss_fun(logits, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            step += 1\n",
    "            predictions = logits.argmax(dim=1)\n",
    "            accuracy = (predictions == labels).float().mean()\n",
    "            steps.append(step)\n",
    "            accuracies.append(accuracy.item())\n",
    "            losses.append(loss.item())\n",
    "            val_acc, val_loss = evaluate(model, val_loader, loss_fun)\n",
    "            val_accs.append(val_acc)\n",
    "            val_losses.append(val_loss)\n",
    "\n",
    "            # Added the scheduler for the cnn training. Ignored for precedent cases\n",
    "            if scheduler is not None:\n",
    "                prev_lr = optimizer.param_groups[0]['lr']\n",
    "                scheduler.step(val_loss)\n",
    "                curr_lr = optimizer.param_groups[0]['lr']\n",
    "                if curr_lr != prev_lr:\n",
    "                    print(f\"Learning rate reduced: {prev_lr} -> {curr_lr}\")\n",
    "\n",
    "            if verbose > 0 and step % (10 ** (3 - verbose)) == 0:\n",
    "                print(f\"{step} [{epoch}] Loss: {loss.item():.4f} Acc: {accuracy.item() * 100:.1f}\")\n",
    "                print(f\"Epoch {epoch}: Val Loss={val_loss:.4f}, Val Acc={val_acc * 100:.1f}\")\n",
    "            \n",
    "            if val_loss < best_val_loss:\n",
    "                best_val_loss = val_loss\n",
    "                patience_counter = 0\n",
    "            else:\n",
    "                patience_counter += 1\n",
    "                if patience_counter >= patience:\n",
    "                    print(f\"Early stopping at epoch {epoch}, step {step}\")\n",
    "                    early_stop = True\n",
    "                    break\n",
    "                \n",
    "    if plotting:\n",
    "        fig, ax1 = plt.subplots(figsize=(fig_width, fig_height))\n",
    "        ax1.set_xlabel(\"Iterations\")\n",
    "        ax1.set_ylabel(\"Accuracy\", color=blue)\n",
    "        ax1.plot(steps, accuracies, label=\"Accuracy\", color=blue, linewidth=2, alpha=0.7, linestyle='dashed')\n",
    "        ax1.plot(steps, val_accs, label=\"Val Accuracy\", color=blue, linewidth=2)\n",
    "        ax1.tick_params(axis='y', labelcolor=blue)\n",
    "        ax1.set_xlim(0, steps[-1])\n",
    "        ax1.set_ylim(0, 1.05)  # accuracy between 0 and 1\n",
    "        ax1.grid(linestyle='dotted')\n",
    "\n",
    "        ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\n",
    "\n",
    "        ax2.set_ylabel(\"Loss\", color=red)\n",
    "        ax2.plot(steps, losses, label=\"Loss\", color=red, linewidth=2, alpha=0.7, linestyle='dashed')\n",
    "        ax2.plot(steps, val_losses, label=\"Val Loss\", color=red, linewidth=2)\n",
    "        ax2.tick_params(axis='y', labelcolor=red)\n",
    "\n",
    "        legend_elements = [\n",
    "            Line2D([0], [0], color='gray', linestyle='dashed', linewidth=2, alpha=0.7),\n",
    "            Line2D([0], [0], color='gray', linestyle='solid', linewidth=2)\n",
    "        ]\n",
    "        labels = ['Training', 'Validation']\n",
    "\n",
    "        ax1.legend(legend_elements, labels, loc='upper right', fontsize=9, handlelength=2.5)\n",
    "\n",
    "        fig.tight_layout()\n",
    "        plt.title(\"Accuracy and Loss over Iterations\")\n",
    "        plt.show()\n",
    "\n",
    "    return steps, accuracies, losses, val_accs, val_losses\n",
    "\n",
    "def test_conf_mat(model, test_loader, nclasses=20, verbose=True):\n",
    "    model.eval()\n",
    "    conf_mat = torch.zeros(nclasses, nclasses, dtype=torch.long)\n",
    "\n",
    "    for imgs, labels in test_loader:\n",
    "        imgs = imgs.to(DEVICE)\n",
    "        labels = labels.to(DEVICE)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            logits = model(imgs)\n",
    "        predictions = torch.argmax(logits, dim=1)\n",
    "\n",
    "        codes = labels * nclasses + predictions\n",
    "        hist = torch.bincount(codes.cpu(), minlength=nclasses ** 2)\n",
    "        conf_mat += hist.view(nclasses, nclasses)\n",
    "\n",
    "    accuracy = conf_mat.diag().sum() / conf_mat.sum()\n",
    "    if verbose:\n",
    "        print(f\"Accuracy: {100 * accuracy:.1f}%\")\n",
    "\n",
    "    # since the confusion matrix is evaluated for the accuracy, it can be safely returned\n",
    "    return conf_mat\n",
    "\n",
    "def plot_conf_mat(conf_mat, classes):\n",
    "    conf_mat_norm = conf_mat / conf_mat.sum(1)\n",
    "    plt.figure(dpi=100)\n",
    "    plt.imshow(conf_mat_norm, cmap=\"Blues\")\n",
    "    plt.xticks(range(len(classes)), classes, rotation=90)\n",
    "    plt.yticks(range(len(classes)), classes)\n",
    "    plt.colorbar()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b8c0b8cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = 0.001\n",
    "\n",
    "class ModelWrapper():\n",
    "    \"\"\" \n",
    "    Wrapper class for logistic regression model using low-level features. \n",
    "    Useful to define directy a new instance when training with different features while retaining taining results.\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "            self,\n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR, \n",
    "            classes=None\n",
    "        ):\n",
    "        \"\"\"\n",
    "        In this constructor, if features_used is None, the given loaders are assumed to already contain low-level features.\n",
    "        Otherwise, features are extracted from the image loaders.\n",
    "        If the loaders are provided directly as low-level features, remember to provide the 'classes' argument explicitly as it cannot be inferred.\n",
    "        \"\"\"\n",
    "        if train_loader is not None and val_loader is not None and test_loader is not None:\n",
    "            if features_used is not None:\n",
    "                self.features_used = features_used\n",
    "                self.train_loader_features, self.val_loader_features, self.test_loader_features, self.num_features = extract_and_standardize(\n",
    "                    train_loader,\n",
    "                    val_loader,\n",
    "                    test_loader,\n",
    "                    features_to_use=features_used\n",
    "                )\n",
    "            else:\n",
    "                self.train_loader_features = train_loader\n",
    "                self.val_loader_features = val_loader\n",
    "                self.test_loader_features = test_loader\n",
    "                self.num_features = train_loader.dataset.tensors[0].shape[1]\n",
    "\n",
    "        self.lr = lr\n",
    "        if classes is not None:\n",
    "            self.classes = classes\n",
    "        elif features_used is not None:\n",
    "            self.classes = train_loader.dataset.classes\n",
    "        else:\n",
    "            raise ValueError(\"Cannot determine classes: please provide the 'classes' argument explicitly when features_used is None.\")\n",
    "        \n",
    "        self.model = None\n",
    "        self.optimizer = None\n",
    "        self.loss_fun = None\n",
    "        self.train_results = {\n",
    "            \"steps\": None,\n",
    "            \"accuracies\": None,\n",
    "            \"losses\": None,\n",
    "            \"val_accs\": None,\n",
    "            \"val_losses\": None\n",
    "        }\n",
    "\n",
    "    def build_model(self, model, optimizer, loss_fun):\n",
    "        if self.model is not None:\n",
    "            print(\"Warning: overwriting existing model, optimizer, and loss function.\")\n",
    "        self.model = model\n",
    "        self.model.to(DEVICE)\n",
    "        self.optimizer = optimizer\n",
    "        self.loss_fun = loss_fun\n",
    "    \n",
    "    def train(self, epochs=20, verbose=0, plotting=True, patience=10):\n",
    "        steps, accuracies, losses, val_accs, val_losses = train(\n",
    "            self.model,\n",
    "            self.optimizer,\n",
    "            self.loss_fun,\n",
    "            self.train_loader_features,\n",
    "            self.val_loader_features,\n",
    "            epochs=epochs,\n",
    "            verbose=verbose,\n",
    "            plotting=plotting,\n",
    "            patience=patience\n",
    "        )\n",
    "\n",
    "        self.train_results[\"steps\"] = steps\n",
    "        self.train_results[\"accuracies\"] = accuracies\n",
    "        self.train_results[\"losses\"] = losses\n",
    "        self.train_results[\"val_accs\"] = val_accs\n",
    "        self.train_results[\"val_losses\"] = val_losses\n",
    "    \n",
    "    def test(self):\n",
    "        conf_mat = test_conf_mat(self.model, self.test_loader_features, nclasses=len(self.classes), verbose=True)\n",
    "        plot_conf_mat(conf_mat, self.classes)\n",
    "\n",
    "\n",
    "class LogisticRegressionModelWrapper(ModelWrapper):\n",
    "    def __init__(\n",
    "            self, \n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR, \n",
    "            classes=None\n",
    "        ):\n",
    "\n",
    "        super().__init__(\n",
    "            features_used=features_used, \n",
    "            train_loader=train_loader, \n",
    "            val_loader=val_loader, \n",
    "            test_loader=test_loader, \n",
    "            lr=lr, \n",
    "            classes=classes\n",
    "        )\n",
    "\n",
    "        log_reg_model = torch.nn.Linear(self.num_features, len(self.classes))\n",
    "        log_reg_model.to(DEVICE)\n",
    "        log_reg_optimizer = torch.optim.Adam(log_reg_model.parameters(), lr=lr)\n",
    "        log_reg_loss_fun = torch.nn.CrossEntropyLoss()\n",
    "        self.build_model(log_reg_model, log_reg_optimizer, log_reg_loss_fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9ca4040b",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_list = [\n",
    "    [\"color_histo\"],\n",
    "    [\"edge_dir_histo\"],\n",
    "    [\"rgb_cooc_mat\"],\n",
    "    [\"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\"],\n",
    "    [\"color_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"color_histo\", \"hog\"],\n",
    "    [\"edge_dir_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"edge_dir_histo\", \"hog\"],\n",
    "    [\"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"hog\"],\n",
    "    [\"color_histo\", \"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "213266fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo']\n",
      "100 [3] Loss: 2.0440 Acc: 45.0\n",
      "Epoch 3: Val Loss=2.4044, Val Acc=28.0\n",
      "200 [7] Loss: 1.7069 Acc: 60.0\n",
      "Epoch 7: Val Loss=2.2144, Val Acc=31.0\n",
      "300 [11] Loss: 1.7047 Acc: 60.0\n",
      "Epoch 11: Val Loss=2.1464, Val Acc=32.0\n",
      "Early stopping at epoch 14, step 364\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 134448 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo']\n",
      "100 [3] Loss: 1.8347 Acc: 50.0\n",
      "Epoch 3: Val Loss=2.0686, Val Acc=30.0\n",
      "200 [7] Loss: 1.6515 Acc: 50.0\n",
      "Epoch 7: Val Loss=1.7729, Val Acc=42.0\n",
      "300 [11] Loss: 1.3914 Acc: 60.0\n",
      "Epoch 11: Val Loss=1.6072, Val Acc=47.0\n",
      "400 [15] Loss: 1.1181 Acc: 80.0\n",
      "Epoch 15: Val Loss=1.5010, Val Acc=49.0\n",
      "500 [19] Loss: 1.5508 Acc: 55.0\n",
      "Epoch 19: Val Loss=1.4234, Val Acc=54.0\n",
      "600 [23] Loss: 1.0186 Acc: 70.0\n",
      "Epoch 23: Val Loss=1.3713, Val Acc=54.0\n",
      "700 [27] Loss: 1.1832 Acc: 75.0\n",
      "Epoch 27: Val Loss=1.3291, Val Acc=54.0\n",
      "800 [31] Loss: 1.3885 Acc: 50.0\n",
      "Epoch 31: Val Loss=1.2973, Val Acc=59.0\n",
      "900 [35] Loss: 1.0259 Acc: 70.0\n",
      "Epoch 35: Val Loss=1.2701, Val Acc=59.0\n",
      "Early stopping at epoch 38, step 951\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 119108 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['rgb_cooc_mat']\n",
      "100 [3] Loss: 1.8094 Acc: 45.0\n",
      "Epoch 3: Val Loss=2.2804, Val Acc=27.0\n",
      "200 [7] Loss: 1.5886 Acc: 45.0\n",
      "Epoch 7: Val Loss=2.1157, Val Acc=32.0\n",
      "Early stopping at epoch 10, step 273\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 126416 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['hog']\n",
      "100 [3] Loss: 0.0546 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0497, Val Acc=100.0\n",
      "200 [7] Loss: 0.0341 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0228, Val Acc=100.0\n",
      "300 [11] Loss: 0.0084 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0151, Val Acc=100.0\n",
      "400 [15] Loss: 0.0058 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0112, Val Acc=100.0\n",
      "500 [19] Loss: 0.0057 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0087, Val Acc=100.0\n",
      "600 [23] Loss: 0.0065 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0073, Val Acc=100.0\n",
      "700 [27] Loss: 0.0044 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0063, Val Acc=100.0\n",
      "800 [31] Loss: 0.0025 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0055, Val Acc=100.0\n",
      "900 [35] Loss: 0.0026 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0049, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0015 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0044, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0026 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0040, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0018 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0037, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41560 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo']\n",
      "100 [3] Loss: 1.3890 Acc: 45.0\n",
      "Epoch 3: Val Loss=1.6440, Val Acc=49.0\n",
      "200 [7] Loss: 0.8723 Acc: 85.0\n",
      "Epoch 7: Val Loss=1.3311, Val Acc=62.0\n",
      "300 [11] Loss: 0.5523 Acc: 100.0\n",
      "Epoch 11: Val Loss=1.2047, Val Acc=59.0\n",
      "400 [15] Loss: 0.6030 Acc: 90.0\n",
      "Epoch 15: Val Loss=1.1274, Val Acc=63.0\n",
      "500 [19] Loss: 0.5407 Acc: 90.0\n",
      "Epoch 19: Val Loss=1.0829, Val Acc=66.0\n",
      "Early stopping at epoch 21, step 542\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 113200 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 1.2342 Acc: 65.0\n",
      "Epoch 3: Val Loss=1.9134, Val Acc=39.0\n",
      "Early stopping at epoch 7, step 194\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 119408 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'hog']\n",
      "100 [3] Loss: 0.0460 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0645, Val Acc=100.0\n",
      "200 [7] Loss: 0.0198 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0328, Val Acc=100.0\n",
      "300 [11] Loss: 0.0046 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0220, Val Acc=100.0\n",
      "400 [15] Loss: 0.0037 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0165, Val Acc=100.0\n",
      "500 [19] Loss: 0.0080 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0133, Val Acc=100.0\n",
      "600 [23] Loss: 0.0034 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0115, Val Acc=100.0\n",
      "Early stopping at epoch 27, step 686\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 45460 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 1.1318 Acc: 70.0\n",
      "Epoch 3: Val Loss=1.3876, Val Acc=58.0\n",
      "200 [7] Loss: 0.7646 Acc: 80.0\n",
      "Epoch 7: Val Loss=1.1715, Val Acc=67.0\n",
      "300 [11] Loss: 0.7272 Acc: 80.0\n",
      "Epoch 11: Val Loss=1.1038, Val Acc=69.0\n",
      "400 [15] Loss: 0.4787 Acc: 95.0\n",
      "Epoch 15: Val Loss=1.0747, Val Acc=70.0\n",
      "Early stopping at epoch 16, step 419\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 118964 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'hog']\n",
      "100 [3] Loss: 0.0270 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0434, Val Acc=100.0\n",
      "200 [7] Loss: 0.0209 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0210, Val Acc=100.0\n",
      "300 [11] Loss: 0.0077 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0128, Val Acc=100.0\n",
      "400 [15] Loss: 0.0091 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0092, Val Acc=100.0\n",
      "500 [19] Loss: 0.0037 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0068, Val Acc=100.0\n",
      "600 [23] Loss: 0.0073 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0054, Val Acc=100.0\n",
      "700 [27] Loss: 0.0037 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0046, Val Acc=100.0\n",
      "800 [31] Loss: 0.0032 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0039, Val Acc=100.0\n",
      "900 [35] Loss: 0.0014 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0034, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0017 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0030, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0017 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0026, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0023 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0024, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41700 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0416 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.1084, Val Acc=98.0\n",
      "200 [7] Loss: 0.0146 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0733, Val Acc=99.0\n",
      "Early stopping at epoch 9, step 239\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 55960 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 0.9760 Acc: 75.0\n",
      "Epoch 3: Val Loss=1.3220, Val Acc=57.0\n",
      "200 [7] Loss: 0.5424 Acc: 95.0\n",
      "Epoch 7: Val Loss=1.1486, Val Acc=61.0\n",
      "300 [11] Loss: 0.2729 Acc: 95.0\n",
      "Epoch 11: Val Loss=1.1032, Val Acc=63.0\n",
      "Early stopping at epoch 15, step 378\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 103348 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'hog']\n",
      "100 [3] Loss: 0.0470 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0396, Val Acc=100.0\n",
      "200 [7] Loss: 0.0166 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0189, Val Acc=100.0\n",
      "300 [11] Loss: 0.0091 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0117, Val Acc=100.0\n",
      "400 [15] Loss: 0.0070 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0085, Val Acc=100.0\n",
      "500 [19] Loss: 0.0054 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0064, Val Acc=100.0\n",
      "600 [23] Loss: 0.0043 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0052, Val Acc=100.0\n",
      "700 [27] Loss: 0.0016 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0043, Val Acc=100.0\n",
      "800 [31] Loss: 0.0020 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0036, Val Acc=100.0\n",
      "900 [35] Loss: 0.0018 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0031, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0011 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0027, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0012 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0024, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0011 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0022, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 40788 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0545 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0523, Val Acc=100.0\n",
      "200 [7] Loss: 0.0138 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0272, Val Acc=100.0\n",
      "300 [11] Loss: 0.0121 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0182, Val Acc=100.0\n",
      "400 [15] Loss: 0.0063 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0140, Val Acc=100.0\n",
      "500 [19] Loss: 0.0059 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0114, Val Acc=100.0\n",
      "600 [23] Loss: 0.0035 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0097, Val Acc=100.0\n",
      "700 [27] Loss: 0.0029 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0083, Val Acc=100.0\n",
      "800 [31] Loss: 0.0015 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0074, Val Acc=100.0\n",
      "900 [35] Loss: 0.0010 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0067, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0016 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0061, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0011 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0055, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0011 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0051, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41364 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0425 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0786, Val Acc=99.0\n",
      "200 [7] Loss: 0.0083 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0489, Val Acc=99.0\n",
      "300 [11] Loss: 0.0174 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0374, Val Acc=99.0\n",
      "400 [15] Loss: 0.0085 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0300, Val Acc=99.0\n",
      "500 [19] Loss: 0.0049 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0251, Val Acc=99.0\n",
      "600 [23] Loss: 0.0035 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0220, Val Acc=99.0\n",
      "700 [27] Loss: 0.0023 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0195, Val Acc=99.0\n",
      "800 [31] Loss: 0.0022 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0177, Val Acc=99.0\n",
      "900 [35] Loss: 0.0012 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0160, Val Acc=99.0\n",
      "1000 [39] Loss: 0.0015 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0146, Val Acc=99.0\n",
      "1100 [43] Loss: 0.0015 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0135, Val Acc=99.0\n",
      "1200 [47] Loss: 0.0009 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0126, Val Acc=99.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 43716 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0273 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0488, Val Acc=100.0\n",
      "200 [7] Loss: 0.0141 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0264, Val Acc=100.0\n",
      "300 [11] Loss: 0.0075 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0178, Val Acc=100.0\n",
      "400 [15] Loss: 0.0034 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0136, Val Acc=100.0\n",
      "500 [19] Loss: 0.0026 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0111, Val Acc=100.0\n",
      "600 [23] Loss: 0.0018 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0091, Val Acc=100.0\n",
      "700 [27] Loss: 0.0035 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0079, Val Acc=100.0\n",
      "800 [31] Loss: 0.0024 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0069, Val Acc=100.0\n",
      "900 [35] Loss: 0.0027 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0061, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0016 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0055, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0007 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0049, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0013 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0045, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41120 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models = {}\n",
    "for features_to_use in features_list:\n",
    "    print(f\"Training model with features: {features_to_use}\")\n",
    "    model_wrapper = LogisticRegressionModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader,\n",
    "        val_loader=val_loader,\n",
    "        test_loader=test_loader,\n",
    "        lr=LR\n",
    "    )\n",
    "    model_wrapper.train(epochs=50, verbose=1, plotting=True, patience=25)\n",
    "    models[\"+\".join(features_to_use)] = model_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "cc8c6a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summary_plot(models, additional_title=\"\", fig_width=fig_width, fig_height=fig_height, include_hog_legend=True):\n",
    "    \"\"\"\n",
    "    Utility to produce a bar plot that summarizes performance of the models trained with different low-level feature types.\n",
    "    \"\"\"\n",
    "    names = list(models.keys())\n",
    "    val_accs = [models[name].train_results[\"val_accs\"][-1] for name in names]\n",
    "    train_accs = [np.array(models[name].train_results[\"accuracies\"][-10:-1]).mean() for name in names]\n",
    "\n",
    "    # Sort by validation accuracy\n",
    "    sorted_data = sorted(zip(val_accs, train_accs, names))\n",
    "    val_accs_sorted, train_accs_sorted, names_sorted = zip(*sorted_data)\n",
    "\n",
    "    x = np.arange(len(names_sorted))\n",
    "    width = 0.5\n",
    "\n",
    "    # Define colors\n",
    "    color_hog = red\n",
    "    color_edge = blue\n",
    "    color_other = gray\n",
    "\n",
    "    bar_colors = []\n",
    "    for name in names_sorted:\n",
    "        if \"hog\" in name:\n",
    "            bar_colors.append(color_hog)\n",
    "        elif \"edge_dir_histo\" in name:\n",
    "            bar_colors.append(color_edge)\n",
    "        else:\n",
    "            bar_colors.append(color_other)\n",
    "\n",
    "    # Simplify names for x-axis\n",
    "    def simplify(name):\n",
    "        return \"+\".join([\n",
    "            \"C\" if \"color_histo\" in name else \"\",\n",
    "            \"E\" if \"edge_dir_histo\" in name else \"\",\n",
    "            \"R\" if \"rgb_cooc_mat\" in name else \"\",\n",
    "            \"H\" if \"hog\" in name else \"\"\n",
    "        ]).replace(\"++\", \"+\").strip(\"+\")\n",
    "\n",
    "    names_simple = [simplify(n) for n in names_sorted]\n",
    "\n",
    "    plt.figure(figsize=(fig_width, fig_height), dpi=300)\n",
    "\n",
    "    for i in range(len(names_sorted)):\n",
    "        plt.bar(x[i], val_accs_sorted[i], width=width, color=bar_colors[i])\n",
    "        plt.bar(x[i], train_accs_sorted[i], width=width, color=bar_colors[i], alpha=0.5)\n",
    "\n",
    "    plt.xticks(x, names_simple, rotation=90)\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.title(f\"Final Validation and Training Accuracy (sorted{'' if additional_title == '' else ' - ' + additional_title})\")\n",
    "    plt.grid(axis='y', linestyle='dotted')\n",
    "\n",
    "    # Custom legend\n",
    "    legend_elements = []\n",
    "    # This is a custom flag added to extend the function for the no hog case used in mlp\n",
    "    if include_hog_legend:\n",
    "        legend_elements.append(Patch(facecolor=color_hog, edgecolor='k', label='w/ HOG', alpha=1.0))\n",
    "    legend_elements.append(Patch(facecolor=color_edge, edgecolor='k', label='w/ Edge Dir', alpha=1.0))\n",
    "    legend_elements.append(Patch(facecolor=color_other, edgecolor='k', label='Other', alpha=1.0))\n",
    "    \n",
    "    plt.legend(handles=legend_elements, loc='upper left', fontsize=9)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "05b339f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 81928 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "summary_plot(models, additional_title=\"RGB Images\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "8fa7e4ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_low_variance_features(train_loader, val_loader, test_loader, threshold_percentile=70, features_to_use=[\"hog\"], standardize_feat=True, verbose=False):\n",
    "    \"\"\"\n",
    "    Utility function to remove low-variance features from the dataset. Retains the 100 - threshold_percentile % of features with the highest variance.\n",
    "    It expects image loaders to extract features from.\n",
    "    Returns new loaders with reduced feature sets.\n",
    "    \"\"\"\n",
    "\n",
    "    train_features, labels_train_features = extract_features(train_loader, features_to_use)\n",
    "    val_features, labels_val_features = extract_features(val_loader, features_to_use)\n",
    "    test_features, labels_test_features = extract_features(test_loader, features_to_use)\n",
    "\n",
    "    variances = train_features.var(dim=0)\n",
    "    threshold = torch.quantile(variances, threshold_percentile / 100.)\n",
    "    print(f\"Variance threshold for feature removal (percentile {threshold_percentile}): {threshold.item():.6f}\")\n",
    "    non_zero_variance_mask = variances > threshold\n",
    "    reduced_train_features = train_features[:, non_zero_variance_mask]\n",
    "    reduced_val_features = val_features[:, non_zero_variance_mask]\n",
    "    reduced_test_features = test_features[:, non_zero_variance_mask]\n",
    "    if standardize_feat:\n",
    "        reduced_train_features, mean, std = standardize(reduced_train_features)\n",
    "        reduced_val_features = (reduced_val_features - mean) / std\n",
    "        reduced_test_features = (reduced_test_features - mean) / std\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"Original features shape: {train_features.shape}\")\n",
    "        print(f\"Reduced features shape (removed zero-variance features): {reduced_train_features.shape}\")\n",
    "    \n",
    "    train_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_train_features, labels_train_features), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_val_features, labels_val_features), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_test_features, labels_test_features), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_reduced, val_loader_reduced, test_loader_reduced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "8a58ca1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variance threshold for feature removal (percentile 70): 0.012101\n",
      "Original features shape: torch.Size([788, 900])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 270])\n",
      "100 [3] Loss: 0.4439 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.3800, Val Acc=98.0\n",
      "200 [7] Loss: 0.2320 Acc: 95.0\n",
      "Epoch 7: Val Loss=0.1970, Val Acc=100.0\n",
      "300 [11] Loss: 0.1412 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.1351, Val Acc=100.0\n",
      "400 [15] Loss: 0.0461 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.1075, Val Acc=100.0\n",
      "500 [19] Loss: 0.0839 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0827, Val Acc=100.0\n",
      "600 [23] Loss: 0.0360 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0727, Val Acc=100.0\n",
      "Early stopping at epoch 25, step 634\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70512 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_loader_hog_reduced, val_loader_hog_reduced, test_loader_hog_reduced = remove_low_variance_features(\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    test_loader,\n",
    "    threshold_percentile=70,\n",
    "    features_to_use=[\"hog\"],\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_reduced = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_reduced,\n",
    "    val_loader=val_loader_hog_reduced,\n",
    "    test_loader=test_loader_hog_reduced,\n",
    "    classes=train_loader.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "\n",
    "model_hog_reduced.train(epochs=50, verbose=1, plotting=True, patience=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "75657d06",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pca(train_loader, val_loader, test_loader, mincomponents=1, retvar=0.95, verbose=False):\n",
    "    \"\"\"\n",
    "    Helper function to perform PCA on the given datasets.\n",
    "    Expects loaders with low-level features.\n",
    "    Return new loaders with reduced feature sets.\n",
    "    \"\"\"\n",
    "    train = train_loader.dataset.tensors[0]\n",
    "    val = val_loader.dataset.tensors[0]\n",
    "    test = test_loader.dataset.tensors[0]\n",
    "\n",
    "    mu = train.mean(0)\n",
    "    sigma = torch.cov(train.T)\n",
    "    eigvals, eigvecs = torch.linalg.eigh(sigma)\n",
    "    order = eigvals.argsort(descending=True)\n",
    "    eigvals = eigvals[order]\n",
    "    r = eigvals.cumsum(0) / eigvals.sum()\n",
    "    k = 1 + (r >= retvar).nonzero()[0][0]\n",
    "    k = max(k, mincomponents)\n",
    "    w = eigvecs[:, order[:k]]\n",
    "    train_pca = (train - mu) @ w\n",
    "    val_pca = (val - mu) @ w\n",
    "    test_pca = (test - mu) @ w\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"PCA reduced features from {train.shape[1]} to {k} components to retain {retvar*100:.2f}% variance.\")\n",
    "\n",
    "    train_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(train_pca, train_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(val_pca, val_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(test_pca, test_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_pca, val_loader_pca, test_loader_pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "dacada29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA reduced features from 270 to 57 components to retain 95.00% variance.\n",
      "100 [3] Loss: 1.2580 Acc: 80.0\n",
      "Epoch 3: Val Loss=1.2921, Val Acc=78.0\n",
      "200 [7] Loss: 0.7266 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.5756, Val Acc=93.0\n",
      "300 [11] Loss: 0.2234 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.3474, Val Acc=95.0\n",
      "400 [15] Loss: 0.2395 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.2500, Val Acc=96.0\n",
      "500 [19] Loss: 0.1944 Acc: 95.0\n",
      "Epoch 19: Val Loss=0.1961, Val Acc=96.0\n",
      "600 [23] Loss: 0.1972 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.1631, Val Acc=97.0\n",
      "700 [27] Loss: 0.1102 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.1406, Val Acc=97.0\n",
      "800 [31] Loss: 0.1703 Acc: 95.0\n",
      "Epoch 31: Val Loss=0.1249, Val Acc=97.0\n",
      "900 [35] Loss: 0.1582 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.1132, Val Acc=97.0\n",
      "1000 [39] Loss: 0.0681 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.1029, Val Acc=97.0\n",
      "1100 [43] Loss: 0.0963 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0955, Val Acc=97.0\n",
      "1200 [47] Loss: 0.0810 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0893, Val Acc=97.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72936 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_loader_hog_pca, val_loader_hog_pca, test_loader_hog_pca = pca(\n",
    "    train_loader_hog_reduced,\n",
    "    val_loader_hog_reduced,\n",
    "    test_loader_hog_reduced,\n",
    "    mincomponents=1,\n",
    "    retvar=0.95,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_pca = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_pca,\n",
    "    val_loader=val_loader_hog_pca,  \n",
    "    test_loader=test_loader_hog_pca,\n",
    "    classes=train_loader.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "model_hog_pca.train(epochs=50, verbose=1, plotting=True, patience=25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "dd3c835f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 97.5%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63756 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_hog_pca.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aee0e0f",
   "metadata": {},
   "source": [
    "But why is hog so effective? It becomes clear the reason when visualizing the image gradients:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "id": "4b33c3b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 160208 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "image = next(iter(train_loader))[0][0]\n",
    "_, hog_image = hog(\n",
    "    image.mean(0).numpy(),\n",
    "    orientations=9,\n",
    "    pixels_per_cell=(16, 16),\n",
    "    cells_per_block=(1, 1),\n",
    "    visualize=True\n",
    ")\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(fig_width, fig_height), sharex=True, sharey=True, dpi=300)\n",
    "\n",
    "ax1.axis('off')\n",
    "ax1.imshow(image.permute(1, 2, 0).numpy(), cmap=plt.cm.gray)\n",
    "ax1.set_title('Input image')\n",
    "\n",
    "# Rescale histogram for better display\n",
    "hog_image_rescaled = exposure.rescale_intensity(hog_image, in_range=(0, 10))\n",
    "\n",
    "ax2.axis('off')\n",
    "ax2.imshow(hog_image_rescaled, cmap=plt.cm.gray)\n",
    "ax2.set_title('Histogram of Oriented Gradients')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "992d4d8d",
   "metadata": {},
   "source": [
    "It's evident how the hog retains a lot of information about shape and edges, powerfully discriminating between signs (actually, this has a better resolution than the one used in training for visualization purposes: even with a lower number of cells (averaging on more pixels), the shapes were identified with very high accuracy by logistic regression)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "dd2bc848",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo']\n",
      "Early stopping at epoch 24, step 620\n",
      "Training model with features: ['edge_dir_histo']\n",
      "Early stopping at epoch 38, step 965\n",
      "Training model with features: ['rgb_cooc_mat']\n",
      "Training model with features: ['hog']\n",
      "Early stopping at epoch 8, step 209\n",
      "Training model with features: ['color_histo', 'edge_dir_histo']\n",
      "Early stopping at epoch 32, step 813\n",
      "Training model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 24, step 621\n",
      "Training model with features: ['color_histo', 'hog']\n",
      "Early stopping at epoch 33, step 843\n",
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 34, step 872\n",
      "Training model with features: ['edge_dir_histo', 'hog']\n",
      "Early stopping at epoch 20, step 507\n",
      "Training model with features: ['rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 18, step 470\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 32, step 824\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'hog']\n",
      "Early stopping at epoch 29, step 742\n",
      "Training model with features: ['color_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 28, step 718\n",
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 13, step 339\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 9, step 236\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 79936 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_gray = {}\n",
    "for features_to_use in features_list:\n",
    "    print(f\"Training model with features: {features_to_use}\")\n",
    "    model_wrapper = LogisticRegressionModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader_gray,\n",
    "        val_loader=val_loader_gray,\n",
    "        test_loader=test_loader_gray,\n",
    "        lr=LR\n",
    "    )\n",
    "    model_wrapper.train(epochs=50, verbose=0, plotting=False, patience=25)\n",
    "    models_gray[\"+\".join(features_to_use)] = model_wrapper\n",
    "\n",
    "summary_plot(models_gray, additional_title=\"Grayscale Images\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "ef49300c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variance threshold for feature removal (percentile 60): 0.010415\n",
      "Original features shape: torch.Size([788, 900])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 360])\n",
      "PCA reduced features from 360 to 69 components to retain 95.00% variance.\n",
      "100 [3] Loss: 1.6406 Acc: 50.0\n",
      "Epoch 3: Val Loss=1.2798, Val Acc=67.0\n",
      "200 [7] Loss: 0.5623 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.4562, Val Acc=96.0\n",
      "300 [11] Loss: 0.2994 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.2696, Val Acc=97.0\n",
      "400 [15] Loss: 0.1574 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.1969, Val Acc=98.0\n",
      "500 [19] Loss: 0.0951 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.1564, Val Acc=100.0\n",
      "600 [23] Loss: 0.1295 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.1298, Val Acc=100.0\n",
      "700 [27] Loss: 0.0685 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.1119, Val Acc=100.0\n",
      "800 [31] Loss: 0.0704 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0980, Val Acc=100.0\n",
      "900 [35] Loss: 0.0612 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0872, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0612 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0794, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0701 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0728, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0359 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0668, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 61536 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_loader_hog_reduced_gray, val_loader_hog_reduced_gray, test_loader_hog_reduced_gray = remove_low_variance_features(\n",
    "    train_loader_gray,\n",
    "    val_loader_gray,\n",
    "    test_loader_gray,\n",
    "    threshold_percentile=60,\n",
    "    features_to_use=[\"hog\"],\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "train_loader_hog_pca_gray, val_loader_hog_pca_gray, test_loader_hog_pca_gray = pca(\n",
    "    train_loader_hog_reduced_gray,\n",
    "    val_loader_hog_reduced_gray,\n",
    "    test_loader_hog_reduced_gray,\n",
    "    mincomponents=1,\n",
    "    retvar=0.95,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_pca_gray = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_pca_gray,\n",
    "    val_loader=val_loader_hog_pca_gray,  \n",
    "    test_loader=test_loader_hog_pca_gray,\n",
    "    classes=train_loader_gray.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "model_hog_pca_gray.train(epochs=50, verbose=1, plotting=True, patience=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "0b49ae7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 98.5%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63680 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_hog_pca_gray.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba424de6",
   "metadata": {},
   "source": [
    "# MLPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "fee28e59",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLPModelWrapper(ModelWrapper):\n",
    "    def __init__(\n",
    "            self, \n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR,\n",
    "            hidden_layers=[],\n",
    "            hidden_layer_reduction_factor=3,\n",
    "            apply_low_var_feat_reduction=False,\n",
    "            apply_pca=False\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Constructor for a MLP model using low-level features.\n",
    "        The hidden_layers argument expects a list defining the number of units in each hidden layer or the 'auto' string \n",
    "        E.g., hidden_layers=[512, 256, 128]\n",
    "        If the auto string is provided, the number of hidden layers will be determined with a reduction factor given by the relative argument (default 3).\n",
    "        The default is no hidden layers.\n",
    "        Pass ONLY image loaders.\n",
    "        \"\"\"\n",
    "        if train_loader is None or val_loader is None or test_loader is None or features_used is None:\n",
    "            raise ValueError(\"Please provide train_loader, val_loader, test_loader and features_used arguments.\")\n",
    "        feat_train_loader = None\n",
    "        feat_val_loader = None\n",
    "        feat_test_loader = None\n",
    "\n",
    "        if apply_low_var_feat_reduction:\n",
    "            # this feature reduction is more conservative since mlp should be able to not rely on low-variance features\n",
    "            # Removal of some low-variance feature helps with reducing the number of requires parameters, producing lighter models\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader = remove_low_variance_features(\n",
    "                train_loader,\n",
    "                val_loader,\n",
    "                test_loader,\n",
    "                threshold_percentile=50,\n",
    "                features_to_use=features_used,\n",
    "                verbose=True\n",
    "            )\n",
    "        else:\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader, _ = extract_and_standardize(\n",
    "                train_loader,\n",
    "                val_loader,\n",
    "                test_loader,\n",
    "                features_to_use=features_used\n",
    "            )\n",
    "\n",
    "        if apply_pca:\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader = pca(\n",
    "                feat_train_loader,\n",
    "                feat_val_loader,\n",
    "                feat_test_loader,\n",
    "                mincomponents=1,\n",
    "                retvar=0.95,\n",
    "                verbose=True\n",
    "            )\n",
    "        \n",
    "        local_classes = train_loader.dataset.classes\n",
    "\n",
    "        # Passing classes explicitly since using pre-processed loaders\n",
    "        super().__init__(\n",
    "            features_used=None, \n",
    "            train_loader=feat_train_loader, \n",
    "            val_loader=feat_val_loader, \n",
    "            test_loader=feat_test_loader, \n",
    "            lr=lr, \n",
    "            classes=local_classes\n",
    "        )\n",
    "\n",
    "        mlp_model = torch.nn.Sequential()\n",
    "        if hidden_layers is None or (isinstance(hidden_layers, list) and len(hidden_layers) == 0):\n",
    "            mlp_model.append(torch.nn.Linear(self.num_features, len(self.classes)))\n",
    "\n",
    "        elif isinstance(hidden_layers, str) and hidden_layers.lower() == \"auto\":\n",
    "            # this uses the fact that log_b(x) = log(x) / log(b)\n",
    "            n_layers = math.ceil(math.log(self.num_features / len(self.classes)) / math.log(hidden_layer_reduction_factor))\n",
    "            prev_dim = self.num_features\n",
    "            for _ in range(n_layers):\n",
    "                current_dim = prev_dim // hidden_layer_reduction_factor\n",
    "                if current_dim <= len(self.classes) * 2:\n",
    "                    break\n",
    "                mlp_model.extend([\n",
    "                    torch.nn.Linear(prev_dim, current_dim),\n",
    "                    torch.nn.ReLU()\n",
    "                ])\n",
    "                prev_dim = current_dim\n",
    "            mlp_model.append(torch.nn.Linear(prev_dim, len(self.classes)))\n",
    "            \n",
    "        elif isinstance(hidden_layers, list):\n",
    "            prev_dim = self.num_features\n",
    "            for hidden_dim in hidden_layers:\n",
    "                mlp_model.extend([\n",
    "                    torch.nn.Linear(prev_dim, hidden_dim),\n",
    "                    torch.nn.ReLU()\n",
    "                ])\n",
    "                prev_dim = hidden_dim   \n",
    "            mlp_model.append(torch.nn.Linear(prev_dim, len(self.classes)))\n",
    "            \n",
    "        else:\n",
    "            raise ValueError(\"Invalid hidden_layers argument: expected None, empty list, 'auto' string, or list of integers.\")\n",
    "            \n",
    "        mlp_model.to(DEVICE)\n",
    "        mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=lr)\n",
    "        mlp_loss_fun = torch.nn.CrossEntropyLoss()\n",
    "        self.build_model(mlp_model, mlp_optimizer, mlp_loss_fun)\n",
    "\n",
    "        print(\"\\nCreated MLP model with architecture:\")\n",
    "        for i, layer in enumerate(self.model):\n",
    "            if isinstance(layer, torch.nn.Linear):\n",
    "                print(f\"Layer {i}: {layer.in_features} -> {layer.out_features}\")\n",
    "        print(\"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "7c53ca7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variance threshold for feature removal (percentile 50): 0.009249\n",
      "Original features shape: torch.Size([788, 900])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 450])\n",
      "PCA reduced features from 450 to 80 components to retain 95.00% variance.\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 80 -> 50\n",
      "Layer 2: 50 -> 30\n",
      "Layer 4: 30 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 0.8276 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.9063, Val Acc=87.0\n",
      "200 [7] Loss: 0.1399 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.1669, Val Acc=98.0\n",
      "300 [11] Loss: 0.1734 Acc: 95.0\n",
      "Epoch 11: Val Loss=0.0675, Val Acc=100.0\n",
      "400 [15] Loss: 0.0159 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0395, Val Acc=100.0\n",
      "500 [19] Loss: 0.0231 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0266, Val Acc=100.0\n",
      "600 [23] Loss: 0.0043 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0198, Val Acc=100.0\n",
      "700 [27] Loss: 0.0040 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0158, Val Acc=100.0\n",
      "800 [31] Loss: 0.0026 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0128, Val Acc=100.0\n",
      "900 [35] Loss: 0.0029 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0113, Val Acc=100.0\n",
      "Early stopping at epoch 39, step 999\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 57296 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# lets try the firts one with only hog and feature redutcion + pca\n",
    "# since we know we get ~75 features after reduction, lets try a small MLP with 2 hidden layers with 35 and 10 weights\n",
    "mlp_model_hog_only = MLPModelWrapper(\n",
    "        features_used=[\"hog\"],\n",
    "        train_loader=train_loader,\n",
    "        val_loader=val_loader,\n",
    "        test_loader=test_loader,\n",
    "        lr=LR,\n",
    "        hidden_layers=[50, 30],\n",
    "        apply_low_var_feat_reduction=True,\n",
    "        apply_pca=True\n",
    "    )\n",
    "\n",
    "mlp_model_hog_only.train(epochs=50, verbose=1, plotting=True, patience=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "9c29bbd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 98.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63748 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mlp_model_hog_only.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c18f103",
   "metadata": {},
   "source": [
    "Since we already know hog is performing really well on this type of task, let's see if mlp can actually do better on everything that isn't hog. Let's use just variance dimensionality reduction and no pca, computing automatically the number and with of hidden layers (because very different number of features would need different hidden layer sizes). We used a reduction ratio of 3:1 between two consecutive hidden layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "ec003ecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000521\n",
      "Original features shape: torch.Size([788, 192])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 96])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 96 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.6040 Acc: 20.0\n",
      "Epoch 3: Val Loss=2.5555, Val Acc=20.0\n",
      "200 [7] Loss: 2.0440 Acc: 35.0\n",
      "Epoch 7: Val Loss=2.3952, Val Acc=22.0\n",
      "300 [11] Loss: 2.0909 Acc: 35.0\n",
      "Epoch 11: Val Loss=2.3225, Val Acc=25.0\n",
      "400 [15] Loss: 1.9550 Acc: 45.0\n",
      "Epoch 15: Val Loss=2.2781, Val Acc=26.0\n",
      "500 [19] Loss: 1.9840 Acc: 30.0\n",
      "Epoch 19: Val Loss=2.2553, Val Acc=27.0\n",
      "Early stopping at epoch 20, step 511\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 127520 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['edge_dir_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000045\n",
      "Original features shape: torch.Size([788, 64])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 32])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 32 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.1921 Acc: 20.0\n",
      "Epoch 3: Val Loss=2.4392, Val Acc=11.0\n",
      "200 [7] Loss: 2.2506 Acc: 25.0\n",
      "Epoch 7: Val Loss=2.1319, Val Acc=20.0\n",
      "300 [11] Loss: 1.7211 Acc: 30.0\n",
      "Epoch 11: Val Loss=1.9754, Val Acc=33.0\n",
      "400 [15] Loss: 1.7607 Acc: 60.0\n",
      "Epoch 15: Val Loss=1.8628, Val Acc=35.0\n",
      "500 [19] Loss: 1.7256 Acc: 40.0\n",
      "Epoch 19: Val Loss=1.7821, Val Acc=36.0\n",
      "600 [23] Loss: 1.8149 Acc: 55.0\n",
      "Epoch 23: Val Loss=1.7205, Val Acc=39.0\n",
      "700 [27] Loss: 1.4455 Acc: 55.0\n",
      "Epoch 27: Val Loss=1.6649, Val Acc=38.0\n",
      "800 [31] Loss: 1.4735 Acc: 70.0\n",
      "Epoch 31: Val Loss=1.6209, Val Acc=39.0\n",
      "900 [35] Loss: 1.4400 Acc: 50.0\n",
      "Epoch 35: Val Loss=1.5841, Val Acc=42.0\n",
      "1000 [39] Loss: 1.2871 Acc: 70.0\n",
      "Epoch 39: Val Loss=1.5496, Val Acc=40.0\n",
      "1100 [43] Loss: 1.4829 Acc: 50.0\n",
      "Epoch 43: Val Loss=1.5230, Val Acc=44.0\n",
      "1200 [47] Loss: 1.5047 Acc: 60.0\n",
      "Epoch 47: Val Loss=1.4984, Val Acc=46.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 114312 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000000\n",
      "Original features shape: torch.Size([788, 729])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 364])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 364 -> 121\n",
      "Layer 2: 121 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 1.4888 Acc: 45.0\n",
      "Epoch 3: Val Loss=2.1505, Val Acc=33.0\n",
      "Early stopping at epoch 7, step 199\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 119484 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'edge_dir_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000342\n",
      "Original features shape: torch.Size([788, 256])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 128])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 128 -> 42\n",
      "Layer 2: 42 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.2484 Acc: 20.0\n",
      "Epoch 3: Val Loss=2.3461, Val Acc=25.0\n",
      "200 [7] Loss: 1.3256 Acc: 65.0\n",
      "Epoch 7: Val Loss=1.9268, Val Acc=31.0\n",
      "300 [11] Loss: 1.2087 Acc: 55.0\n",
      "Epoch 11: Val Loss=1.7647, Val Acc=35.0\n",
      "400 [15] Loss: 1.1049 Acc: 65.0\n",
      "Epoch 15: Val Loss=1.7148, Val Acc=37.0\n",
      "Early stopping at epoch 18, step 465\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 128696 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000000\n",
      "Original features shape: torch.Size([788, 921])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 460])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 460 -> 153\n",
      "Layer 2: 153 -> 51\n",
      "Layer 4: 51 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 1.4809 Acc: 45.0\n",
      "Epoch 3: Val Loss=1.6840, Val Acc=38.0\n",
      "200 [7] Loss: 0.7756 Acc: 70.0\n",
      "Epoch 7: Val Loss=1.4660, Val Acc=52.0\n",
      "Early stopping at epoch 8, step 222\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 116028 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000000\n",
      "Original features shape: torch.Size([788, 793])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 396])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 396 -> 132\n",
      "Layer 2: 132 -> 44\n",
      "Layer 4: 44 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 0.7524 Acc: 80.0\n",
      "Epoch 3: Val Loss=1.1718, Val Acc=63.0\n",
      "200 [7] Loss: 0.4689 Acc: 80.0\n",
      "Epoch 7: Val Loss=0.8170, Val Acc=70.0\n",
      "Early stopping at epoch 10, step 255\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 114872 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000000\n",
      "Original features shape: torch.Size([788, 985])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 492])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 492 -> 164\n",
      "Layer 2: 164 -> 54\n",
      "Layer 4: 54 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 0.6703 Acc: 85.0\n",
      "Epoch 3: Val Loss=1.1554, Val Acc=53.0\n",
      "Early stopping at epoch 7, step 182\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 108156 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_mlp = {}\n",
    "no_hog_features_list = [f for f in features_list if \"hog\" not in f]\n",
    "for features_to_use in no_hog_features_list:\n",
    "    print(f\"Training MLP model with features: {features_to_use}\")\n",
    "    mlp_model_wrapper = MLPModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader,\n",
    "        val_loader=val_loader,\n",
    "        test_loader=test_loader,\n",
    "        lr=LR,\n",
    "        hidden_layers='auto',\n",
    "        hidden_layer_reduction_factor=3,\n",
    "        apply_low_var_feat_reduction=True,\n",
    "        apply_pca=False\n",
    "    )\n",
    "    mlp_model_wrapper.train(epochs=50, verbose=1, plotting=True, patience=25)\n",
    "    models_mlp[\"+\".join(features_to_use)] = mlp_model_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "5047a2db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 63992 characters]",
      "text/plain": [
       "<Figure size 1103.17x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "summary_plot(models_mlp, additional_title=\"MLP - No HOG\", fig_width=fig_height, fig_height=fig_height, include_hog_legend=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "b820a227",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000453\n",
      "Original features shape: torch.Size([788, 64])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 32])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 32 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.8604 Acc: 10.0\n",
      "Epoch 3: Val Loss=3.0449, Val Acc=5.0\n",
      "200 [7] Loss: 2.6635 Acc: 15.0\n",
      "Epoch 7: Val Loss=2.9217, Val Acc=5.0\n",
      "300 [11] Loss: 2.5140 Acc: 20.0\n",
      "Epoch 11: Val Loss=2.8443, Val Acc=7.0\n",
      "400 [15] Loss: 2.5576 Acc: 25.0\n",
      "Epoch 15: Val Loss=2.7938, Val Acc=9.0\n",
      "500 [19] Loss: 2.4089 Acc: 25.0\n",
      "Epoch 19: Val Loss=2.7603, Val Acc=11.0\n",
      "600 [23] Loss: 2.3875 Acc: 25.0\n",
      "Epoch 23: Val Loss=2.7355, Val Acc=12.0\n",
      "700 [27] Loss: 2.3351 Acc: 20.0\n",
      "Epoch 27: Val Loss=2.7182, Val Acc=11.0\n",
      "800 [31] Loss: 2.5647 Acc: 20.0\n",
      "Epoch 31: Val Loss=2.7061, Val Acc=11.0\n",
      "900 [35] Loss: 2.2651 Acc: 35.0\n",
      "Epoch 35: Val Loss=2.6935, Val Acc=11.0\n",
      "1000 [39] Loss: 2.0136 Acc: 35.0\n",
      "Epoch 39: Val Loss=2.6861, Val Acc=13.0\n",
      "1100 [43] Loss: 2.1828 Acc: 40.0\n",
      "Epoch 43: Val Loss=2.6772, Val Acc=12.0\n",
      "Early stopping at epoch 44, step 1121\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 119892 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['edge_dir_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000044\n",
      "Original features shape: torch.Size([788, 64])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 32])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 32 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.4816 Acc: 20.0\n",
      "Epoch 3: Val Loss=2.4074, Val Acc=19.0\n",
      "200 [7] Loss: 2.1064 Acc: 35.0\n",
      "Epoch 7: Val Loss=2.0805, Val Acc=26.0\n",
      "300 [11] Loss: 2.0809 Acc: 30.0\n",
      "Epoch 11: Val Loss=1.9090, Val Acc=35.0\n",
      "400 [15] Loss: 1.8773 Acc: 30.0\n",
      "Epoch 15: Val Loss=1.7950, Val Acc=38.0\n",
      "500 [19] Loss: 2.0144 Acc: 40.0\n",
      "Epoch 19: Val Loss=1.7111, Val Acc=42.0\n",
      "600 [23] Loss: 1.6375 Acc: 45.0\n",
      "Epoch 23: Val Loss=1.6453, Val Acc=43.0\n",
      "700 [27] Loss: 1.7776 Acc: 35.0\n",
      "Epoch 27: Val Loss=1.5936, Val Acc=46.0\n",
      "800 [31] Loss: 1.4327 Acc: 65.0\n",
      "Epoch 31: Val Loss=1.5459, Val Acc=46.0\n",
      "900 [35] Loss: 1.1671 Acc: 80.0\n",
      "Epoch 35: Val Loss=1.5105, Val Acc=47.0\n",
      "1000 [39] Loss: 1.2269 Acc: 60.0\n",
      "Epoch 39: Val Loss=1.4776, Val Acc=47.0\n",
      "1100 [43] Loss: 1.3669 Acc: 70.0\n",
      "Epoch 43: Val Loss=1.4550, Val Acc=45.0\n",
      "1200 [47] Loss: 1.4897 Acc: 55.0\n",
      "Epoch 47: Val Loss=1.4290, Val Acc=47.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 117344 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.001236\n",
      "Original features shape: torch.Size([788, 9])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 3])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 3 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 3.2340 Acc: 5.0\n",
      "Epoch 3: Val Loss=3.2634, Val Acc=6.0\n",
      "200 [7] Loss: 3.3060 Acc: 5.0\n",
      "Epoch 7: Val Loss=3.1645, Val Acc=6.0\n",
      "300 [11] Loss: 2.9455 Acc: 5.0\n",
      "Epoch 11: Val Loss=3.0863, Val Acc=5.0\n",
      "400 [15] Loss: 3.1496 Acc: 5.0\n",
      "Epoch 15: Val Loss=3.0254, Val Acc=5.0\n",
      "500 [19] Loss: 2.9212 Acc: 15.0\n",
      "Epoch 19: Val Loss=2.9761, Val Acc=5.0\n",
      "600 [23] Loss: 2.9217 Acc: 5.0\n",
      "Epoch 23: Val Loss=2.9370, Val Acc=7.0\n",
      "700 [27] Loss: 2.9565 Acc: 0.0\n",
      "Epoch 27: Val Loss=2.9060, Val Acc=14.0\n",
      "800 [31] Loss: 2.7294 Acc: 5.0\n",
      "Epoch 31: Val Loss=2.8809, Val Acc=14.0\n",
      "900 [35] Loss: 2.8840 Acc: 30.0\n",
      "Epoch 35: Val Loss=2.8617, Val Acc=14.0\n",
      "1000 [39] Loss: 2.9535 Acc: 10.0\n",
      "Epoch 39: Val Loss=2.8462, Val Acc=16.0\n",
      "1100 [43] Loss: 2.8478 Acc: 25.0\n",
      "Epoch 43: Val Loss=2.8341, Val Acc=17.0\n",
      "1200 [47] Loss: 2.9216 Acc: 20.0\n",
      "Epoch 47: Val Loss=2.8238, Val Acc=18.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 112976 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'edge_dir_histo']\n",
      "Variance threshold for feature removal (percentile 50): 0.000128\n",
      "Original features shape: torch.Size([788, 128])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 64])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 64 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.3416 Acc: 35.0\n",
      "Epoch 3: Val Loss=2.4933, Val Acc=20.0\n",
      "200 [7] Loss: 2.1314 Acc: 35.0\n",
      "Epoch 7: Val Loss=2.2376, Val Acc=22.0\n",
      "300 [11] Loss: 1.6972 Acc: 45.0\n",
      "Epoch 11: Val Loss=2.1164, Val Acc=22.0\n",
      "400 [15] Loss: 1.7077 Acc: 60.0\n",
      "Epoch 15: Val Loss=2.0492, Val Acc=27.0\n",
      "500 [19] Loss: 1.8402 Acc: 45.0\n",
      "Epoch 19: Val Loss=2.0024, Val Acc=28.0\n",
      "600 [23] Loss: 1.6407 Acc: 30.0\n",
      "Epoch 23: Val Loss=1.9750, Val Acc=31.0\n",
      "700 [27] Loss: 1.5307 Acc: 55.0\n",
      "Epoch 27: Val Loss=1.9547, Val Acc=33.0\n",
      "Early stopping at epoch 31, step 797\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 130124 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000524\n",
      "Original features shape: torch.Size([788, 73])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 36])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 36 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.9037 Acc: 15.0\n",
      "Epoch 3: Val Loss=2.8195, Val Acc=14.0\n",
      "200 [7] Loss: 2.6168 Acc: 10.0\n",
      "Epoch 7: Val Loss=2.6565, Val Acc=21.0\n",
      "300 [11] Loss: 2.5334 Acc: 15.0\n",
      "Epoch 11: Val Loss=2.5514, Val Acc=23.0\n",
      "400 [15] Loss: 2.4819 Acc: 25.0\n",
      "Epoch 15: Val Loss=2.4717, Val Acc=28.0\n",
      "500 [19] Loss: 2.3398 Acc: 30.0\n",
      "Epoch 19: Val Loss=2.4135, Val Acc=28.0\n",
      "600 [23] Loss: 2.3899 Acc: 35.0\n",
      "Epoch 23: Val Loss=2.3668, Val Acc=30.0\n",
      "700 [27] Loss: 2.2994 Acc: 30.0\n",
      "Epoch 27: Val Loss=2.3302, Val Acc=32.0\n",
      "800 [31] Loss: 2.0655 Acc: 45.0\n",
      "Epoch 31: Val Loss=2.3039, Val Acc=32.0\n",
      "900 [35] Loss: 2.1887 Acc: 30.0\n",
      "Epoch 35: Val Loss=2.2781, Val Acc=32.0\n",
      "1000 [39] Loss: 2.0017 Acc: 20.0\n",
      "Epoch 39: Val Loss=2.2537, Val Acc=35.0\n",
      "1100 [43] Loss: 2.2378 Acc: 45.0\n",
      "Epoch 43: Val Loss=2.2329, Val Acc=34.0\n",
      "1200 [47] Loss: 2.2324 Acc: 30.0\n",
      "Epoch 47: Val Loss=2.2177, Val Acc=34.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 118604 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000050\n",
      "Original features shape: torch.Size([788, 73])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 36])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 36 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.3145 Acc: 30.0\n",
      "Epoch 3: Val Loss=2.4033, Val Acc=19.0\n",
      "200 [7] Loss: 2.0988 Acc: 25.0\n",
      "Epoch 7: Val Loss=2.0686, Val Acc=33.0\n",
      "300 [11] Loss: 2.0887 Acc: 50.0\n",
      "Epoch 11: Val Loss=1.8888, Val Acc=33.0\n",
      "400 [15] Loss: 1.8741 Acc: 35.0\n",
      "Epoch 15: Val Loss=1.7772, Val Acc=35.0\n",
      "500 [19] Loss: 1.4860 Acc: 65.0\n",
      "Epoch 19: Val Loss=1.6913, Val Acc=38.0\n",
      "600 [23] Loss: 1.2876 Acc: 75.0\n",
      "Epoch 23: Val Loss=1.6322, Val Acc=44.0\n",
      "700 [27] Loss: 1.5582 Acc: 55.0\n",
      "Epoch 27: Val Loss=1.5827, Val Acc=47.0\n",
      "800 [31] Loss: 1.4134 Acc: 70.0\n",
      "Epoch 31: Val Loss=1.5418, Val Acc=47.0\n",
      "900 [35] Loss: 1.2662 Acc: 55.0\n",
      "Epoch 35: Val Loss=1.5090, Val Acc=44.0\n",
      "1000 [39] Loss: 1.4287 Acc: 55.0\n",
      "Epoch 39: Val Loss=1.4809, Val Acc=48.0\n",
      "1100 [43] Loss: 1.1805 Acc: 65.0\n",
      "Epoch 43: Val Loss=1.4575, Val Acc=45.0\n",
      "1200 [47] Loss: 1.1484 Acc: 65.0\n",
      "Epoch 47: Val Loss=1.4340, Val Acc=46.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 118304 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "Variance threshold for feature removal (percentile 50): 0.000176\n",
      "Original features shape: torch.Size([788, 137])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 68])\n",
      "\n",
      "Created MLP model with architecture:\n",
      "Layer 0: 68 -> 20\n",
      "\n",
      "\n",
      "100 [3] Loss: 2.4471 Acc: 50.0\n",
      "Epoch 3: Val Loss=2.4878, Val Acc=23.0\n",
      "200 [7] Loss: 2.2377 Acc: 25.0\n",
      "Epoch 7: Val Loss=2.2313, Val Acc=31.0\n",
      "300 [11] Loss: 1.9362 Acc: 45.0\n",
      "Epoch 11: Val Loss=2.0973, Val Acc=29.0\n",
      "400 [15] Loss: 2.0254 Acc: 40.0\n",
      "Epoch 15: Val Loss=2.0140, Val Acc=31.0\n",
      "500 [19] Loss: 1.4916 Acc: 70.0\n",
      "Epoch 19: Val Loss=1.9527, Val Acc=33.0\n",
      "600 [23] Loss: 1.6764 Acc: 50.0\n",
      "Epoch 23: Val Loss=1.9137, Val Acc=31.0\n",
      "700 [27] Loss: 1.6109 Acc: 45.0\n",
      "Epoch 27: Val Loss=1.8744, Val Acc=33.0\n",
      "800 [31] Loss: 1.4698 Acc: 65.0\n",
      "Epoch 31: Val Loss=1.8446, Val Acc=33.0\n",
      "900 [35] Loss: 1.3929 Acc: 55.0\n",
      "Epoch 35: Val Loss=1.8216, Val Acc=34.0\n",
      "1000 [39] Loss: 1.2255 Acc: 80.0\n",
      "Epoch 39: Val Loss=1.8001, Val Acc=37.0\n",
      "1100 [43] Loss: 1.1493 Acc: 65.0\n",
      "Epoch 43: Val Loss=1.7841, Val Acc=38.0\n",
      "1200 [47] Loss: 1.0832 Acc: 60.0\n",
      "Epoch 47: Val Loss=1.7689, Val Acc=38.0\n",
      "Early stopping at epoch 49, step 1241\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 118204 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_mlp_gray = {}\n",
    "for features_to_use in no_hog_features_list:\n",
    "    print(f\"Training MLP model with features: {features_to_use}\")\n",
    "    mlp_model_wrapper = MLPModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader_gray,\n",
    "        val_loader=val_loader_gray,\n",
    "        test_loader=test_loader_gray,\n",
    "        lr=LR,\n",
    "        hidden_layers='auto',\n",
    "        hidden_layer_reduction_factor=3,\n",
    "        apply_low_var_feat_reduction=True,\n",
    "        apply_pca=False\n",
    "    )\n",
    "    mlp_model_wrapper.train(epochs=50, verbose=1, plotting=True, patience=25)\n",
    "    models_mlp_gray[\"+\".join(features_to_use)] = mlp_model_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "135dcd0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 71888 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def comparison_plot(models_rgb, models_gray, additional_title=\"\", fig_width=fig_width, fig_height=fig_height):\n",
    "    \"\"\"\n",
    "    Bar plot comparing RGB vs Grayscale MLP models validation accuracy side by side.\n",
    "    Solid bars = val acc, semi-transparent = train acc.\n",
    "    \"\"\"\n",
    "    names_rgb = set(models_rgb.keys())\n",
    "    names_gray = set(models_gray.keys())\n",
    "    names = sorted(names_rgb & names_gray)\n",
    "\n",
    "    val_accs_rgb   = [models_rgb[n].train_results[\"val_accs\"][-1] for n in names]\n",
    "    val_accs_gray  = [models_gray[n].train_results[\"val_accs\"][-1] for n in names]\n",
    "    train_accs_rgb  = [np.array(models_rgb[n].train_results[\"accuracies\"][-10:-1]).mean() for n in names]\n",
    "    train_accs_gray = [np.array(models_gray[n].train_results[\"accuracies\"][-10:-1]).mean() for n in names]\n",
    "\n",
    "    # sort by rgb val acc\n",
    "    sorted_data = sorted(zip(val_accs_rgb, val_accs_gray, train_accs_rgb, train_accs_gray, names))\n",
    "    val_accs_rgb_s, val_accs_gray_s, train_accs_rgb_s, train_accs_gray_s, names_sorted = zip(*sorted_data)\n",
    "\n",
    "    def simplify(name):\n",
    "        return \"+\".join([\n",
    "            \"C\" if \"color_histo\" in name else \"\",\n",
    "            \"E\" if \"edge_dir_histo\" in name else \"\",\n",
    "            \"R\" if \"rgb_cooc_mat\" in name else \"\",\n",
    "        ]).replace(\"++\", \"+\").strip(\"+\")\n",
    "\n",
    "    names_simple = [simplify(n) for n in names_sorted]\n",
    "\n",
    "    x = np.arange(len(names_sorted))\n",
    "    width = 0.35\n",
    "\n",
    "    plt.figure(figsize=(fig_width, fig_height), dpi=300)\n",
    "\n",
    "    # val acc (solid)\n",
    "    plt.bar(x - width/2, val_accs_rgb_s,  width=width, color=red, label='RGB val')\n",
    "    plt.bar(x + width/2, val_accs_gray_s, width=width, color=blue, label='Grayscale val')\n",
    "    # train acc (transparent overlay)\n",
    "    plt.bar(x - width/2, train_accs_rgb_s,  width=width, color=red, alpha=0.4, label='RGB train')\n",
    "    plt.bar(x + width/2, train_accs_gray_s, width=width, color=blue, alpha=0.4, label='Grayscale train')\n",
    "\n",
    "    plt.xticks(x, names_simple, rotation=90)\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.title(f\"RGB vs Grayscale MLP Accuracy{'' if additional_title == '' else ' - ' + additional_title}\")\n",
    "    plt.grid(axis='y', linestyle='dotted')\n",
    "\n",
    "    legend_elements = [\n",
    "        Patch(facecolor=red, edgecolor='k', label='RGB val'),\n",
    "        Patch(facecolor=red, edgecolor='k', alpha=0.4, label='RGB train'),\n",
    "        Patch(facecolor=blue, edgecolor='k', label='Grayscale val'),\n",
    "        Patch(facecolor=blue, edgecolor='k', alpha=0.4, label='Grayscale train'),\n",
    "    ]\n",
    "    plt.legend(handles=legend_elements, loc='upper left', fontsize=8)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "comparison_plot(models_mlp, models_mlp_gray, additional_title=\"No HOG\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "id": "a64865ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.001"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9249b7a3",
   "metadata": {},
   "source": [
    "# CNN\n",
    "\n",
    "In the following part a simple convolutional neural network is implemented to evaluate the performance in comparison with classical metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "c6fc1efd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 [0] Loss: 2.4435 Acc: 37.5\n",
      "Epoch 0: Val Loss=3.2623, Val Acc=5.0\n",
      "20 [0] Loss: 2.6075 Acc: 18.8\n",
      "Epoch 0: Val Loss=2.9072, Val Acc=14.0\n",
      "30 [1] Loss: 2.7349 Acc: 21.9\n",
      "Epoch 1: Val Loss=2.4989, Val Acc=20.0\n",
      "40 [1] Loss: 1.7482 Acc: 37.5\n",
      "Epoch 1: Val Loss=1.9440, Val Acc=39.0\n",
      "50 [1] Loss: 2.2015 Acc: 35.0\n",
      "Epoch 1: Val Loss=1.3132, Val Acc=59.0\n",
      "60 [2] Loss: 1.3836 Acc: 53.1\n",
      "Epoch 2: Val Loss=0.9213, Val Acc=72.0\n",
      "70 [2] Loss: 1.4316 Acc: 53.1\n",
      "Epoch 2: Val Loss=0.7329, Val Acc=78.0\n",
      "80 [3] Loss: 1.4660 Acc: 40.6\n",
      "Epoch 3: Val Loss=0.7121, Val Acc=75.0\n",
      "90 [3] Loss: 0.8756 Acc: 65.6\n",
      "Epoch 3: Val Loss=0.6482, Val Acc=79.0\n",
      "100 [3] Loss: 1.1413 Acc: 60.0\n",
      "Epoch 3: Val Loss=0.4125, Val Acc=95.0\n",
      "110 [4] Loss: 1.0978 Acc: 59.4\n",
      "Epoch 4: Val Loss=0.4008, Val Acc=90.0\n",
      "120 [4] Loss: 0.8498 Acc: 68.8\n",
      "Epoch 4: Val Loss=0.3253, Val Acc=89.0\n",
      "130 [5] Loss: 0.8867 Acc: 62.5\n",
      "Epoch 5: Val Loss=0.3389, Val Acc=94.0\n",
      "140 [5] Loss: 0.8383 Acc: 68.8\n",
      "Epoch 5: Val Loss=0.2892, Val Acc=91.0\n",
      "150 [5] Loss: 0.8943 Acc: 65.0\n",
      "Epoch 5: Val Loss=0.2398, Val Acc=96.0\n",
      "160 [6] Loss: 0.7991 Acc: 62.5\n",
      "Epoch 6: Val Loss=0.2932, Val Acc=90.0\n",
      "170 [6] Loss: 0.5631 Acc: 81.2\n",
      "Epoch 6: Val Loss=0.2513, Val Acc=95.0\n",
      "180 [7] Loss: 0.4665 Acc: 84.4\n",
      "Epoch 7: Val Loss=0.2128, Val Acc=94.0\n",
      "190 [7] Loss: 0.5778 Acc: 81.2\n",
      "Epoch 7: Val Loss=0.1726, Val Acc=95.0\n",
      "200 [7] Loss: 0.5751 Acc: 85.0\n",
      "Epoch 7: Val Loss=0.3949, Val Acc=83.0\n",
      "210 [8] Loss: 0.5752 Acc: 81.2\n",
      "Epoch 8: Val Loss=0.2450, Val Acc=90.0\n",
      "220 [8] Loss: 0.9792 Acc: 59.4\n",
      "Epoch 8: Val Loss=0.1719, Val Acc=95.0\n",
      "230 [9] Loss: 0.6329 Acc: 78.1\n",
      "Epoch 9: Val Loss=0.1162, Val Acc=98.0\n",
      "240 [9] Loss: 0.5806 Acc: 81.2\n",
      "Epoch 9: Val Loss=0.1032, Val Acc=98.0\n",
      "250 [9] Loss: 0.4305 Acc: 90.0\n",
      "Epoch 9: Val Loss=0.1258, Val Acc=100.0\n",
      "260 [10] Loss: 0.5788 Acc: 75.0\n",
      "Epoch 10: Val Loss=0.1408, Val Acc=98.0\n",
      "270 [10] Loss: 0.2342 Acc: 87.5\n",
      "Epoch 10: Val Loss=0.0931, Val Acc=98.0\n",
      "280 [11] Loss: 0.5529 Acc: 78.1\n",
      "Epoch 11: Val Loss=0.0702, Val Acc=97.0\n",
      "290 [11] Loss: 0.4494 Acc: 84.4\n",
      "Epoch 11: Val Loss=0.0821, Val Acc=98.0\n",
      "300 [11] Loss: 0.2503 Acc: 90.0\n",
      "Epoch 11: Val Loss=0.1001, Val Acc=97.0\n",
      "310 [12] Loss: 0.4648 Acc: 87.5\n",
      "Epoch 12: Val Loss=0.0701, Val Acc=99.0\n",
      "320 [12] Loss: 0.4309 Acc: 81.2\n",
      "Epoch 12: Val Loss=0.0669, Val Acc=100.0\n",
      "330 [13] Loss: 0.2383 Acc: 90.6\n",
      "Epoch 13: Val Loss=0.0651, Val Acc=98.0\n",
      "340 [13] Loss: 0.2300 Acc: 93.8\n",
      "Epoch 13: Val Loss=0.0738, Val Acc=98.0\n",
      "350 [13] Loss: 0.4821 Acc: 85.0\n",
      "Epoch 13: Val Loss=0.0618, Val Acc=99.0\n",
      "360 [14] Loss: 0.3873 Acc: 84.4\n",
      "Epoch 14: Val Loss=0.0621, Val Acc=98.0\n",
      "370 [14] Loss: 0.2278 Acc: 90.6\n",
      "Epoch 14: Val Loss=0.0335, Val Acc=99.0\n",
      "380 [15] Loss: 0.2960 Acc: 90.6\n",
      "Epoch 15: Val Loss=0.1147, Val Acc=97.0\n",
      "390 [15] Loss: 0.3877 Acc: 87.5\n",
      "Epoch 15: Val Loss=0.0529, Val Acc=100.0\n",
      "400 [15] Loss: 0.3553 Acc: 85.0\n",
      "Epoch 15: Val Loss=0.0820, Val Acc=97.0\n",
      "410 [16] Loss: 0.5376 Acc: 84.4\n",
      "Epoch 16: Val Loss=0.0588, Val Acc=98.0\n",
      "Learning rate reduced: 0.001 -> 0.0001\n",
      "420 [16] Loss: 0.2869 Acc: 87.5\n",
      "Epoch 16: Val Loss=0.0401, Val Acc=100.0\n",
      "430 [17] Loss: 0.2402 Acc: 93.8\n",
      "Epoch 17: Val Loss=0.0439, Val Acc=99.0\n",
      "440 [17] Loss: 0.2869 Acc: 87.5\n",
      "Epoch 17: Val Loss=0.0428, Val Acc=99.0\n",
      "450 [17] Loss: 0.3953 Acc: 80.0\n",
      "Epoch 17: Val Loss=0.0386, Val Acc=99.0\n",
      "460 [18] Loss: 0.3949 Acc: 84.4\n",
      "Epoch 18: Val Loss=0.0350, Val Acc=99.0\n",
      "470 [18] Loss: 0.4479 Acc: 81.2\n",
      "Epoch 18: Val Loss=0.0338, Val Acc=99.0\n",
      "480 [19] Loss: 0.3097 Acc: 87.5\n",
      "Epoch 19: Val Loss=0.0285, Val Acc=99.0\n",
      "490 [19] Loss: 0.2520 Acc: 84.4\n",
      "Epoch 19: Val Loss=0.0264, Val Acc=99.0\n",
      "500 [19] Loss: 0.3703 Acc: 85.0\n",
      "Epoch 19: Val Loss=0.0249, Val Acc=99.0\n",
      "510 [20] Loss: 0.5106 Acc: 71.9\n",
      "Epoch 20: Val Loss=0.0231, Val Acc=99.0\n",
      "520 [20] Loss: 0.1584 Acc: 93.8\n",
      "Epoch 20: Val Loss=0.0212, Val Acc=100.0\n",
      "530 [21] Loss: 0.2531 Acc: 87.5\n",
      "Epoch 21: Val Loss=0.0201, Val Acc=100.0\n",
      "540 [21] Loss: 0.4033 Acc: 78.1\n",
      "Epoch 21: Val Loss=0.0206, Val Acc=99.0\n",
      "550 [21] Loss: 0.2845 Acc: 85.0\n",
      "Epoch 21: Val Loss=0.0215, Val Acc=99.0\n",
      "560 [22] Loss: 0.3805 Acc: 84.4\n",
      "Epoch 22: Val Loss=0.0205, Val Acc=99.0\n",
      "570 [22] Loss: 0.2792 Acc: 87.5\n",
      "Epoch 22: Val Loss=0.0201, Val Acc=99.0\n",
      "580 [23] Loss: 0.0982 Acc: 96.9\n",
      "Epoch 23: Val Loss=0.0195, Val Acc=99.0\n",
      "590 [23] Loss: 0.2359 Acc: 90.6\n",
      "Epoch 23: Val Loss=0.0180, Val Acc=100.0\n",
      "600 [23] Loss: 0.2579 Acc: 90.0\n",
      "Epoch 23: Val Loss=0.0168, Val Acc=100.0\n",
      "610 [24] Loss: 0.1679 Acc: 90.6\n",
      "Epoch 24: Val Loss=0.0169, Val Acc=99.0\n",
      "620 [24] Loss: 0.2633 Acc: 90.6\n",
      "Epoch 24: Val Loss=0.0168, Val Acc=99.0\n",
      "630 [25] Loss: 0.1612 Acc: 93.8\n",
      "Epoch 25: Val Loss=0.0165, Val Acc=100.0\n",
      "640 [25] Loss: 0.3947 Acc: 84.4\n",
      "Epoch 25: Val Loss=0.0151, Val Acc=100.0\n",
      "650 [25] Loss: 0.2353 Acc: 95.0\n",
      "Epoch 25: Val Loss=0.0162, Val Acc=99.0\n",
      "660 [26] Loss: 0.2150 Acc: 93.8\n",
      "Epoch 26: Val Loss=0.0160, Val Acc=99.0\n",
      "670 [26] Loss: 0.0904 Acc: 96.9\n",
      "Epoch 26: Val Loss=0.0166, Val Acc=99.0\n",
      "680 [27] Loss: 0.2066 Acc: 93.8\n",
      "Epoch 27: Val Loss=0.0177, Val Acc=99.0\n",
      "Learning rate reduced: 0.0001 -> 1e-05\n",
      "690 [27] Loss: 0.3868 Acc: 84.4\n",
      "Epoch 27: Val Loss=0.0177, Val Acc=99.0\n",
      "700 [27] Loss: 0.0287 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0187, Val Acc=99.0\n",
      "710 [28] Loss: 0.0347 Acc: 100.0\n",
      "Epoch 28: Val Loss=0.0186, Val Acc=99.0\n",
      "720 [28] Loss: 0.1364 Acc: 96.9\n",
      "Epoch 28: Val Loss=0.0185, Val Acc=99.0\n",
      "730 [29] Loss: 0.1476 Acc: 96.9\n",
      "Epoch 29: Val Loss=0.0181, Val Acc=99.0\n",
      "740 [29] Loss: 0.1960 Acc: 93.8\n",
      "Epoch 29: Val Loss=0.0183, Val Acc=99.0\n",
      "Learning rate reduced: 1e-05 -> 1.0000000000000002e-06\n",
      "750 [29] Loss: 0.4107 Acc: 90.0\n",
      "Epoch 29: Val Loss=0.0186, Val Acc=99.0\n",
      "760 [30] Loss: 0.1852 Acc: 93.8\n",
      "Epoch 30: Val Loss=0.0179, Val Acc=99.0\n",
      "770 [30] Loss: 0.1313 Acc: 96.9\n",
      "Epoch 30: Val Loss=0.0187, Val Acc=99.0\n",
      "780 [31] Loss: 0.2006 Acc: 90.6\n",
      "Epoch 31: Val Loss=0.0189, Val Acc=99.0\n",
      "790 [31] Loss: 0.0441 Acc: 96.9\n",
      "Epoch 31: Val Loss=0.0183, Val Acc=99.0\n",
      "800 [31] Loss: 0.0923 Acc: 95.0\n",
      "Epoch 31: Val Loss=0.0178, Val Acc=99.0\n",
      "810 [32] Loss: 0.3107 Acc: 87.5\n",
      "Epoch 32: Val Loss=0.0171, Val Acc=99.0\n",
      "820 [32] Loss: 0.0966 Acc: 96.9\n",
      "Epoch 32: Val Loss=0.0181, Val Acc=99.0\n",
      "830 [33] Loss: 0.2717 Acc: 87.5\n",
      "Epoch 33: Val Loss=0.0176, Val Acc=99.0\n",
      "Early stopping at epoch 33, step 839\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 97088 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnn_model = torch.nn.Sequential(\n",
    "    torch.nn.Conv2d(3, 16, kernel_size=5, padding=2),\n",
    "    torch.nn.BatchNorm2d(16),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 100 x 100\n",
    "    torch.nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "    torch.nn.BatchNorm2d(32),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 50 x 50\n",
    "    torch.nn.Conv2d(32, 64, kernel_size=3, padding=1),\n",
    "    torch.nn.BatchNorm2d(64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 25 x 25\n",
    "    torch.nn.Flatten(),\n",
    "    torch.nn.Linear(64 * 25 * 25, 256),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Dropout(0.5),\n",
    "    torch.nn.Linear(256, len(train_loader.dataset.classes))\n",
    ")\n",
    "cnn_model.to(DEVICE)\n",
    "cnn_optimizer = torch.optim.Adam(cnn_model.parameters(), lr=LR)\n",
    "cnn_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(cnn_optimizer, mode='min', factor=0.1, patience=50, min_lr=1e-6)\n",
    "cnn_loss_fun = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "cnn_steps, cnn_accuracies, cnn_losses, cnn_val_accs, cnn_val_losses = train(\n",
    "    cnn_model,\n",
    "    cnn_optimizer,\n",
    "    cnn_loss_fun,\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    epochs=50,\n",
    "    verbose=2,\n",
    "    plotting=True,\n",
    "    patience=200,\n",
    "    scheduler=cnn_scheduler\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "id": "03c0d88f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 100.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63588 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_conf_mat(test_conf_mat(cnn_model, test_loader, nclasses=len(train_loader.dataset.classes), verbose=True), train_loader.dataset.classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8e2afa4",
   "metadata": {},
   "source": [
    "And now it would be interesting to see the comparison with grayscale images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "ba259d2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 [0] Loss: 6.6252 Acc: 6.2\n",
      "Epoch 0: Val Loss=2.9040, Val Acc=10.0\n",
      "20 [0] Loss: 2.7106 Acc: 25.0\n",
      "Epoch 0: Val Loss=2.8356, Val Acc=16.0\n",
      "30 [1] Loss: 2.1122 Acc: 21.9\n",
      "Epoch 1: Val Loss=2.4850, Val Acc=22.0\n",
      "40 [1] Loss: 1.2523 Acc: 62.5\n",
      "Epoch 1: Val Loss=1.7472, Val Acc=49.0\n",
      "50 [1] Loss: 1.3964 Acc: 55.0\n",
      "Epoch 1: Val Loss=1.2559, Val Acc=69.0\n",
      "60 [2] Loss: 1.0342 Acc: 62.5\n",
      "Epoch 2: Val Loss=0.8430, Val Acc=82.0\n",
      "70 [2] Loss: 1.5962 Acc: 46.9\n",
      "Epoch 2: Val Loss=0.7233, Val Acc=86.0\n",
      "80 [3] Loss: 1.0901 Acc: 62.5\n",
      "Epoch 3: Val Loss=0.4754, Val Acc=84.0\n",
      "90 [3] Loss: 0.6960 Acc: 81.2\n",
      "Epoch 3: Val Loss=0.4469, Val Acc=91.0\n",
      "100 [3] Loss: 0.8370 Acc: 80.0\n",
      "Epoch 3: Val Loss=0.3662, Val Acc=91.0\n",
      "110 [4] Loss: 0.4139 Acc: 87.5\n",
      "Epoch 4: Val Loss=0.4051, Val Acc=89.0\n",
      "120 [4] Loss: 0.6170 Acc: 75.0\n",
      "Epoch 4: Val Loss=0.2704, Val Acc=90.0\n",
      "130 [5] Loss: 0.5726 Acc: 78.1\n",
      "Epoch 5: Val Loss=0.2756, Val Acc=92.0\n",
      "140 [5] Loss: 0.4742 Acc: 81.2\n",
      "Epoch 5: Val Loss=0.7559, Val Acc=71.0\n",
      "150 [5] Loss: 0.5775 Acc: 80.0\n",
      "Epoch 5: Val Loss=0.1444, Val Acc=95.0\n",
      "160 [6] Loss: 0.6759 Acc: 87.5\n",
      "Epoch 6: Val Loss=0.1257, Val Acc=98.0\n",
      "170 [6] Loss: 0.4482 Acc: 84.4\n",
      "Epoch 6: Val Loss=0.1278, Val Acc=95.0\n",
      "180 [7] Loss: 0.4529 Acc: 84.4\n",
      "Epoch 7: Val Loss=0.1768, Val Acc=93.0\n",
      "190 [7] Loss: 0.3844 Acc: 84.4\n",
      "Epoch 7: Val Loss=0.1054, Val Acc=97.0\n",
      "200 [7] Loss: 0.2819 Acc: 85.0\n",
      "Epoch 7: Val Loss=0.0850, Val Acc=99.0\n",
      "210 [8] Loss: 0.4028 Acc: 87.5\n",
      "Epoch 8: Val Loss=0.0798, Val Acc=97.0\n",
      "220 [8] Loss: 0.6945 Acc: 84.4\n",
      "Epoch 8: Val Loss=0.0708, Val Acc=97.0\n",
      "230 [9] Loss: 0.2063 Acc: 90.6\n",
      "Epoch 9: Val Loss=0.0609, Val Acc=97.0\n",
      "240 [9] Loss: 0.2506 Acc: 90.6\n",
      "Epoch 9: Val Loss=0.0798, Val Acc=98.0\n",
      "250 [9] Loss: 0.6460 Acc: 70.0\n",
      "Epoch 9: Val Loss=0.0776, Val Acc=97.0\n",
      "260 [10] Loss: 0.2857 Acc: 93.8\n",
      "Epoch 10: Val Loss=0.0639, Val Acc=98.0\n",
      "270 [10] Loss: 0.3324 Acc: 81.2\n",
      "Epoch 10: Val Loss=0.0626, Val Acc=98.0\n",
      "Learning rate reduced: 0.001 -> 0.0001\n",
      "280 [11] Loss: 0.2945 Acc: 90.6\n",
      "Epoch 11: Val Loss=0.0710, Val Acc=99.0\n",
      "290 [11] Loss: 0.2794 Acc: 90.6\n",
      "Epoch 11: Val Loss=0.0478, Val Acc=99.0\n",
      "300 [11] Loss: 0.2209 Acc: 85.0\n",
      "Epoch 11: Val Loss=0.0468, Val Acc=99.0\n",
      "310 [12] Loss: 0.1667 Acc: 93.8\n",
      "Epoch 12: Val Loss=0.0396, Val Acc=99.0\n",
      "320 [12] Loss: 0.1180 Acc: 96.9\n",
      "Epoch 12: Val Loss=0.0335, Val Acc=99.0\n",
      "330 [13] Loss: 0.3688 Acc: 87.5\n",
      "Epoch 13: Val Loss=0.0250, Val Acc=99.0\n",
      "340 [13] Loss: 0.0949 Acc: 96.9\n",
      "Epoch 13: Val Loss=0.0226, Val Acc=100.0\n",
      "350 [13] Loss: 0.2467 Acc: 90.0\n",
      "Epoch 13: Val Loss=0.0210, Val Acc=100.0\n",
      "360 [14] Loss: 0.2439 Acc: 87.5\n",
      "Epoch 14: Val Loss=0.0203, Val Acc=100.0\n",
      "370 [14] Loss: 0.1588 Acc: 96.9\n",
      "Epoch 14: Val Loss=0.0198, Val Acc=100.0\n",
      "380 [15] Loss: 0.2314 Acc: 93.8\n",
      "Epoch 15: Val Loss=0.0196, Val Acc=100.0\n",
      "390 [15] Loss: 0.2004 Acc: 90.6\n",
      "Epoch 15: Val Loss=0.0181, Val Acc=100.0\n",
      "400 [15] Loss: 0.0941 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0173, Val Acc=100.0\n",
      "410 [16] Loss: 0.2208 Acc: 93.8\n",
      "Epoch 16: Val Loss=0.0161, Val Acc=100.0\n",
      "420 [16] Loss: 0.2620 Acc: 90.6\n",
      "Epoch 16: Val Loss=0.0174, Val Acc=100.0\n",
      "430 [17] Loss: 0.3476 Acc: 87.5\n",
      "Epoch 17: Val Loss=0.0169, Val Acc=100.0\n",
      "440 [17] Loss: 0.0483 Acc: 100.0\n",
      "Epoch 17: Val Loss=0.0139, Val Acc=100.0\n",
      "450 [17] Loss: 0.3220 Acc: 85.0\n",
      "Epoch 17: Val Loss=0.0133, Val Acc=100.0\n",
      "460 [18] Loss: 0.3467 Acc: 90.6\n",
      "Epoch 18: Val Loss=0.0150, Val Acc=100.0\n",
      "470 [18] Loss: 0.1757 Acc: 87.5\n",
      "Epoch 18: Val Loss=0.0146, Val Acc=100.0\n",
      "480 [19] Loss: 0.1580 Acc: 93.8\n",
      "Epoch 19: Val Loss=0.0140, Val Acc=100.0\n",
      "490 [19] Loss: 0.3017 Acc: 90.6\n",
      "Epoch 19: Val Loss=0.0141, Val Acc=100.0\n",
      "Learning rate reduced: 0.0001 -> 1e-05\n",
      "500 [19] Loss: 0.1068 Acc: 95.0\n",
      "Epoch 19: Val Loss=0.0131, Val Acc=100.0\n",
      "510 [20] Loss: 0.1889 Acc: 90.6\n",
      "Epoch 20: Val Loss=0.0119, Val Acc=100.0\n",
      "520 [20] Loss: 0.0956 Acc: 96.9\n",
      "Epoch 20: Val Loss=0.0123, Val Acc=100.0\n",
      "530 [21] Loss: 0.1263 Acc: 93.8\n",
      "Epoch 21: Val Loss=0.0122, Val Acc=100.0\n",
      "540 [21] Loss: 0.2149 Acc: 96.9\n",
      "Epoch 21: Val Loss=0.0119, Val Acc=100.0\n",
      "550 [21] Loss: 0.1209 Acc: 100.0\n",
      "Epoch 21: Val Loss=0.0120, Val Acc=100.0\n",
      "560 [22] Loss: 0.1233 Acc: 96.9\n",
      "Epoch 22: Val Loss=0.0120, Val Acc=100.0\n",
      "570 [22] Loss: 0.2578 Acc: 90.6\n",
      "Epoch 22: Val Loss=0.0123, Val Acc=100.0\n",
      "Learning rate reduced: 1e-05 -> 1.0000000000000002e-06\n",
      "580 [23] Loss: 0.2716 Acc: 87.5\n",
      "Epoch 23: Val Loss=0.0119, Val Acc=100.0\n",
      "590 [23] Loss: 0.3095 Acc: 84.4\n",
      "Epoch 23: Val Loss=0.0121, Val Acc=100.0\n",
      "600 [23] Loss: 0.1519 Acc: 95.0\n",
      "Epoch 23: Val Loss=0.0123, Val Acc=100.0\n",
      "610 [24] Loss: 0.0626 Acc: 100.0\n",
      "Epoch 24: Val Loss=0.0118, Val Acc=100.0\n",
      "620 [24] Loss: 0.2279 Acc: 90.6\n",
      "Epoch 24: Val Loss=0.0122, Val Acc=100.0\n",
      "630 [25] Loss: 0.3460 Acc: 84.4\n",
      "Epoch 25: Val Loss=0.0123, Val Acc=100.0\n",
      "640 [25] Loss: 0.1327 Acc: 96.9\n",
      "Epoch 25: Val Loss=0.0123, Val Acc=100.0\n",
      "650 [25] Loss: 0.2524 Acc: 90.0\n",
      "Epoch 25: Val Loss=0.0116, Val Acc=100.0\n",
      "660 [26] Loss: 0.2458 Acc: 84.4\n",
      "Epoch 26: Val Loss=0.0116, Val Acc=100.0\n",
      "670 [26] Loss: 0.1452 Acc: 96.9\n",
      "Epoch 26: Val Loss=0.0120, Val Acc=100.0\n",
      "680 [27] Loss: 0.0248 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0121, Val Acc=100.0\n",
      "690 [27] Loss: 0.1970 Acc: 96.9\n",
      "Epoch 27: Val Loss=0.0121, Val Acc=100.0\n",
      "700 [27] Loss: 0.4352 Acc: 85.0\n",
      "Epoch 27: Val Loss=0.0119, Val Acc=100.0\n",
      "710 [28] Loss: 0.3780 Acc: 90.6\n",
      "Epoch 28: Val Loss=0.0116, Val Acc=100.0\n",
      "720 [28] Loss: 0.2830 Acc: 87.5\n",
      "Epoch 28: Val Loss=0.0125, Val Acc=100.0\n",
      "730 [29] Loss: 0.1230 Acc: 96.9\n",
      "Epoch 29: Val Loss=0.0125, Val Acc=100.0\n",
      "740 [29] Loss: 0.3143 Acc: 84.4\n",
      "Epoch 29: Val Loss=0.0125, Val Acc=100.0\n",
      "750 [29] Loss: 0.0542 Acc: 100.0\n",
      "Epoch 29: Val Loss=0.0119, Val Acc=100.0\n",
      "760 [30] Loss: 0.1717 Acc: 93.8\n",
      "Epoch 30: Val Loss=0.0118, Val Acc=100.0\n",
      "770 [30] Loss: 0.2745 Acc: 93.8\n",
      "Epoch 30: Val Loss=0.0120, Val Acc=100.0\n",
      "780 [31] Loss: 0.1279 Acc: 96.9\n",
      "Epoch 31: Val Loss=0.0121, Val Acc=100.0\n",
      "790 [31] Loss: 0.2957 Acc: 87.5\n",
      "Epoch 31: Val Loss=0.0118, Val Acc=100.0\n",
      "800 [31] Loss: 0.0902 Acc: 95.0\n",
      "Epoch 31: Val Loss=0.0125, Val Acc=100.0\n",
      "810 [32] Loss: 0.1621 Acc: 96.9\n",
      "Epoch 32: Val Loss=0.0121, Val Acc=100.0\n",
      "820 [32] Loss: 0.2399 Acc: 87.5\n",
      "Epoch 32: Val Loss=0.0118, Val Acc=100.0\n",
      "830 [33] Loss: 0.1504 Acc: 93.8\n",
      "Epoch 33: Val Loss=0.0123, Val Acc=100.0\n",
      "840 [33] Loss: 0.1104 Acc: 96.9\n",
      "Epoch 33: Val Loss=0.0122, Val Acc=100.0\n",
      "850 [33] Loss: 0.2619 Acc: 90.0\n",
      "Epoch 33: Val Loss=0.0120, Val Acc=100.0\n",
      "Early stopping at epoch 34, step 858\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 85628 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnn_model_gray = torch.nn.Sequential(\n",
    "    torch.nn.Conv2d(1, 16, kernel_size=5, padding=2),\n",
    "    torch.nn.BatchNorm2d(16),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 100 x 100\n",
    "    torch.nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "    torch.nn.BatchNorm2d(32),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 50 x 50\n",
    "    torch.nn.Conv2d(32, 64, kernel_size=3, padding=1),\n",
    "    torch.nn.BatchNorm2d(64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.MaxPool2d(2),      # here dimension reduced to 25 x 25\n",
    "    torch.nn.Flatten(),\n",
    "    torch.nn.Linear(64 * 25 * 25, 256),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Dropout(0.5),\n",
    "    torch.nn.Linear(256, len(train_loader_gray.dataset.classes))\n",
    ")\n",
    "cnn_model_gray.to(DEVICE)\n",
    "cnn_optimizer_gray = torch.optim.Adam(cnn_model_gray.parameters(), lr=LR)\n",
    "cnn_scheduler_gray = torch.optim.lr_scheduler.ReduceLROnPlateau(cnn_optimizer_gray, mode='min', factor=0.1, patience=50, min_lr=1e-6)\n",
    "cnn_loss_fun_gray = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "cnn_steps_gray, cnn_accuracies_gray, cnn_losses_gray, cnn_val_accs_gray, cnn_val_losses_gray = train(\n",
    "    cnn_model_gray,\n",
    "    cnn_optimizer_gray,\n",
    "    cnn_loss_fun_gray,\n",
    "    train_loader_gray,\n",
    "    val_loader_gray,\n",
    "    epochs=50,\n",
    "    verbose=2,\n",
    "    plotting=True,\n",
    "    patience=200,\n",
    "    scheduler=cnn_scheduler_gray\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "71c1e134",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 100.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63588 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_conf_mat(test_conf_mat(cnn_model_gray, test_loader_gray, nclasses=len(train_loader_gray.dataset.classes), verbose=True), train_loader_gray.dataset.classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "4ec9cbca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 1191044 characters]",
      "text/plain": [
       "<Figure size 2141.97x735.449 with 13 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_rgb  = next(iter(test_loader))[0][0]\n",
    "sample_gray = next(iter(test_loader_gray))[0][0]\n",
    "\n",
    "cnn_model.eval()\n",
    "cnn_model_gray.eval()\n",
    "\n",
    "# get the activations of the two models and put on cpu\n",
    "with torch.no_grad():\n",
    "    act_rgb  = cnn_model[0](sample_rgb.unsqueeze(0).to(DEVICE))[0].cpu()    # [16, H, W]\n",
    "    act_gray = cnn_model_gray[0](sample_gray.unsqueeze(0).to(DEVICE))[0].cpu()\n",
    "\n",
    "n_maps = 6\n",
    "vmin = min(act_rgb[:n_maps].min(), act_gray[:n_maps].min()).item()\n",
    "vmax = max(act_rgb[:n_maps].max(), act_gray[:n_maps].max()).item()\n",
    "\n",
    "fig, axs = plt.subplots(2, n_maps, figsize=(fig_width*1.8, fig_height), dpi=200, constrained_layout=True)\n",
    "for i in range(n_maps):\n",
    "    for row, act in enumerate([act_rgb, act_gray]):\n",
    "        axs[row, i].imshow(act[i], cmap='plasma', vmin=vmin, vmax=vmax)\n",
    "        axs[row, i].set_xticks([])\n",
    "        axs[row, i].set_yticks([])\n",
    "    axs[0, i].set_title(f'FM {i+1}', fontsize=13)\n",
    "\n",
    "axs[0, 0].set_ylabel('RGB', fontsize=16, fontweight='bold')\n",
    "axs[1, 0].set_ylabel('Grayscale', fontsize=16, fontweight='bold')\n",
    "\n",
    "cbar = fig.colorbar(axs[0, 0].images[0], ax=axs, orientation='vertical', fraction=0.025, pad=0.02)\n",
    "cbar.set_label('Activation', fontsize=12)\n",
    "fig.suptitle(\"First 6 Feature Maps (Conv1) — RGB vs Grayscale CNN\", fontsize=18)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lab",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
