{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a9cea997",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torchvision\n",
    "from skimage.feature import hog\n",
    "\n",
    "import os\n",
    "import random\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "plt.rcParams['font.family'] = 'Charter'\n",
    "plt.rcParams['text.latex.preamble'] = r'\\usepackage[charter]{mathdesign}'\n",
    "\n",
    "plt.rcParams['font.size'] = 12\n",
    "plt.rcParams['axes.labelsize'] = 12\n",
    "plt.rcParams['axes.titlesize'] = 12\n",
    "plt.rcParams['legend.fontsize'] = 12\n",
    "plt.rcParams['xtick.labelsize'] = 10\n",
    "plt.rcParams['ytick.labelsize'] = 10\n",
    "\n",
    "golden = (1 + 5 ** 0.5) / 2\n",
    "pt_to_inch = 1 / 72.27\n",
    "textwidth = 430 * pt_to_inch\n",
    "fig_width = textwidth\n",
    "fig_height = fig_width / golden\n",
    "\n",
    "from cycler import cycler  # Optional for custom cycling\n",
    "default_colors = plt.rcParams['axes.prop_cycle'].by_key()['color']\n",
    "blue, orange, green, red, purple, brown, pink, gray, yellow, cyan = default_colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "44e8e38d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 788\n",
      "Validation size: 100\n",
      "Test size: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['warning-slippery-road', 'warning-slippery-road', 'misc-bus-stop', 'mandatory-keep-left', 'prohibitory-no-trucks', 'prohibitory-no-transit', 'warning-pedestrian-crossing', 'prohibitory-no-overtaking', 'misc-parking', 'mandatory-go-straight', 'misc-gas-station', 'mandatory-cycle-path', 'prohibitory-no-stopping', 'mandatory-roundabout', 'prohibitory-no-transit', 'misc-bus-stop', 'prohibitory-no-trucks', 'misc-parking', 'misc-pedestrian-crossing', 'warning-curve', 'misc-parking', 'misc-gas-station', 'warning-pedestrian-crossing', 'warning-steep-ascent', 'mandatory-keep-left', 'prohibitory-no-transit', 'prohibitory-no-stopping', 'prohibitory-no-stopping', 'misc-gas-station', 'prohibitory-no-stopping', 'prohibitory-no-overtaking', 'mandatory-go-straight']\n",
      "torch.Size([32, 3, 200, 200])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 934100 characters]",
      "text/plain": [
       "<Figure size 1280x960 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def create_loaders(train_transform, test_transform, folder=\"./road-signs\"):\n",
    "    \"\"\"\n",
    "    Utility function to create loaders from transforms.\n",
    "    Used mainly to speed up creation of custom loaders (in this project grayscale).\n",
    "    The same transform is applied to validation and test set.\n",
    "    \"\"\"\n",
    "\n",
    "    # From the docs is_valid_file is normally used to check for corrupt files, here a cutom version is used to quickly split train and test\n",
    "    # this is done because of the nested folder structure:\n",
    "    # Graphical representation of the folder structure:\n",
    "    # road-signs/\n",
    "    # ├── <sign name>/\n",
    "    # │   ├── train/\n",
    "    # │   └── test/\n",
    "\n",
    "    # on top of that, the same is_valid_file mechanics is leveraged to select the validation set (5 samples per class from train folder)\n",
    "    val_paths = set()\n",
    "\n",
    "    for class_folder in os.listdir(folder):\n",
    "        train_folder = os.path.join(folder, class_folder, \"train\")\n",
    "        if os.path.isdir(train_folder):\n",
    "            train_files = os.listdir(train_folder)\n",
    "            val_samples = random.sample(train_files, 5)\n",
    "            for sample in val_samples:\n",
    "                val_paths.add(os.path.normpath(os.path.join(train_folder, sample)))\n",
    "\n",
    "\n",
    "    is_train = lambda p: os.path.normpath(p).split(os.sep)[-2] == 'train' and os.path.normpath(p) not in val_paths\n",
    "    is_validation = lambda p: os.path.normpath(p) in val_paths\n",
    "    is_test  = lambda p: os.path.normpath(p).split(os.sep)[-2] == 'test'\n",
    "\n",
    "    train_set = torchvision.datasets.ImageFolder(folder, transform=train_transform, is_valid_file=is_train)\n",
    "    train_loader = torch.utils.data.DataLoader(train_set, batch_size=32, shuffle=True, drop_last=False, num_workers=2)\n",
    "\n",
    "    # no augmentation for validation and test set\n",
    "    val_set = torchvision.datasets.ImageFolder(folder, transform=test_transform, is_valid_file=is_validation)\n",
    "    val_loader = torch.utils.data.DataLoader(val_set, batch_size=32, shuffle=False, drop_last=False, num_workers=2)\n",
    "    test_set = torchvision.datasets.ImageFolder(folder, transform=test_transform, is_valid_file=is_test)\n",
    "    test_loader = torch.utils.data.DataLoader(test_set, batch_size=32, shuffle=False, drop_last=False, num_workers=2)\n",
    "\n",
    "    print(f\"Train size: {len(train_set)}\")\n",
    "    print(f\"Validation size: {len(val_set)}\")\n",
    "    print(f\"Test size: {len(test_set)}\")\n",
    "\n",
    "    return train_loader, val_loader, test_loader\n",
    "\n",
    "\n",
    "transform = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.RandomRotation(10),\n",
    "    torchvision.transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.05),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "train_loader, val_loader, test_loader = create_loaders(transform, torchvision.transforms.ToTensor())\n",
    "num_classes = len(train_loader.dataset.classes)\n",
    "\n",
    "images, labels = next(iter(train_loader))\n",
    "plt.figure(dpi=200)\n",
    "grid = torchvision.utils.make_grid(images, nrow=8, normalize=True).permute(1, 2, 0)\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(grid)\n",
    "print([train_loader.dataset.classes[i] for i in labels])\n",
    "print(images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "592f05e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train size: 788\n",
      "Validation size: 100\n",
      "Test size: 200\n",
      "['prohibitory-no-overtaking', 'mandatory-chains', 'warning-series', 'warning-slippery-road', 'warning-steep-ascent', 'prohibitory-no-trucks', 'misc-pedestrian-crossing', 'prohibitory-no-transit', 'warning-pedestrian-crossing', 'mandatory-keep-left', 'prohibitory-no-transit', 'warning-steep-ascent', 'mandatory-roundabout', 'warning-curve', 'warning-steep-ascent', 'mandatory-cycle-path', 'mandatory-cycle-path', 'prohibitory-no-stopping', 'misc-gas-station', 'prohibitory-no-transit', 'misc-parking', 'prohibitory-no-stopping', 'misc-parking', 'warning-steep-ascent', 'prohibitory-no-stopping', 'mandatory-cycle-path', 'misc-gas-station', 'warning-series', 'misc-pedestrian-crossing', 'warning-steep-ascent', 'misc-dead-end', 'warning-series']\n",
      "torch.Size([32, 1, 200, 200])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 510488 characters]",
      "text/plain": [
       "<Figure size 1280x960 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# here let's create the grayscale dataset, same as before\n",
    "transform_gray = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.RandomRotation(10),\n",
    "    torchvision.transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.05),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    torchvision.transforms.Grayscale(num_output_channels=1)\n",
    "])\n",
    "\n",
    "# no augmentation for test set\n",
    "transform_gray_test = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.ToTensor(),\n",
    "    torchvision.transforms.Grayscale(num_output_channels=1)\n",
    "])\n",
    "\n",
    "train_loader_gray, val_loader_gray, test_loader_gray = create_loaders(transform_gray, transform_gray_test)\n",
    "\n",
    "images, labels = next(iter(train_loader_gray))\n",
    "plt.figure(dpi=200)\n",
    "grid = torchvision.utils.make_grid(images, nrow=8, normalize=True).permute(1, 2, 0)\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(grid)\n",
    "print([train_loader_gray.dataset.classes[i] for i in labels])\n",
    "print(images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "53deabdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Color histogram shape: torch.Size([7, 192])\n",
      "Edge direction histogram shape: torch.Size([7, 64])\n",
      "RGB Co-occurrence shape: torch.Size([7, 729])\n",
      "HOG shape: torch.Size([7, 900])\n"
     ]
    }
   ],
   "source": [
    "def color_histogram(img, bins=64):\n",
    "    n = img.shape[0]\n",
    "    c = img.shape[1]  # num channels, useful to compute both grayscales c=1 and rgb c=3\n",
    "    img = (img * bins).long().clamp(0, bins - 1)\n",
    "    img = img + torch.arange(0, c * n * bins, bins, device=img.device).view(n, c, 1, 1)\n",
    "    hist = torch.bincount(img.view(-1), minlength=(c * n * bins)).view(n, c * bins)\n",
    "    return hist / img[0, 0].numel()\n",
    "\n",
    "\n",
    "def edge_direction_histogram(img, bins=64):\n",
    "    n = img.shape[0]\n",
    "    SOBEL = torch.tensor([\n",
    "        [[[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]],\n",
    "        [[[-1, -2, -1], [0, 0, 0], [1, 2, 1]]]\n",
    "    ], dtype=img.dtype, device=img.device)\n",
    "    gray = img.mean(1, True)\n",
    "    grad = torch.nn.functional.conv2d(gray, SOBEL)\n",
    "    magnitude = grad.norm(dim=1)\n",
    "    angle = torch.atan2(grad[:, 0], grad[:, 1])\n",
    "    directions = (bins * angle / (2 * torch.pi)).long() % bins\n",
    "    directions = directions + torch.arange(0, n * bins, bins, device=img.device).view(n, 1, 1)\n",
    "    hist = torch.bincount(directions.view(-1), weights=magnitude.view(-1), minlength=n * bins).view(n, bins)\n",
    "    hist = hist / hist.sum(1, True).clamp(1e-6)\n",
    "    return hist\n",
    "\n",
    "\n",
    "def _cooccurrence_matrix_dir(values, bins, di, dj):\n",
    "    b, m, n = values.shape\n",
    "    codes = values[:, :m - di, :n - dj] + bins * values[:, di:, dj:]\n",
    "    codes += torch.arange(0, b * (bins ** 2), bins ** 2, device=values.device).view(b, 1, 1)\n",
    "    entries = torch.bincount(codes.view(-1), minlength=b * (bins ** 2)).view(b, bins, bins)\n",
    "    return entries\n",
    "\n",
    "\n",
    "def rgb_cooccurrence_matrix(img, quantization=3, distance=10):\n",
    "    n = img.shape[0]\n",
    "    c = img.shape[1]  # num channels, useful to compute both grayscales c=1 and rgb c=3, same as color histogram\n",
    "    img = (img * quantization).long().clamp(0, quantization - 1)\n",
    "    img = (img * (quantization ** torch.arange(c, device=img.device)).view(1, c, 1, 1)).sum(1)\n",
    "    bins = quantization ** c           # the num of features is actually the square of this cause i do 2 mats\n",
    "    mat = _cooccurrence_matrix_dir(img, bins, distance, 0)\n",
    "    mat += _cooccurrence_matrix_dir(img, bins, 0, distance)\n",
    "    mat = (mat + mat.permute(0, 2, 1)).view(n, -1)\n",
    "    mat = mat / mat.sum(1, True)\n",
    "    return mat\n",
    "\n",
    "\n",
    "def histogram_oriented_gradients(img, orientations=9, pixels_per_cell=(32, 32)):\n",
    "    # orientations: Number of orientation bins\n",
    "    # pixels_per_cell: Tuple of (cell_height, cell_width)\n",
    "    \n",
    "    n = img.shape[0]\n",
    "    features = []\n",
    "    \n",
    "    for i in range(n):\n",
    "        # Convert to grayscale numpy array [H, W] and move to CPU (scikin on cpu only)\n",
    "        gray = img[i].mean(0).cpu().numpy()\n",
    "        \n",
    "        hog_feat = hog(\n",
    "            gray,\n",
    "            orientations=orientations,\n",
    "            pixels_per_cell=pixels_per_cell,\n",
    "            cells_per_block=(2, 2),\n",
    "            feature_vector=True,\n",
    "            transform_sqrt=True       # power law compression\n",
    "        )\n",
    "        \n",
    "        features.append(torch.tensor(hog_feat, dtype=img.dtype, device=img.device))\n",
    "    \n",
    "    return torch.stack(features)\n",
    "\n",
    "\n",
    "# test on random tensors to se shapes\n",
    "x = torch.rand(7, 3, 200, 200)\n",
    "ch = color_histogram(x.clone())\n",
    "print(f\"Color histogram shape: {ch.shape}\")\n",
    "edh = edge_direction_histogram(x.clone())\n",
    "print(f\"Edge direction histogram shape: {edh.shape}\")\n",
    "h = rgb_cooccurrence_matrix(x.clone())\n",
    "print(f\"RGB Co-occurrence shape: {h.shape}\")\n",
    "hog_tensor = histogram_oriented_gradients(x.clone())\n",
    "print(f\"HOG shape: {hog_tensor.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "b23ffdf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "def standardize(X):\n",
    "    \"\"\"\n",
    "    Standardize the input tensor X (zero mean, unit variance).\n",
    "    \"\"\"\n",
    "    mean = X.mean(dim=0, keepdim=True)\n",
    "    std = X.std(dim=0, keepdim=True).clamp(min=1e-6)\n",
    "    X_standardized = (X - mean) / std\n",
    "    return X_standardized, mean, std\n",
    "\n",
    "def extract_features(loader, features_to_use=[\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]):\n",
    "    \"\"\"\n",
    "    Utility function to extract selected features from the given image loader.\n",
    "    Returns a tensor of features and corresponding labels.\n",
    "    \"\"\"\n",
    "    all_features = []\n",
    "    all_labels = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for images, labels in loader:\n",
    "            images = images.to(DEVICE)\n",
    "            \n",
    "            color = color_histogram(images) if \"color_histo\" in features_to_use else None\n",
    "            edge = edge_direction_histogram(images) if \"edge_dir_histo\" in features_to_use else None\n",
    "            cooccur = rgb_cooccurrence_matrix(images) if \"rgb_cooc_mat\" in features_to_use else None\n",
    "            hog_feat = histogram_oriented_gradients(images) if \"hog\" in features_to_use else None\n",
    "            \n",
    "            features = torch.cat([f for f in [color, edge, cooccur, hog_feat] if f is not None], dim=1)\n",
    "            all_features.append(features.cpu())\n",
    "            all_labels.append(labels)\n",
    "    \n",
    "    return torch.cat(all_features), torch.cat(all_labels)\n",
    "\n",
    "\n",
    "def extract_and_standardize(train_img_loader, val_img_loader, test_img_loader, features_to_use=[\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]):\n",
    "    \"\"\"\n",
    "    From the given image loader, extract features and standardize them.\n",
    "    Return a new loader with standardized selected features.\n",
    "    \"\"\"\n",
    "    \n",
    "    train_features, train_labels = extract_features(train_img_loader, features_to_use)\n",
    "    val_features, val_labels = extract_features(val_img_loader, features_to_use)\n",
    "    test_features, test_labels = extract_features(test_img_loader, features_to_use)\n",
    "\n",
    "    # Standard scaling, good for SVM/Logistic Regression/MLP\n",
    "    train_features_norm, mean, std = standardize(train_features)\n",
    "    val_features_norm = (val_features - mean) / std\n",
    "    test_features_norm = (test_features - mean) / std\n",
    "\n",
    "    train_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(train_features_norm, train_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(val_features_norm, val_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_features = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(test_features_norm, test_labels), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_features, val_loader_features, test_loader_features, train_features_norm.shape[1]  # also return num features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9c4107b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, loader, loss_fun):\n",
    "    model.eval()\n",
    "    total_loss = 0.0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for imgs, labels in loader:\n",
    "            imgs = imgs.to(DEVICE)\n",
    "            labels = labels.to(DEVICE)\n",
    "            logits = model(imgs)\n",
    "            loss = loss_fun(logits, labels)\n",
    "            total_loss += loss.item() * imgs.size(0)\n",
    "            preds = logits.argmax(dim=1)\n",
    "            correct += (preds == labels).sum().item()\n",
    "            total += labels.size(0)\n",
    "    avg_loss = total_loss / total\n",
    "    avg_acc = correct / total\n",
    "    model.train()\n",
    "    return avg_acc, avg_loss\n",
    "\n",
    "def train(model, optimizer, loss_fun, train_loader, val_loader, epochs=20, verbose=0, plotting=False, patience=10):\n",
    "    if verbose > 2:\n",
    "        verbose = 2  # cap verbosity to 2\n",
    "    \n",
    "    steps = []\n",
    "    accuracies = []\n",
    "    losses = []\n",
    "    val_accs = []\n",
    "    val_losses = []\n",
    "\n",
    "    step = 0\n",
    "    model.train()\n",
    "    best_val_loss = float('inf')\n",
    "    patience_counter = 0\n",
    "    early_stop = False\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        # early stopping check\n",
    "        if early_stop:\n",
    "            break\n",
    "        # note that here, if the loader passes low-level features, imgs variable contains images expressed as low-level features!!!\n",
    "        for imgs, labels in train_loader:\n",
    "            imgs = imgs.to(DEVICE)\n",
    "            labels = labels.to(DEVICE)\n",
    "            optimizer.zero_grad()\n",
    "            logits = model(imgs)\n",
    "            loss = loss_fun(logits, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            step += 1\n",
    "            predictions = logits.argmax(dim=1)\n",
    "            accuracy = (predictions == labels).float().mean()\n",
    "            steps.append(step)\n",
    "            accuracies.append(accuracy.item())\n",
    "            losses.append(loss.item())\n",
    "            val_acc, val_loss = evaluate(model, val_loader, loss_fun)\n",
    "            val_accs.append(val_acc)\n",
    "            val_losses.append(val_loss)\n",
    "\n",
    "            if verbose > 0 and step % (10 ** (3 - verbose)) == 0:\n",
    "                print(f\"{step} [{epoch}] Loss: {loss.item():.4f} Acc: {accuracy.item() * 100:.1f}\")\n",
    "                print(f\"Epoch {epoch}: Val Loss={val_loss:.4f}, Val Acc={val_acc * 100:.1f}\")\n",
    "            \n",
    "            if val_loss < best_val_loss:\n",
    "                best_val_loss = val_loss\n",
    "                patience_counter = 0\n",
    "            else:\n",
    "                patience_counter += 1\n",
    "                if patience_counter >= patience:\n",
    "                    print(f\"Early stopping at epoch {epoch}, step {step}\")\n",
    "                    early_stop = True\n",
    "                    break\n",
    "                \n",
    "    if plotting:\n",
    "        fig, ax1 = plt.subplots(figsize=(fig_width, fig_height))\n",
    "        ax1.set_xlabel(\"Iterations\")\n",
    "        ax1.set_ylabel(\"Accuracy\", color=blue)\n",
    "        ax1.plot(steps, accuracies, label=\"Accuracy\", color=blue, linewidth=2, alpha=0.7, linestyle='dashed')\n",
    "        ax1.plot(steps, val_accs, label=\"Val Accuracy\", color=blue, linewidth=2)\n",
    "        ax1.tick_params(axis='y', labelcolor=blue)\n",
    "        ax1.set_xlim(0, steps[-1])\n",
    "        ax1.set_ylim(0, 1.05)  # accuracy between 0 and 1\n",
    "        ax1.grid(linestyle='dotted')\n",
    "\n",
    "        ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\n",
    "\n",
    "        ax2.set_ylabel(\"Loss\", color=red)\n",
    "        ax2.plot(steps, losses, label=\"Loss\", color=red, linewidth=2, alpha=0.7, linestyle='dashed')\n",
    "        ax2.plot(steps, val_losses, label=\"Val Loss\", color=red, linewidth=2)\n",
    "        ax2.tick_params(axis='y', labelcolor=red)\n",
    "\n",
    "        fig.tight_layout()\n",
    "        plt.title(\"Accuracy and Loss over Iterations\")\n",
    "        plt.show()\n",
    "\n",
    "    return steps, accuracies, losses, val_accs, val_losses\n",
    "\n",
    "def test_conf_mat(model, test_loader, nclasses=20, verbose=True):\n",
    "    conf_mat = torch.zeros(nclasses, nclasses, dtype=torch.long)\n",
    "\n",
    "    for imgs, labels in test_loader:\n",
    "        imgs = imgs.to(DEVICE)\n",
    "        labels = labels.to(DEVICE)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            logits = model(imgs)\n",
    "        predictions = torch.argmax(logits, dim=1)\n",
    "\n",
    "        codes = labels * nclasses + predictions\n",
    "        hist = torch.bincount(codes.cpu(), minlength=nclasses ** 2)\n",
    "        conf_mat += hist.view(nclasses, nclasses)\n",
    "\n",
    "    accuracy = conf_mat.diag().sum() / conf_mat.sum()\n",
    "    if verbose:\n",
    "        print(f\"Accuracy: {100 * accuracy:.1f}%\")\n",
    "\n",
    "    # since the confusion matrix is evaluated for the accuracy, it can be safely returned\n",
    "    return conf_mat\n",
    "\n",
    "def plot_conf_mat(conf_mat, classes):\n",
    "    conf_mat_norm = conf_mat / conf_mat.sum(1)\n",
    "    plt.figure(dpi=100)\n",
    "    plt.imshow(conf_mat_norm, cmap=\"Blues\")\n",
    "    plt.xticks(range(len(classes)), classes, rotation=90)\n",
    "    plt.yticks(range(len(classes)), classes)\n",
    "    plt.colorbar()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "b8c0b8cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = 0.001\n",
    "\n",
    "class ModelWrapper():\n",
    "    \"\"\" \n",
    "    Wrapper class for logistic regression model using low-level features. \n",
    "    Useful to define directy a new instance when training with different features while retaining taining results.\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "            self,\n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR, \n",
    "            classes=None\n",
    "        ):\n",
    "        \"\"\"\n",
    "        In this constructor, if features_used is None, the given loaders are assumed to already contain low-level features.\n",
    "        Otherwise, features are extracted from the image loaders.\n",
    "        If the loaders are provided directly as low-level features, remember to provide the 'classes' argument explicitly as it cannot be inferred.\n",
    "        \"\"\"\n",
    "        if train_loader is not None and val_loader is not None and test_loader is not None:\n",
    "            if features_used is not None:\n",
    "                self.features_used = features_used\n",
    "                self.train_loader_features, self.val_loader_features, self.test_loader_features, self.num_features = extract_and_standardize(\n",
    "                    train_loader,\n",
    "                    val_loader,\n",
    "                    test_loader,\n",
    "                    features_to_use=features_used\n",
    "                )\n",
    "            else:\n",
    "                self.train_loader_features = train_loader\n",
    "                self.val_loader_features = val_loader\n",
    "                self.test_loader_features = test_loader\n",
    "                self.num_features = train_loader.dataset.tensors[0].shape[1]\n",
    "\n",
    "        self.lr = lr\n",
    "        if classes is not None:\n",
    "            self.classes = classes\n",
    "        elif features_used is not None:\n",
    "            self.classes = train_loader.dataset.classes\n",
    "        else:\n",
    "            raise ValueError(\"Cannot determine classes: please provide the 'classes' argument explicitly when features_used is None.\")\n",
    "        \n",
    "        self.model = None\n",
    "        self.optimizer = None\n",
    "        self.loss_fun = None\n",
    "        self.train_results = {\n",
    "            \"steps\": None,\n",
    "            \"accuracies\": None,\n",
    "            \"losses\": None,\n",
    "            \"val_accs\": None,\n",
    "            \"val_losses\": None\n",
    "        }\n",
    "\n",
    "    def build_model(self, model, optimizer, loss_fun):\n",
    "        if self.model is not None:\n",
    "            print(\"Warning: overwriting existing model, optimizer, and loss function.\")\n",
    "        self.model = model\n",
    "        self.model.to(DEVICE)\n",
    "        self.optimizer = optimizer\n",
    "        self.loss_fun = loss_fun\n",
    "    \n",
    "    def train(self, epochs=20, verbose=0, plotting=True, patience=10):\n",
    "        steps, accuracies, losses, val_accs, val_losses = train(\n",
    "            self.model,\n",
    "            self.optimizer,\n",
    "            self.loss_fun,\n",
    "            self.train_loader_features,\n",
    "            self.val_loader_features,\n",
    "            epochs=epochs,\n",
    "            verbose=verbose,\n",
    "            plotting=plotting,\n",
    "            patience=patience\n",
    "        )\n",
    "\n",
    "        self.train_results[\"steps\"] = steps\n",
    "        self.train_results[\"accuracies\"] = accuracies\n",
    "        self.train_results[\"losses\"] = losses\n",
    "        self.train_results[\"val_accs\"] = val_accs\n",
    "        self.train_results[\"val_losses\"] = val_losses\n",
    "    \n",
    "    def test(self):\n",
    "        conf_mat = test_conf_mat(self.model, self.test_loader_features, nclasses=len(self.classes), verbose=True)\n",
    "        plot_conf_mat(conf_mat, self.classes)\n",
    "\n",
    "\n",
    "class LogisticRegressionModelWrapper(ModelWrapper):\n",
    "    def __init__(\n",
    "            self, \n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR, \n",
    "            classes=None\n",
    "        ):\n",
    "\n",
    "        super().__init__(\n",
    "            features_used=features_used, \n",
    "            train_loader=train_loader, \n",
    "            val_loader=val_loader, \n",
    "            test_loader=test_loader, \n",
    "            lr=lr, \n",
    "            classes=classes\n",
    "        )\n",
    "\n",
    "        log_reg_model = torch.nn.Linear(self.num_features, len(self.classes))\n",
    "        log_reg_model.to(DEVICE)\n",
    "        log_reg_optimizer = torch.optim.Adam(log_reg_model.parameters(), lr=lr)\n",
    "        log_reg_loss_fun = torch.nn.CrossEntropyLoss()\n",
    "        self.build_model(log_reg_model, log_reg_optimizer, log_reg_loss_fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "213266fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo']\n",
      "100 [3] Loss: 2.2684 Acc: 40.0\n",
      "Epoch 3: Val Loss=2.2921, Val Acc=27.0\n",
      "200 [7] Loss: 1.6101 Acc: 50.0\n",
      "Epoch 7: Val Loss=2.1178, Val Acc=29.0\n",
      "300 [11] Loss: 1.3771 Acc: 65.0\n",
      "Epoch 11: Val Loss=2.0474, Val Acc=29.0\n",
      "Early stopping at epoch 14, step 371\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 125252 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo']\n",
      "100 [3] Loss: 1.9794 Acc: 40.0\n",
      "Epoch 3: Val Loss=1.9587, Val Acc=38.0\n",
      "200 [7] Loss: 1.6408 Acc: 55.0\n",
      "Epoch 7: Val Loss=1.6360, Val Acc=51.0\n",
      "300 [11] Loss: 1.3314 Acc: 75.0\n",
      "Epoch 11: Val Loss=1.4644, Val Acc=57.0\n",
      "400 [15] Loss: 1.4773 Acc: 55.0\n",
      "Epoch 15: Val Loss=1.3541, Val Acc=61.0\n",
      "500 [19] Loss: 1.2145 Acc: 80.0\n",
      "Epoch 19: Val Loss=1.2710, Val Acc=61.0\n",
      "600 [23] Loss: 1.1811 Acc: 65.0\n",
      "Epoch 23: Val Loss=1.2153, Val Acc=62.0\n",
      "700 [27] Loss: 1.1335 Acc: 80.0\n",
      "Epoch 27: Val Loss=1.1683, Val Acc=62.0\n",
      "800 [31] Loss: 1.3060 Acc: 65.0\n",
      "Epoch 31: Val Loss=1.1315, Val Acc=64.0\n",
      "Early stopping at epoch 35, step 893\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 124724 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['rgb_cooc_mat']\n",
      "100 [3] Loss: 1.9313 Acc: 40.0\n",
      "Epoch 3: Val Loss=2.1116, Val Acc=34.0\n",
      "200 [7] Loss: 1.1786 Acc: 65.0\n",
      "Epoch 7: Val Loss=1.9724, Val Acc=42.0\n",
      "300 [11] Loss: 1.4066 Acc: 65.0\n",
      "Epoch 11: Val Loss=1.9212, Val Acc=45.0\n",
      "Early stopping at epoch 15, step 397\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 134816 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['hog']\n",
      "100 [3] Loss: 0.1267 Acc: 95.0\n",
      "Epoch 3: Val Loss=0.0626, Val Acc=100.0\n",
      "200 [7] Loss: 0.0188 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0333, Val Acc=100.0\n",
      "300 [11] Loss: 0.0160 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0243, Val Acc=100.0\n",
      "400 [15] Loss: 0.0074 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0195, Val Acc=100.0\n",
      "500 [19] Loss: 0.0043 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0163, Val Acc=100.0\n",
      "600 [23] Loss: 0.0039 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0144, Val Acc=100.0\n",
      "700 [27] Loss: 0.0036 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0128, Val Acc=100.0\n",
      "800 [31] Loss: 0.0051 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0115, Val Acc=100.0\n",
      "900 [35] Loss: 0.0025 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0108, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0021 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0101, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0021 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0093, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0013 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0088, Val Acc=100.0\n",
      "Early stopping at epoch 48, step 1220\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 39660 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo']\n",
      "100 [3] Loss: 1.6493 Acc: 55.0\n",
      "Epoch 3: Val Loss=1.5038, Val Acc=58.0\n",
      "200 [7] Loss: 0.9741 Acc: 75.0\n",
      "Epoch 7: Val Loss=1.1874, Val Acc=66.0\n",
      "300 [11] Loss: 0.6844 Acc: 100.0\n",
      "Epoch 11: Val Loss=1.0544, Val Acc=67.0\n",
      "400 [15] Loss: 0.5982 Acc: 85.0\n",
      "Epoch 15: Val Loss=0.9830, Val Acc=68.0\n",
      "Early stopping at epoch 19, step 495\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 109544 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 1.4186 Acc: 65.0\n",
      "Epoch 3: Val Loss=1.9677, Val Acc=39.0\n",
      "200 [7] Loss: 1.2446 Acc: 60.0\n",
      "Epoch 7: Val Loss=1.8505, Val Acc=44.0\n",
      "Early stopping at epoch 10, step 251\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 115116 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'hog']\n",
      "100 [3] Loss: 0.0453 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0647, Val Acc=100.0\n",
      "200 [7] Loss: 0.0122 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0366, Val Acc=100.0\n",
      "300 [11] Loss: 0.0077 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0268, Val Acc=100.0\n",
      "400 [15] Loss: 0.0079 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0214, Val Acc=100.0\n",
      "500 [19] Loss: 0.0071 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0185, Val Acc=100.0\n",
      "600 [23] Loss: 0.0051 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0163, Val Acc=100.0\n",
      "700 [27] Loss: 0.0031 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0144, Val Acc=100.0\n",
      "800 [31] Loss: 0.0018 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0131, Val Acc=100.0\n",
      "900 [35] Loss: 0.0019 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0120, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0017 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0110, Val Acc=100.0\n",
      "Early stopping at epoch 40, step 1021\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 39704 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 1.0591 Acc: 75.0\n",
      "Epoch 3: Val Loss=1.4051, Val Acc=57.0\n",
      "200 [7] Loss: 0.8623 Acc: 85.0\n",
      "Epoch 7: Val Loss=1.2197, Val Acc=63.0\n",
      "Early stopping at epoch 11, step 288\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 108808 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'hog']\n",
      "100 [3] Loss: 0.0405 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0810, Val Acc=99.0\n",
      "200 [7] Loss: 0.0200 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0503, Val Acc=99.0\n",
      "300 [11] Loss: 0.0041 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0386, Val Acc=99.0\n",
      "Early stopping at epoch 12, step 312\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 53156 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0605 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.1259, Val Acc=98.0\n",
      "200 [7] Loss: 0.0120 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0941, Val Acc=99.0\n",
      "300 [11] Loss: 0.0152 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0843, Val Acc=98.0\n",
      "Early stopping at epoch 14, step 362\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 51920 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "100 [3] Loss: 0.7394 Acc: 85.0\n",
      "Epoch 3: Val Loss=1.2289, Val Acc=55.0\n",
      "200 [7] Loss: 0.7347 Acc: 85.0\n",
      "Epoch 7: Val Loss=1.0390, Val Acc=66.0\n",
      "300 [11] Loss: 0.3593 Acc: 95.0\n",
      "Epoch 11: Val Loss=0.9881, Val Acc=70.0\n",
      "Early stopping at epoch 12, step 319\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 105972 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'hog']\n",
      "100 [3] Loss: 0.0533 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0513, Val Acc=100.0\n",
      "200 [7] Loss: 0.0087 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0317, Val Acc=100.0\n",
      "300 [11] Loss: 0.0133 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0223, Val Acc=100.0\n",
      "400 [15] Loss: 0.0083 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0183, Val Acc=100.0\n",
      "500 [19] Loss: 0.0075 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.0157, Val Acc=100.0\n",
      "600 [23] Loss: 0.0022 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0135, Val Acc=100.0\n",
      "700 [27] Loss: 0.0025 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0123, Val Acc=100.0\n",
      "Early stopping at epoch 31, step 795\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 40500 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0286 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.1074, Val Acc=97.0\n",
      "200 [7] Loss: 0.0213 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0776, Val Acc=99.0\n",
      "Early stopping at epoch 10, step 265\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 54272 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0186 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0794, Val Acc=99.0\n",
      "200 [7] Loss: 0.0105 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0528, Val Acc=99.0\n",
      "300 [11] Loss: 0.0121 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0415, Val Acc=99.0\n",
      "400 [15] Loss: 0.0075 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0342, Val Acc=99.0\n",
      "Early stopping at epoch 16, step 424\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 49196 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "100 [3] Loss: 0.0312 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.0829, Val Acc=98.0\n",
      "Early stopping at epoch 6, step 155\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56592 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "features_list = [\n",
    "    [\"color_histo\"],\n",
    "    [\"edge_dir_histo\"],\n",
    "    [\"rgb_cooc_mat\"],\n",
    "    [\"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\"],\n",
    "    [\"color_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"color_histo\", \"hog\"],\n",
    "    [\"edge_dir_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"edge_dir_histo\", \"hog\"],\n",
    "    [\"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"hog\"],\n",
    "    [\"color_histo\", \"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"],\n",
    "    [\"color_histo\", \"edge_dir_histo\", \"rgb_cooc_mat\", \"hog\"]\n",
    "]\n",
    "\n",
    "models = {}\n",
    "for features_to_use in features_list:\n",
    "    print(f\"Training model with features: {features_to_use}\")\n",
    "    model_wrapper = LogisticRegressionModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader,\n",
    "        val_loader=val_loader,\n",
    "        test_loader=test_loader,\n",
    "        lr=LR\n",
    "    )\n",
    "    model_wrapper.train(epochs=50, verbose=1, plotting=True, patience=25)\n",
    "    models[\"+\".join(features_to_use)] = model_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "05b339f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 81244 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.patches import Patch\n",
    "\n",
    "def summary_plot(models, additional_title=\"\"):\n",
    "    \"\"\"\n",
    "    Utility to produce a bar plot that summarizes performance of the models trained with different low-level feature types.\n",
    "    \"\"\"\n",
    "    names = list(models.keys())\n",
    "    val_accs = [models[name].train_results[\"val_accs\"][-1] for name in names]\n",
    "    train_accs = [np.array(models[name].train_results[\"accuracies\"][-10:-1]).mean() for name in names]\n",
    "\n",
    "    # Sort by validation accuracy\n",
    "    sorted_data = sorted(zip(val_accs, train_accs, names))\n",
    "    val_accs_sorted, train_accs_sorted, names_sorted = zip(*sorted_data)\n",
    "\n",
    "    x = np.arange(len(names_sorted))\n",
    "    width = 0.5\n",
    "\n",
    "    # Define colors\n",
    "    color_hog = red\n",
    "    color_edge = blue\n",
    "    color_other = gray\n",
    "\n",
    "    bar_colors = []\n",
    "    for name in names_sorted:\n",
    "        if \"hog\" in name:\n",
    "            bar_colors.append(color_hog)\n",
    "        elif \"edge_dir_histo\" in name:\n",
    "            bar_colors.append(color_edge)\n",
    "        else:\n",
    "            bar_colors.append(color_other)\n",
    "\n",
    "    # Simplify names for x-axis\n",
    "    def simplify(name):\n",
    "        return \"+\".join([\n",
    "            \"C\" if \"color_histo\" in name else \"\",\n",
    "            \"E\" if \"edge_dir_histo\" in name else \"\",\n",
    "            \"R\" if \"rgb_cooc_mat\" in name else \"\",\n",
    "            \"H\" if \"hog\" in name else \"\"\n",
    "        ]).replace(\"++\", \"+\").strip(\"+\")\n",
    "\n",
    "    names_simple = [simplify(n) for n in names_sorted]\n",
    "\n",
    "    plt.figure(figsize=(fig_width, fig_height), dpi=300)\n",
    "\n",
    "    for i in range(len(names_sorted)):\n",
    "        plt.bar(x[i], val_accs_sorted[i], width=width, color=bar_colors[i])\n",
    "        plt.bar(x[i], train_accs_sorted[i], width=width, color=bar_colors[i], alpha=0.5)\n",
    "\n",
    "    plt.xticks(x, names_simple, rotation=90)\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.title(f\"Final Validation and Training Accuracy (sorted{'' if additional_title == '' else ' - ' + additional_title})\")\n",
    "    plt.grid(axis='y', linestyle='dotted')\n",
    "\n",
    "    # Custom legend\n",
    "    legend_elements = [\n",
    "        Patch(facecolor=color_hog, edgecolor='k', label='w/ HOG', alpha=1.0),\n",
    "        Patch(facecolor=color_edge, edgecolor='k', label='w/ Edge Dir', alpha=1.0),\n",
    "        Patch(facecolor=color_other, edgecolor='k', label='Other', alpha=1.0),\n",
    "    ]\n",
    "    plt.legend(handles=legend_elements, loc='upper left', fontsize=9)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "summary_plot(models, additional_title=\"RGB Images\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "8a58ca1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original features shape: torch.Size([788, 900])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 401])\n",
      "100 [3] Loss: 0.1574 Acc: 100.0\n",
      "Epoch 3: Val Loss=0.2247, Val Acc=98.0\n",
      "200 [7] Loss: 0.0643 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.0985, Val Acc=100.0\n",
      "300 [11] Loss: 0.0359 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.0675, Val Acc=100.0\n",
      "400 [15] Loss: 0.0287 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.0501, Val Acc=100.0\n",
      "Early stopping at epoch 16, step 407\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 64116 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def remove_low_variance_features(train_loader, val_loader, test_loader, threshold=0.01, features_to_use=[\"hog\"], standardize_feat=True, verbose=False):\n",
    "    \"\"\"\n",
    "    Utility function to remove low-variance features from the dataset.\n",
    "    It expects image loaders to extract features from.\n",
    "    Returns new loaders with reduced feature sets.\n",
    "    \"\"\"\n",
    "\n",
    "    train_features, labels_train_features = extract_features(train_loader, features_to_use)\n",
    "    val_features, labels_val_features = extract_features(val_loader, features_to_use)\n",
    "    test_features, labels_test_features = extract_features(test_loader, features_to_use)\n",
    "\n",
    "    variances = train_features.var(dim=0)\n",
    "    non_zero_variance_mask = variances > threshold\n",
    "    reduced_train_features = train_features[:, non_zero_variance_mask]\n",
    "    reduced_val_features = val_features[:, non_zero_variance_mask]\n",
    "    reduced_test_features = test_features[:, non_zero_variance_mask]\n",
    "    if standardize_feat:\n",
    "        reduced_train_features, mean, std = standardize(reduced_train_features)\n",
    "        reduced_val_features = (reduced_val_features - mean) / std\n",
    "        reduced_test_features = (reduced_test_features - mean) / std\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"Original features shape: {train_features.shape}\")\n",
    "        print(f\"Reduced features shape (removed zero-variance features): {reduced_train_features.shape}\")\n",
    "    \n",
    "    train_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_train_features, labels_train_features), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_val_features, labels_val_features), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_reduced = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(reduced_test_features, labels_test_features), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_reduced, val_loader_reduced, test_loader_reduced\n",
    "\n",
    "\n",
    "train_loader_hog_reduced, val_loader_hog_reduced, test_loader_hog_reduced = remove_low_variance_features(\n",
    "    train_loader,\n",
    "    val_loader,\n",
    "    test_loader,\n",
    "    threshold=0.01,\n",
    "    features_to_use=[\"hog\"],\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_reduced = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_reduced,\n",
    "    val_loader=val_loader_hog_reduced,\n",
    "    test_loader=test_loader_hog_reduced,\n",
    "    classes=train_loader.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "\n",
    "model_hog_reduced.train(epochs=50, verbose=1, plotting=True, patience=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "dacada29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA reduced features from 401 to 76 components to retain 95.00% variance.\n",
      "100 [3] Loss: 1.3563 Acc: 50.0\n",
      "Epoch 3: Val Loss=1.0610, Val Acc=74.0\n",
      "200 [7] Loss: 0.2957 Acc: 95.0\n",
      "Epoch 7: Val Loss=0.3820, Val Acc=98.0\n",
      "300 [11] Loss: 0.3330 Acc: 95.0\n",
      "Epoch 11: Val Loss=0.2142, Val Acc=100.0\n",
      "400 [15] Loss: 0.1562 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.1475, Val Acc=100.0\n",
      "500 [19] Loss: 0.0823 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.1138, Val Acc=100.0\n",
      "600 [23] Loss: 0.0521 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.0914, Val Acc=100.0\n",
      "700 [27] Loss: 0.0639 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0783, Val Acc=100.0\n",
      "800 [31] Loss: 0.0374 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0681, Val Acc=100.0\n",
      "900 [35] Loss: 0.0526 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0596, Val Acc=100.0\n",
      "1000 [39] Loss: 0.0328 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0533, Val Acc=100.0\n",
      "1100 [43] Loss: 0.0295 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0484, Val Acc=100.0\n",
      "1200 [47] Loss: 0.0286 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0445, Val Acc=100.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 61768 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def pca(train_loader, val_loader, test_loader, mincomponents=1, retvar=0.95, verbose=False):\n",
    "    \"\"\"\n",
    "    Helper function to perform PCA on the given datasets.\n",
    "    Expects loaders with low-level features.\n",
    "    Return new loaders with reduced feature sets.\n",
    "    \"\"\"\n",
    "    train = train_loader.dataset.tensors[0]\n",
    "    val = val_loader.dataset.tensors[0]\n",
    "    test = test_loader.dataset.tensors[0]\n",
    "\n",
    "    mu = train.mean(0)\n",
    "    sigma = torch.cov(train.T)\n",
    "    eigvals, eigvecs = torch.linalg.eigh(sigma)\n",
    "    order = eigvals.argsort(descending=True)\n",
    "    eigvals = eigvals[order]\n",
    "    r = eigvals.cumsum(0) / eigvals.sum()\n",
    "    k = 1 + (r >= retvar).nonzero()[0][0]\n",
    "    k = max(k, mincomponents)\n",
    "    w = eigvecs[:, order[:k]]\n",
    "    train_pca = (train - mu) @ w\n",
    "    val_pca = (val - mu) @ w\n",
    "    test_pca = (test - mu) @ w\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"PCA reduced features from {train.shape[1]} to {k} components to retain {retvar*100:.2f}% variance.\")\n",
    "\n",
    "    train_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(train_pca, train_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=True\n",
    "    )\n",
    "    val_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(val_pca, val_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "    test_loader_pca = torch.utils.data.DataLoader(\n",
    "        torch.utils.data.TensorDataset(test_pca, test_loader.dataset.tensors[1]), \n",
    "        batch_size=32, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    return train_loader_pca, val_loader_pca, test_loader_pca\n",
    "\n",
    "\n",
    "train_loader_hog_pca, val_loader_hog_pca, test_loader_hog_pca = pca(\n",
    "    train_loader_hog_reduced,\n",
    "    val_loader_hog_reduced,\n",
    "    test_loader_hog_reduced,\n",
    "    mincomponents=1,\n",
    "    retvar=0.95,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_pca = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_pca,\n",
    "    val_loader=val_loader_hog_pca,  \n",
    "    test_loader=test_loader_hog_pca,\n",
    "    classes=train_loader.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "model_hog_pca.train(epochs=50, verbose=1, plotting=True, patience=25)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "dd3c835f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 98.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63748 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_hog_pca.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "dd2bc848",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with features: ['color_histo']\n",
      "Early stopping at epoch 16, step 423\n",
      "Training model with features: ['edge_dir_histo']\n",
      "Training model with features: ['rgb_cooc_mat']\n",
      "Training model with features: ['hog']\n",
      "Early stopping at epoch 31, step 793\n",
      "Training model with features: ['color_histo', 'edge_dir_histo']\n",
      "Early stopping at epoch 29, step 741\n",
      "Training model with features: ['color_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 38, step 963\n",
      "Training model with features: ['color_histo', 'hog']\n",
      "Early stopping at epoch 8, step 214\n",
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 36, step 915\n",
      "Training model with features: ['edge_dir_histo', 'hog']\n",
      "Early stopping at epoch 13, step 342\n",
      "Training model with features: ['rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 39, step 991\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat']\n",
      "Early stopping at epoch 31, step 788\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'hog']\n",
      "Early stopping at epoch 12, step 314\n",
      "Training model with features: ['color_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 11, step 290\n",
      "Training model with features: ['edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 9, step 248\n",
      "Training model with features: ['color_histo', 'edge_dir_histo', 'rgb_cooc_mat', 'hog']\n",
      "Early stopping at epoch 42, step 1072\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 79260 characters]",
      "text/plain": [
       "<Figure size 1784.97x1103.17 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_gray = {}\n",
    "for features_to_use in features_list:\n",
    "    print(f\"Training model with features: {features_to_use}\")\n",
    "    model_wrapper = LogisticRegressionModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader_gray,\n",
    "        val_loader=val_loader_gray,\n",
    "        test_loader=test_loader_gray,\n",
    "        lr=LR\n",
    "    )\n",
    "    model_wrapper.train(epochs=50, verbose=0, plotting=False, patience=25)\n",
    "    models_gray[\"+\".join(features_to_use)] = model_wrapper\n",
    "\n",
    "summary_plot(models_gray, additional_title=\"Grayscale Images\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "ef49300c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original features shape: torch.Size([788, 900])\n",
      "Reduced features shape (removed zero-variance features): torch.Size([788, 389])\n",
      "PCA reduced features from 389 to 71 components to retain 95.00% variance.\n",
      "100 [3] Loss: 1.1268 Acc: 75.0\n",
      "Epoch 3: Val Loss=1.1602, Val Acc=75.0\n",
      "200 [7] Loss: 0.2800 Acc: 100.0\n",
      "Epoch 7: Val Loss=0.4484, Val Acc=95.0\n",
      "300 [11] Loss: 0.2709 Acc: 100.0\n",
      "Epoch 11: Val Loss=0.2425, Val Acc=98.0\n",
      "400 [15] Loss: 0.1593 Acc: 100.0\n",
      "Epoch 15: Val Loss=0.1637, Val Acc=100.0\n",
      "500 [19] Loss: 0.0637 Acc: 100.0\n",
      "Epoch 19: Val Loss=0.1258, Val Acc=100.0\n",
      "600 [23] Loss: 0.0778 Acc: 100.0\n",
      "Epoch 23: Val Loss=0.1029, Val Acc=100.0\n",
      "700 [27] Loss: 0.0698 Acc: 100.0\n",
      "Epoch 27: Val Loss=0.0884, Val Acc=100.0\n",
      "800 [31] Loss: 0.0495 Acc: 100.0\n",
      "Epoch 31: Val Loss=0.0781, Val Acc=99.0\n",
      "900 [35] Loss: 0.0387 Acc: 100.0\n",
      "Epoch 35: Val Loss=0.0706, Val Acc=99.0\n",
      "1000 [39] Loss: 0.0415 Acc: 100.0\n",
      "Epoch 39: Val Loss=0.0642, Val Acc=99.0\n",
      "1100 [43] Loss: 0.0290 Acc: 100.0\n",
      "Epoch 43: Val Loss=0.0597, Val Acc=99.0\n",
      "1200 [47] Loss: 0.0516 Acc: 100.0\n",
      "Epoch 47: Val Loss=0.0559, Val Acc=99.0\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 61672 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_loader_hog_reduced_gray, val_loader_hog_reduced_gray, test_loader_hog_reduced_gray = remove_low_variance_features(\n",
    "    train_loader_gray,\n",
    "    val_loader_gray,\n",
    "    test_loader_gray,\n",
    "    threshold=0.01,\n",
    "    features_to_use=[\"hog\"],\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "train_loader_hog_pca_gray, val_loader_hog_pca_gray, test_loader_hog_pca_gray = pca(\n",
    "    train_loader_hog_reduced_gray,\n",
    "    val_loader_hog_reduced_gray,\n",
    "    test_loader_hog_reduced_gray,\n",
    "    mincomponents=1,\n",
    "    retvar=0.95,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "model_hog_pca_gray = LogisticRegressionModelWrapper(\n",
    "    features_used=None,\n",
    "    train_loader=train_loader_hog_pca_gray,\n",
    "    val_loader=val_loader_hog_pca_gray,  \n",
    "    test_loader=test_loader_hog_pca_gray,\n",
    "    classes=train_loader_gray.dataset.classes,\n",
    "    lr=LR\n",
    ")\n",
    "model_hog_pca_gray.train(epochs=50, verbose=1, plotting=True, patience=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "0b49ae7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 99.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 63704 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_hog_pca_gray.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba424de6",
   "metadata": {},
   "source": [
    "# MLPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "fee28e59",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLPModelWrapper(ModelWrapper):\n",
    "    def __init__(\n",
    "            self, \n",
    "            features_used=None, \n",
    "            train_loader=None, \n",
    "            val_loader=None, \n",
    "            test_loader=None, \n",
    "            lr=LR, \n",
    "            classes=None,\n",
    "            hidden_layers=[],\n",
    "            apply_low_var_feat_reduction=False,\n",
    "            apply_pca=False\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Constructor for a MLP model using low-level features.\n",
    "        The hidden_layers argument expects a list defining the number of units in each hidden layer.\n",
    "        E.g., hidden_layers=[512, 256, 128]\n",
    "        The default is no hidden layers.\n",
    "        Pass ONLY image loaders.\n",
    "        \"\"\"\n",
    "        if train_loader is None or val_loader is None or test_loader is None or features_used is None:\n",
    "            raise ValueError(\"Please provide train_loader, val_loader, test_loader and features_used arguments.\")\n",
    "        feat_train_loader = None\n",
    "        feat_val_loader = None\n",
    "        feat_test_loader = None\n",
    "\n",
    "        if apply_low_var_feat_reduction:\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader = remove_low_variance_features(\n",
    "                train_loader,\n",
    "                val_loader,\n",
    "                test_loader,\n",
    "                threshold=0.01,\n",
    "                features_to_use=features_used,\n",
    "                verbose=True\n",
    "            )\n",
    "        else:\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader, _ = extract_and_standardize(\n",
    "                train_loader,\n",
    "                val_loader,\n",
    "                test_loader,\n",
    "                features_to_use=features_used\n",
    "            )\n",
    "\n",
    "        if apply_pca:\n",
    "            feat_train_loader, feat_val_loader, feat_test_loader = pca(\n",
    "                feat_train_loader,\n",
    "                feat_val_loader,\n",
    "                feat_test_loader,\n",
    "                mincomponents=1,\n",
    "                retvar=0.95,\n",
    "                verbose=True\n",
    "            )\n",
    "        \n",
    "        local_classes = train_loader.dataset.classes\n",
    "\n",
    "        # Passing classes explicitly since using pre-processed loaders\n",
    "        super().__init__(\n",
    "            features_used=None, \n",
    "            train_loader=feat_train_loader, \n",
    "            val_loader=feat_val_loader, \n",
    "            test_loader=feat_test_loader, \n",
    "            lr=lr, \n",
    "            classes=local_classes\n",
    "        )\n",
    "\n",
    "        mlp_model = torch.nn.Sequential()\n",
    "        if len(hidden_layers) == 0:\n",
    "            mlp_model.append(torch.nn.Linear(self.num_features, len(self.classes)))\n",
    "        else:\n",
    "            prev_dimension = self.num_features\n",
    "            for hidden_dimension in hidden_layers:\n",
    "                mlp_model.extend([\n",
    "                    torch.nn.Linear(prev_dimension, hidden_dimension),\n",
    "                    torch.nn.ReLU()\n",
    "                ])\n",
    "                prev_dimension = hidden_dimension\n",
    "            mlp_model.append(torch.nn.Linear(prev_dimension, len(self.classes)))\n",
    "            \n",
    "        mlp_model.to(DEVICE)\n",
    "        mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=lr)\n",
    "        mlp_loss_fun = torch.nn.CrossEntropyLoss()\n",
    "        self.build_model(mlp_model, mlp_optimizer, mlp_loss_fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "ec003ecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['color_histo']\n",
      "100 [3] Loss: 2.5520 Acc: 20.0\n",
      "Epoch 3: Val Loss=2.6694, Val Acc=14.0\n",
      "200 [7] Loss: 1.6968 Acc: 55.0\n",
      "Epoch 7: Val Loss=2.0456, Val Acc=26.0\n",
      "Early stopping at epoch 11, step 282\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 115224 characters]",
      "text/plain": [
       "<Figure size 594.991x367.725 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MLP model with features: ['edge_dir_histo']\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mKeyboardInterrupt\u001b[39m                         Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[193]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m      2\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m features_to_use \u001b[38;5;129;01min\u001b[39;00m features_list:\n\u001b[32m      3\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTraining MLP model with features: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfeatures_to_use\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m     mlp_model_wrapper = \u001b[43mMLPModelWrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m      5\u001b[39m \u001b[43m        \u001b[49m\u001b[43mfeatures_used\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfeatures_to_use\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m      6\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m      7\u001b[39m \u001b[43m        \u001b[49m\u001b[43mval_loader\u001b[49m\u001b[43m=\u001b[49m\u001b[43mval_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m      8\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtest_loader\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtest_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m      9\u001b[39m \u001b[43m        \u001b[49m\u001b[43mlr\u001b[49m\u001b[43m=\u001b[49m\u001b[43mLR\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m     10\u001b[39m \u001b[43m        \u001b[49m\u001b[43mhidden_layers\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m50\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m20\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m     11\u001b[39m \u001b[43m        \u001b[49m\u001b[43mapply_low_var_feat_reduction\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m     12\u001b[39m \u001b[43m        \u001b[49m\u001b[43mapply_pca\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[32m     13\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     14\u001b[39m     mlp_model_wrapper.train(epochs=\u001b[32m50\u001b[39m, verbose=\u001b[32m1\u001b[39m, plotting=\u001b[38;5;28;01mTrue\u001b[39;00m, patience=\u001b[32m25\u001b[39m)\n\u001b[32m     15\u001b[39m     models_mlp[\u001b[33m\"\u001b[39m\u001b[33m+\u001b[39m\u001b[33m\"\u001b[39m.join(features_to_use)] = mlp_model_wrapper\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[191]\u001b[39m\u001b[32m, line 37\u001b[39m, in \u001b[36mMLPModelWrapper.__init__\u001b[39m\u001b[34m(self, features_used, train_loader, val_loader, test_loader, lr, classes, hidden_layers, apply_low_var_feat_reduction, apply_pca)\u001b[39m\n\u001b[32m     28\u001b[39m     feat_train_loader, feat_val_loader, feat_test_loader = remove_low_variance_features(\n\u001b[32m     29\u001b[39m         train_loader,\n\u001b[32m     30\u001b[39m         val_loader,\n\u001b[32m   (...)\u001b[39m\u001b[32m     34\u001b[39m         verbose=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m     35\u001b[39m     )\n\u001b[32m     36\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m37\u001b[39m     feat_train_loader, feat_val_loader, feat_test_loader, _ = \u001b[43mextract_and_standardize\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m     38\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m     39\u001b[39m \u001b[43m        \u001b[49m\u001b[43mval_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m     40\u001b[39m \u001b[43m        \u001b[49m\u001b[43mtest_loader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m     41\u001b[39m \u001b[43m        \u001b[49m\u001b[43mfeatures_to_use\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfeatures_used\u001b[49m\n\u001b[32m     42\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     44\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m apply_pca:\n\u001b[32m     45\u001b[39m     feat_train_loader, feat_val_loader, feat_test_loader = pca(\n\u001b[32m     46\u001b[39m         feat_train_loader,\n\u001b[32m     47\u001b[39m         feat_val_loader,\n\u001b[32m   (...)\u001b[39m\u001b[32m     51\u001b[39m         verbose=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m     52\u001b[39m     )\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[113]\u001b[39m\u001b[32m, line 42\u001b[39m, in \u001b[36mextract_and_standardize\u001b[39m\u001b[34m(train_img_loader, val_img_loader, test_img_loader, features_to_use)\u001b[39m\n\u001b[32m     36\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mextract_and_standardize\u001b[39m(train_img_loader, val_img_loader, test_img_loader, features_to_use=[\u001b[33m\"\u001b[39m\u001b[33mcolor_histo\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33medge_dir_histo\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mrgb_cooc_mat\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mhog\u001b[39m\u001b[33m\"\u001b[39m]):\n\u001b[32m     37\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m     38\u001b[39m \u001b[33;03m    From the given image loader, extract features and standardize them.\u001b[39;00m\n\u001b[32m     39\u001b[39m \u001b[33;03m    Return a new loader with standardized selected features.\u001b[39;00m\n\u001b[32m     40\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m42\u001b[39m     train_features, train_labels = \u001b[43mextract_features\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_img_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfeatures_to_use\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     43\u001b[39m     val_features, val_labels = extract_features(val_img_loader, features_to_use)\n\u001b[32m     44\u001b[39m     test_features, test_labels = extract_features(test_img_loader, features_to_use)\n",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[113]\u001b[39m\u001b[32m, line 21\u001b[39m, in \u001b[36mextract_features\u001b[39m\u001b[34m(loader, features_to_use)\u001b[39m\n\u001b[32m     18\u001b[39m all_labels = []\n\u001b[32m     20\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.no_grad():\n\u001b[32m---> \u001b[39m\u001b[32m21\u001b[39m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mimages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mloader\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m     22\u001b[39m \u001b[43m        \u001b[49m\u001b[43mimages\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mimages\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mDEVICE\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     24\u001b[39m \u001b[43m        \u001b[49m\u001b[43mcolor\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolor_histogram\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimages\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mcolor_histo\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfeatures_to_use\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/lab/lib/python3.14/site-packages/torch/utils/data/dataloader.py:741\u001b[39m, in \u001b[36m_BaseDataLoaderIter.__next__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    738\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    739\u001b[39m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[32m    740\u001b[39m     \u001b[38;5;28mself\u001b[39m._reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m741\u001b[39m data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    742\u001b[39m \u001b[38;5;28mself\u001b[39m._num_yielded += \u001b[32m1\u001b[39m\n\u001b[32m    743\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[32m    744\u001b[39m     \u001b[38;5;28mself\u001b[39m._dataset_kind == _DatasetKind.Iterable\n\u001b[32m    745\u001b[39m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    746\u001b[39m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m._num_yielded > \u001b[38;5;28mself\u001b[39m._IterableDataset_len_called\n\u001b[32m    747\u001b[39m ):\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/lab/lib/python3.14/site-packages/torch/utils/data/dataloader.py:1524\u001b[39m, in \u001b[36m_MultiProcessingDataLoaderIter._next_data\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m   1520\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._shutdown \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._tasks_outstanding <= \u001b[32m0\u001b[39m:\n\u001b[32m   1521\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m(\n\u001b[32m   1522\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mInvalid iterator state: shutdown or no outstanding tasks when fetching next data\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m   1523\u001b[39m     )\n\u001b[32m-> \u001b[39m\u001b[32m1524\u001b[39m idx, data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_get_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1525\u001b[39m \u001b[38;5;28mself\u001b[39m._tasks_outstanding -= \u001b[32m1\u001b[39m\n\u001b[32m   1526\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._dataset_kind == _DatasetKind.Iterable:\n\u001b[32m   1527\u001b[39m     \u001b[38;5;66;03m# Check for _IterableDatasetStopIteration\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/lab/lib/python3.14/site-packages/torch/utils/data/dataloader.py:1483\u001b[39m, in \u001b[36m_MultiProcessingDataLoaderIter._get_data\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m   1479\u001b[39m     \u001b[38;5;66;03m# In this case, `self._data_queue` is a `queue.Queue`,. But we don't\u001b[39;00m\n\u001b[32m   1480\u001b[39m     \u001b[38;5;66;03m# need to call `.task_done()` because we don't use `.join()`.\u001b[39;00m\n\u001b[32m   1481\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m   1482\u001b[39m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1483\u001b[39m         success, data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_try_get_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1484\u001b[39m         \u001b[38;5;28;01mif\u001b[39;00m success:\n\u001b[32m   1485\u001b[39m             \u001b[38;5;28;01mreturn\u001b[39;00m data\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/lab/lib/python3.14/site-packages/torch/utils/data/dataloader.py:1310\u001b[39m, in \u001b[36m_MultiProcessingDataLoaderIter._try_get_data\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m   1297\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_try_get_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout=_utils.MP_STATUS_CHECK_INTERVAL):\n\u001b[32m   1298\u001b[39m     \u001b[38;5;66;03m# Tries to fetch data from `self._data_queue` once for a given timeout.\u001b[39;00m\n\u001b[32m   1299\u001b[39m     \u001b[38;5;66;03m# This can also be used as inner loop of fetching without timeout, with\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m   1307\u001b[39m     \u001b[38;5;66;03m# Returns a 2-tuple:\u001b[39;00m\n\u001b[32m   1308\u001b[39m     \u001b[38;5;66;03m#   (bool: whether successfully get data, any: data if successful else None)\u001b[39;00m\n\u001b[32m   1309\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1310\u001b[39m         data = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_data_queue\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1311\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m (\u001b[38;5;28;01mTrue\u001b[39;00m, data)\n\u001b[32m   1312\u001b[39m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m   1313\u001b[39m         \u001b[38;5;66;03m# At timeout and error, we manually check whether any worker has\u001b[39;00m\n\u001b[32m   1314\u001b[39m         \u001b[38;5;66;03m# failed. Note that this is the only mechanism for Windows to detect\u001b[39;00m\n\u001b[32m   1315\u001b[39m         \u001b[38;5;66;03m# worker failures.\u001b[39;00m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.14.2/lib/python3.14/multiprocessing/queues.py:111\u001b[39m, in \u001b[36mQueue.get\u001b[39m\u001b[34m(self, block, timeout)\u001b[39m\n\u001b[32m    109\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m block:\n\u001b[32m    110\u001b[39m     timeout = deadline - time.monotonic()\n\u001b[32m--> \u001b[39m\u001b[32m111\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_poll\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[32m    112\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m Empty\n\u001b[32m    113\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m._poll():\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.14.2/lib/python3.14/multiprocessing/connection.py:263\u001b[39m, in \u001b[36m_ConnectionBase.poll\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m    261\u001b[39m \u001b[38;5;28mself\u001b[39m._check_closed()\n\u001b[32m    262\u001b[39m \u001b[38;5;28mself\u001b[39m._check_readable()\n\u001b[32m--> \u001b[39m\u001b[32m263\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_poll\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.14.2/lib/python3.14/multiprocessing/connection.py:457\u001b[39m, in \u001b[36mConnection._poll\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m    456\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_poll\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout):\n\u001b[32m--> \u001b[39m\u001b[32m457\u001b[39m     r = \u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    458\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mbool\u001b[39m(r)\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.14.2/lib/python3.14/multiprocessing/connection.py:1165\u001b[39m, in \u001b[36mwait\u001b[39m\u001b[34m(object_list, timeout)\u001b[39m\n\u001b[32m   1162\u001b[39m     deadline = time.monotonic() + timeout\n\u001b[32m   1164\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1165\u001b[39m     ready = \u001b[43mselector\u001b[49m\u001b[43m.\u001b[49m\u001b[43mselect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1166\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m ready:\n\u001b[32m   1167\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m [key.fileobj \u001b[38;5;28;01mfor\u001b[39;00m (key, events) \u001b[38;5;129;01min\u001b[39;00m ready]\n",
      "\u001b[36mFile \u001b[39m\u001b[32m~/.pyenv/versions/3.14.2/lib/python3.14/selectors.py:398\u001b[39m, in \u001b[36m_PollLikeSelector.select\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m    396\u001b[39m ready = []\n\u001b[32m    397\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m398\u001b[39m     fd_event_list = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_selector\u001b[49m\u001b[43m.\u001b[49m\u001b[43mpoll\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    399\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mInterruptedError\u001b[39;00m:\n\u001b[32m    400\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m ready\n",
      "\u001b[31mKeyboardInterrupt\u001b[39m: "
     ]
    }
   ],
   "source": [
    "models_mlp = {}\n",
    "for features_to_use in features_list:\n",
    "    print(f\"Training MLP model with features: {features_to_use}\")\n",
    "    mlp_model_wrapper = MLPModelWrapper(\n",
    "        features_used=features_to_use,\n",
    "        train_loader=train_loader,\n",
    "        val_loader=val_loader,\n",
    "        test_loader=test_loader,\n",
    "        lr=LR,\n",
    "        hidden_layers=[50, 20],\n",
    "        apply_low_var_feat_reduction=False,\n",
    "        apply_pca=False\n",
    "    )\n",
    "    mlp_model_wrapper.train(epochs=50, verbose=1, plotting=True, patience=25)\n",
    "    models_mlp[\"+\".join(features_to_use)] = mlp_model_wrapper"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lab",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
