genre_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aGTJm-VTever"
   },
   "outputs": [],
   "source": [
    "!pip install -U scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vzHA5u9nevew"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/ultralytics/yolov5/master/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dkNRwGgzfI1M",
    "outputId": "bce3fc0c-d622-4f2a-a6c3-15445211eaf4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8ZBDlBdchaxf"
   },
   "outputs": [],
   "source": [
    "!unzip gdrive/My\\ Drive/movies/metadata.zip\n",
    "!unzip gdrive/My\\ Drive/movies/dataset.zip \n",
    "!unzip gdrive/My\\ Drive/movies/croppedonce.zip \n",
    "!unzip gdrive/My\\ Drive/movies/cropped.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SHzAcngKZNSJ"
   },
   "outputs": [],
   "source": [
    "#!unzip gdrive/My\\ Drive/movieset/trainingsmall.zip \n",
    "#!unzip gdrive/My\\ Drive/movieset/testingsmall.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "V0MlomrGAhhg",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import shutil\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "import torch.utils.data as data_utils\n",
    "from torch.nn.modules import MSELoss, L1Loss\n",
    "\n",
    "import sklearn.preprocessing\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import glob\n",
    "import csv\n",
    "import cv2\n",
    "import random\n",
    "from PIL import Image\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "Wo0qf35reve0"
   },
   "outputs": [],
   "source": [
    "path1 = \"./Movie_Poster_Metadata/groundtruth\"\n",
    "temp_path = \"./Movie_Poster_Metadata/temp_groundtruth\"\n",
    "path2 = \"./Movie_Poster_Metadata/updated_groundtruth\"\n",
    "cropped_dataset = \"./Dataset_Cropped\"\n",
    "# Jay's gdrive paths:\n",
    "#path1 = \"/content/drive/MyDrive/Uppsala University/Study material/NCML/Movie_Poster_Metadata/groundtruth\"\n",
    "#temp_path = \"./Movie_Poster_Metadata/temp_groundtruth\"\n",
    "#path2 = \"/content/gdrive/MyDrive/Uppsala University/Study material/NCML/Movie_Poster_Metadata/updated_groundtruth\"\n",
    "\n",
    "# path1 = \"./Movie_Poster_Metadata/groundtruth\"\n",
    "# temp_path = \"./Movie_Poster_Metadata/temp_groundtruth\"\n",
    "# path2 = \"./Movie_Poster_Metadata/updated_groundtruth\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "snqkcEj7eve1"
   },
   "source": [
    "### Reading the input file and creating a clean one\n",
    "Note: only run once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Z_gG-mC7eve3"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "dir_list = os.listdir(path1)\n",
    " \n",
    "if not os.path.exists(temp_path):\n",
    "  os.makedirs(temp_path)    \n",
    "\n",
    "if not os.path.exists(path2):\n",
    "  os.makedirs(path2)\n",
    "\n",
    "\n",
    "\n",
    "for file_name in dir_list:\n",
    "    \n",
    "    with open(path1+'/'+file_name,'r',encoding='utf-16-le') as file1:\n",
    "\n",
    "        temp_file = open(temp_path+'/'+file_name,'w',encoding='utf-8')\n",
    "\n",
    "        for line in file1.readlines():\n",
    "\n",
    "            line = line.replace(\"}\\n\",\"},\\n\")\n",
    "            \n",
    "            # reading all lines that begin with \"  \"_id\"\"\n",
    "            y = re.findall(\"^  \\\"_id\\\"\", line)\n",
    "            if not y:\n",
    "                temp_file.write(line)\n",
    "\n",
    "    file1.close()\n",
    "    temp_file.close()\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OxohJlpieve6"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "dir_list = os.listdir(temp_path)\n",
    " \n",
    "for file_name in dir_list:\n",
    "    \n",
    "    with open(temp_path+'/'+file_name,'r',encoding='utf-8') as temp_file:\n",
    "    \n",
    "        file2 = open(path2+'/'+file_name,'w',encoding='utf-8')\n",
    "\n",
    "        lines = temp_file.readlines()\n",
    "        lines = lines[1:-1]\n",
    "\n",
    "        file2.write(\"[{\")\n",
    "        file2.writelines(lines)\n",
    "        file2.write(\"}]\")\n",
    "        \n",
    "    temp_file.close()\n",
    "    file2.close()\n",
    "\n",
    "shutil.rmtree(temp_path)  \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0Q8-JNQ4eve7"
   },
   "source": [
    "### Augmenting the data set\n",
    "Note: only run once\n",
    "\n",
    "To-Do: Balance data according to occurence of genres. Summarize genres with little data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SBlOuf5aeve9"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "path3 = \"./Movie_Poster_Dataset\"\n",
    "\n",
    "# Going through all jpg-files, they are chopped up into 100x100 chunks and saved into a new folder\n",
    "for dirname in os.listdir(path3):\n",
    "    for filename in os.listdir(path3 + \"/\" + dirname):\n",
    "        name, ext = os.path.splitext(filename)\n",
    "        if(ext == '.jpg'):\n",
    "            image = Image.open(os.path.join(path3 + \"/\" + dirname, filename))\n",
    "            width, height = image.size\n",
    "            chopsize = 100\n",
    "            for x0 in range(0, width, chopsize):\n",
    "                for y0 in range(0, height, chopsize):\n",
    "                    if(y0+chopsize <= height and x0+chopsize <= width):\n",
    "                        box = (x0, y0, x0+chopsize, y0+chopsize)\n",
    "                        image.crop(box).save('gdrive/MyDrive/movies/Movie_Poster_Dataset_Cropped/%s.x%03d.y%03d.jpg' % (filename.replace('.jpg',''), x0, y0))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9YKiuwACeve_"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "path3 = \"'gdrive/MyDrive/movies/Movie_Poster_Dataset\"\n",
    "\n",
    "# Going through all jpg-files, they are chopped up into 100x100 chunks and saved into a new folder\n",
    "for dirname in os.listdir(path3):\n",
    "    for filename in os.listdir(path3 + \"/\" + dirname):\n",
    "        name, ext = os.path.splitext(filename)\n",
    "        if(ext == '.jpg'):\n",
    "            image = Image.open(os.path.join(path3 + \"/\" + dirname, filename))\n",
    "            box = (0, 0, 100, 100)\n",
    "            image.crop(box).save('./Movie_Poster_Dataset_Cropped_Once/%s.jpg' % (filename))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mRwNR4vzevfA"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "\n",
    "#to be used later to augment data of underrepresented genres (balance data)\n",
    "\n",
    "print('Nr of movies in json: '+str(len(dicts)))\n",
    "missing = []\n",
    "for obj in dicts:\n",
    "    genrelist = obj.get('Genre').split(',')\n",
    "    fname = obj.get('imdbID') + '.jpg'\n",
    "    if(path.exists(fname)):\n",
    "        for genre in genrelist:\n",
    "            #copy the file with name obj.key(\"imdbID\") to each genre folder\n",
    "            if(genre == 'N/A'):\n",
    "                shutil.copy2(os.path.join('.', fname), './NotApplicable')\n",
    "            elif(genre == 'Adult' || genre == 'Game-Show' || genre == 'News' || genre == 'Reality-TV' || genre == 'Talk-Show' || genre == 'Western'):\n",
    "                shutil.copy2(os.path.join('.', fname), './Other')\n",
    "            else:\n",
    "                shutil.copy2(os.path.join('.', fname), './'+genre.lstrip())\n",
    "    else:\n",
    "        missing.append(fname)\n",
    "\n",
    "\n",
    "print('Nr of missing IDs: '+str(len(missing)))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VOd-ltaPevfB"
   },
   "source": [
    "### Function to append all the json objects into dataframe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "pUpfYvtWevfB",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dir_list = os.listdir(path2)\n",
    "\n",
    "movies_df = pd.DataFrame()\n",
    "\n",
    "for file_name in dir_list:    \n",
    "\n",
    "#     try:\n",
    "    df = pd.read_json(path2+'/'+file_name,encoding='utf-8',orient='records')\n",
    "    df = df[['imdbID','Director','Genre','imdbRating']]\n",
    "    movies_df = pd.concat([movies_df,df], ignore_index=True)\n",
    "\n",
    "#     except:\n",
    "#         print(file_name)\n",
    "        \n",
    "# print(movies_df.dtypes)\n",
    "# print(movies_df.head(20))\n",
    "# print(movies_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AkmXxvUBevfC"
   },
   "source": [
    "### Creating multi-hot encoded genre vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "DXEDdPjaevfC"
   },
   "outputs": [],
   "source": [
    "#remove duplicates and set imdbID as index\n",
    "movies_df = movies_df.drop_duplicates(subset=[\"imdbID\"], keep=\"last\")\n",
    "movies_df.set_index(\"imdbID\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "f_gx5brSevfD"
   },
   "outputs": [],
   "source": [
    "mlb = MultiLabelBinarizer()\n",
    "multihot_temp = mlb.fit_transform(movies_df[\"Genre\"].dropna().str.split(\", \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#['Action' 'Adult' 'Adventure' 'Animation' 'Biography' \n",
    "#'Comedy' 'Crime' 'Documentary' 'Drama' 'Family' \n",
    "#'Fantasy' 'Game-Show' 'History' 'Horror' 'Music' \n",
    "#'Musical' 'Mystery' 'N/A' 'News' 'Reality-TV' \n",
    "#'Romance' 'Sci-Fi' 'Short' 'Sport' 'Talk-Show' \n",
    "#'Thriller' 'War' 'Western']\n",
    "\n",
    "multihot = [0]*len(multihot_temp)\n",
    "\n",
    "item = 0\n",
    "for vec in multihot_temp:\n",
    "    \n",
    "    new_vec = np.empty([20])\n",
    "    for i in range(20):\n",
    "        new_vec[i] = 0\n",
    "    \n",
    "    index = 0\n",
    "    for i in range(28):\n",
    "        \n",
    "        #If genre is set in multihot-encoded vector\n",
    "        if vec[i] == 1:\n",
    "            \n",
    "            #Adult, Game-Show, News, Reality-TV, Short, Talk-Show, Western -> Other\n",
    "            other_genres = [1, 11, 18, 19, 22, 24, 27]\n",
    "            if i in other_genres:\n",
    "                new_vec[19] = 1 \n",
    "                \n",
    "            #Put Musical in same category as Music\n",
    "            if i == 15:\n",
    "                new_vec[index-1] = 1\n",
    "        \n",
    "            #Skip N/A\n",
    "            if i == 17:\n",
    "                continue\n",
    "            \n",
    "            else:\n",
    "                new_vec[index] = 1\n",
    "                index += 1\n",
    "                \n",
    "        if vec[i] == 0:\n",
    "            relevant_genres = [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 16, 20, 21, 23, 25]\n",
    "            if i in relevant_genres:\n",
    "                index += 1\n",
    "                \n",
    "    multihot[item] = new_vec\n",
    "    item += 1\n",
    "#print(multihot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Action' 'Adult' 'Adventure' 'Animation' 'Biography' 'Comedy' 'Crime'\n",
      " 'Documentary' 'Drama' 'Family' 'Fantasy' 'Game-Show' 'History' 'Horror'\n",
      " 'Music' 'Musical' 'Mystery' 'N/A' 'News' 'Reality-TV' 'Romance' 'Sci-Fi'\n",
      " 'Short' 'Sport' 'Talk-Show' 'Thriller' 'War' 'Western']\n",
      "                                           Director  \\\n",
      "imdbID                                                \n",
      "tt0080684                            Irvin Kershner   \n",
      "tt0081562                            Sidney Poitier   \n",
      "tt0080339  Jim Abrahams, David Zucker, Jerry Zucker   \n",
      "tt0080377                            Buddy Van Horn   \n",
      "tt0081375                              Howard Zieff   \n",
      "tt0080549                             Michael Apted   \n",
      "tt0081529                               Hal Needham   \n",
      "tt0080453                            Randal Kleiser   \n",
      "tt0080455                               John Landis   \n",
      "tt0081283                            Robert Redford   \n",
      "\n",
      "                                Genre imdbRating  \\\n",
      "imdbID                                             \n",
      "tt0080684  Action, Adventure, Fantasy        8.8   \n",
      "tt0081562               Comedy, Crime        6.8   \n",
      "tt0080339                      Comedy        7.8   \n",
      "tt0080377              Action, Comedy        6.0   \n",
      "tt0081375                 Comedy, War        6.1   \n",
      "tt0080549     Biography, Drama, Music        7.5   \n",
      "tt0081529              Action, Comedy        5.1   \n",
      "tt0080453   Adventure, Drama, Romance        5.7   \n",
      "tt0080455       Action, Comedy, Crime        7.9   \n",
      "tt0081283                       Drama        7.8   \n",
      "\n",
      "                                                    multihot  \n",
      "imdbID                                                        \n",
      "tt0080684  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0081562  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0080339  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0080377  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0081375  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0080549  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0081529  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0080453  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0080455  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n",
      "tt0081283  [[1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,...  \n"
     ]
    }
   ],
   "source": [
    "genres_df = pd.DataFrame({\"multihot\":[multihot_temp.astype(int)]}, index = movies_df.index)\n",
    "movies_df = pd.concat([movies_df, genres_df], axis=1 )\n",
    "print(mlb.classes_)\n",
    "print(movies_df.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "wJd1F0g-evfD"
   },
   "outputs": [],
   "source": [
    "#create a dictionary with multi-hot encoded vectors; index = imdbID\n",
    "multihot_dict = {movies_df.index.tolist()[i] : multihot[i] for i in range(0, len(multihot))}\n",
    "#print(multihot_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OsZB9cOIevfE"
   },
   "source": [
    "### Adding the images to the dataframe\n",
    "Note: not used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yOW3UMBxevfE"
   },
   "outputs": [],
   "source": [
    "flist=glob.glob('./Movie_Poster_Dataset/*/*.jpg')\n",
    "\n",
    "imdb_id_arr = [\"0\" for a in range(len(flist))]\n",
    "image_arr = [\"0\" for a in range(len(flist))]\n",
    "index = 0\n",
    "\n",
    "for filename in flist:\n",
    "        \n",
    "    imdb_id = filename[filename.index(\"tt\"):filename.index(\".jpg\")]\n",
    "        \n",
    "    imdb_id_arr[index] = imdb_id\n",
    "                \n",
    "    img = np.array(cv2.imread(filename))\n",
    "    img = np.swapaxes(img, 2,0)\n",
    "    img = np.swapaxes(img, 2,1)\n",
    "    \n",
    "    image_arr[index] = img\n",
    "    \n",
    "    index +=1 \n",
    "        \n",
    "image_dict = {\n",
    "    \"imdbID\": imdb_id_arr,\n",
    "    \"Poster\": image_arr\n",
    "}\n",
    "\n",
    "images_df = pd.DataFrame.from_dict(image_dict)\n",
    "images_df = images_df.drop_duplicates(subset=[\"imdbID\"], keep=\"last\")\n",
    "images_df.set_index(\"imdbID\", inplace=True)\n",
    "movies_df = pd.concat([movies_df, images_df], axis=1)\n",
    "print(movies_df.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6syvhuE-yj2m"
   },
   "source": [
    "## Hyper Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "FANO93KTyif-"
   },
   "outputs": [],
   "source": [
    "#training controls\n",
    "batch_size = 1\n",
    "number_of_labels = 20\n",
    "epochs = 10\n",
    "training_size = 0.7\n",
    "learning_rate = 0.5 #0.1 #0.01 #0.001\n",
    "dropout = [0.3, 0.3, 0.3, 0.3, 0.2, 0.2, 0.2, 0.2, 0.15]\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 100, 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TXI9-JvvevfF"
   },
   "source": [
    "### Passing the images through a convolutional network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "b0axZikbevfF"
   },
   "outputs": [],
   "source": [
    "# the data holders\n",
    "x_test = []\n",
    "x_train = []\n",
    "y_test = []\n",
    "y_train = []\n",
    "\n",
    "#images need to have the same size!!\n",
    "flist=glob.glob('./Dataset_Cropped/*.jpg')\n",
    "\n",
    "length=int(len(flist)*training_size)\n",
    "i = 0\n",
    "\n",
    "#create lists with input data (images) and output data (multi-hot encoded genre vectors)\n",
    "for filename in flist:\n",
    "        \n",
    "    imdb_id = filename[filename.index(\"tt\"):filename.index(\".x\")]\n",
    "      \n",
    "    if imdb_id in multihot_dict:\n",
    "        img = np.array(cv2.imread(filename))\n",
    "        img = np.swapaxes(img, 2,0)\n",
    "        img = np.swapaxes(img, 2,1)\n",
    "        \n",
    "        genre_arr = np.empty([20])\n",
    "        \n",
    "        for j in range(len(multihot_dict[imdb_id])):\n",
    "            genre_arr[j] = multihot_dict[imdb_id][j]\n",
    "    \n",
    "        if(i<length):  \n",
    "            x_train.append(img)\n",
    "            y_train.append(genre_arr)\n",
    "        else:\n",
    "            x_test.append(img)\n",
    "            y_test.append(genre_arr)\n",
    "        \n",
    "        i +=1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "waUcfZkSevfG"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37032\n",
      "37032\n",
      "3\n",
      "20\n",
      "3\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "print(len(x_train))\n",
    "print(len(y_train))\n",
    "\n",
    "print(len(x_train[0]))\n",
    "print(len(y_train[0]))\n",
    "\n",
    "print(len(x_test[0]))\n",
    "print(len(y_test[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "770N0xHeevfH",
    "outputId": "fbff51fb-0322-4a71-9a3a-6f880ba20f1c",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (37032, 3, 100, 100)\n",
      "37032 train samples\n",
      "15872 test samples\n"
     ]
    }
   ],
   "source": [
    "#converting the data from lists to numpy arrays\n",
    "x_train=np.asarray(x_train,dtype=float)\n",
    "x_test=np.asarray(x_test,dtype=float)\n",
    "y_train=np.asarray(y_train,dtype=float)\n",
    "y_test=np.asarray(y_test,dtype=float)\n",
    "\n",
    "#scaling down the RGB data\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "#printing stats about the features\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "train_length = x_train.shape[0]\n",
    "\n",
    "x_train=torch.from_numpy(x_train)\n",
    "x_test=torch.from_numpy(x_test)\n",
    "y_train=torch.from_numpy(y_train)\n",
    "y_test=torch.from_numpy(y_test)\n",
    "\n",
    "train = data_utils.TensorDataset(x_train, y_train)\n",
    "train_loader = data_utils.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "test = data_utils.TensorDataset(x_test, y_test)\n",
    "test_loader = data_utils.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IdbLrLz4p0K8"
   },
   "source": [
    "### DenseNet-121"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TyK6balPp9LZ",
    "outputId": "e9144425-8a48-4a75-efa3-34b8d78b6424"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in C:\\Users\\carol/.cache\\torch\\hub\\pytorch_vision_v0.10.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model loaded on cpu\n",
      "DenseNet(\n",
      "  (features): Sequential(\n",
      "    (conv0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
      "    (norm0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (relu0): ReLU(inplace=True)\n",
      "    (pool0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
      "    (denseblock1): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(96, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition1): _Transition(\n",
      "      (norm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock2): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(160, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(160, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(192, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(224, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(224, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition2): _Transition(\n",
      "      (norm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock3): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(288, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(320, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(352, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(352, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(384, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(416, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(416, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(448, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(448, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(480, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer13): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer14): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer15): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer16): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer17): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer18): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer19): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer20): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer21): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer22): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer23): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer24): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (transition3): _Transition(\n",
      "      (norm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (pool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
      "    )\n",
      "    (denseblock4): _DenseBlock(\n",
      "      (denselayer1): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer2): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(544, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(544, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer3): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(576, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer4): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(608, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(608, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer5): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(640, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(640, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer6): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(672, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer7): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(704, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer8): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(736, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(736, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer9): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer10): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(800, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(800, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer11): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(832, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer12): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(864, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(864, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer13): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(896, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(896, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer14): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(928, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(928, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer15): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(960, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(960, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "      (denselayer16): _DenseLayer(\n",
      "        (norm1): BatchNorm2d(992, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu1): ReLU(inplace=True)\n",
      "        (conv1): Conv2d(992, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (relu2): ReLU(inplace=True)\n",
      "        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      )\n",
      "    )\n",
      "    (norm5): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Linear(in_features=1024, out_features=20, bias=True)\n",
      "    (1): Sigmoid()\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = torch.hub.load('pytorch/vision:v0.10.0', 'densenet121', pretrained=True) # TODO: Modify the fully connected layer of denseNet\n",
    "\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False \n",
    "    # Replace the last fully-connected layer\n",
    "    # Parameters of newly constructed modules have requires_grad=True by default\n",
    "# num_ftrs = model.fc.in_features # num_ftrs = 1024\n",
    "# model.classifier = nn.Linear(num_ftrs, 28)\n",
    "# model.fc = nn.Linear(num_ftrs, 28) \n",
    "# sigmoid = nn.Sigmoid(model.fc)\n",
    "# sigmoid(model.fc)\n",
    "#TODO: make a sigmoid activation func, decision threshold of 0.3\n",
    "\n",
    "model.classifier = nn.Sequential(\n",
    "    nn.Linear(1024, 20),\n",
    "    # nn.Linear(1024, 512),\n",
    "    # nn.Dropout(p=0.1),\n",
    "    # nn.ReLU(),\n",
    "    # nn.Linear(512, 28),\n",
    "    nn.Sigmoid()\n",
    ")\n",
    "\n",
    "\n",
    "model.double() \n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"model loaded on {device}\")\n",
    "model.to(device) #hopefully runs model on cuda core.\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L5iWf_9WpfaP"
   },
   "source": [
    "## Custom CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 244
    },
    "id": "sJvV2PKLevfI",
    "outputId": "e3abc600-aacb-4a3c-9b15-8968cbf6ed77"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-7d3771e8fef8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#BUG: forward function doesnt work and the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# layers are made in the order in which they are declared in __init__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mNetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg_rows\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg_cols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNetwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'nn' is not defined"
     ]
    }
   ],
   "source": [
    "#BUG: forward function doesnt work and the \n",
    "# layers are made in the order in which they are declared in __init__\n",
    "class Network(nn.Module):\n",
    "    def __init__(self, input_shape=(3, img_rows, img_cols)):\n",
    "        super(Network, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 128, kernel_size=2)\n",
    "        self.conv1_drop = nn.Dropout2d(p=dropout[0])\n",
    "        self.conv2 = nn.Conv2d(128, 64, kernel_size=2)\n",
    "        self.conv2_drop = nn.Dropout2d(p=dropout[1])\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=2)\n",
    "        self.conv3_drop = nn.Dropout2d(p=dropout[2])\n",
    "        self.conv4 = nn.Conv2d(64, 64, kernel_size=2)\n",
    "        self.conv4_drop = nn.Dropout2d(p=dropout[3])\n",
    "        self.conv5 = nn.Conv2d(64, 32, kernel_size=2)\n",
    "        self.conv5_drop = nn.Dropout2d(p=dropout[4])\n",
    "        self.conv6 = nn.Conv2d(32, 16, kernel_size=2)\n",
    "        self.conv6_drop = nn.Dropout2d(p=dropout[5])\n",
    "        \n",
    "        self.n_size = self._get_conv_output(input_shape)\n",
    "        \"\"\"\n",
    "        self.fc1 = nn.Linear(n_size, 16)\n",
    "        self.fc1_drop = nn.Dropout(p=dropout[6])\n",
    "        self.fc2 = nn.Linear(16, 16)\n",
    "        self.fc2_drop = nn.Dropout(p=dropout[7])\n",
    "        self.fc3 = nn.Linear(16, 8)\n",
    "        self.fc3_drop = nn.Dropout(p=dropout[8])\n",
    "        self.fc4 = nn.Linear(8, 28)\n",
    "        \"\"\"\n",
    "        self.fc1 = nn.Linear(self.n_size, 28)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "        \n",
    "    def _get_conv_output(self, shape):\n",
    "        bs = 1\n",
    "        input = Variable(torch.rand(bs, *shape))\n",
    "        # output_feat = self._forward_features(input)\n",
    "        output_feat = self.forward(input)\n",
    "        n_size = output_feat.data.view(bs, -1).size(1)\n",
    "        return n_size\n",
    "        \n",
    "    def _forward_features(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1_drop(self.conv1(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv3_drop(self.conv3(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv4_drop(self.conv4(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv5_drop(self.conv5(x)), 2))\n",
    "        #x = F.relu(F.max_pool2d(self.conv6_drop(self.conv6(x)), 2))\n",
    "        return x\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1_drop(self.conv1(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv3_drop(self.conv3(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv4_drop(self.conv4(x)), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv5_drop(self.conv5(x)), 2))\n",
    "        #x = F.relu(F.max_pool2d(self.conv6_drop(self.conv6(x)), 2))\n",
    "        x = x.view(x.size(0), -1)\n",
    "        # x = self.sigmoid(x)\n",
    "        # x = torch.sigmoid(self.fc1(x))\n",
    "\n",
    "        #x = F.relu(self.fc1_drop(self.fc1(x)))\n",
    "        #x = F.relu(self.fc2_drop(self.fc2(x)))\n",
    "        #x = F.relu(self.fc3_drop(self.fc3(x)))\n",
    "        #x = self.fc4(x)\n",
    "        return x\n",
    "\n",
    "# model = Network()\n",
    "# model.double() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o4jb48Z8rtgX"
   },
   "source": [
    "## Loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "f9opQp38UYY5"
   },
   "outputs": [],
   "source": [
    "loss_fn = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0) #TODO: check best params here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tJ0kg2ICmmdq"
   },
   "source": [
    "## Train and Test functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ZQIvRfhTevfI"
   },
   "outputs": [],
   "source": [
    "# Function to save the model\n",
    "def saveModel():\n",
    "    path = \"./myFirstModel.pth\"\n",
    "    torch.save(model.state_dict(), path)\n",
    "\n",
    "# Function to test the model with the test dataset and print the accuracy for the test images\n",
    "def testAccuracy():\n",
    "    \n",
    "    model.eval() #TODO: check model.train() which is used along model.eval()\n",
    "    accuracy = 0.0\n",
    "    total = 0.0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "        print(f\"model testing on {device}\")\n",
    "        some_accuracy_measure = 0\n",
    "        final_accuracy = 0\n",
    "        for i, (images, labels) in enumerate(test_loader,0):\n",
    "            sum = 0\n",
    "            images = Variable(images.to(device))\n",
    "            labels = Variable(labels.to(device))\n",
    "            outputs = model(images) \n",
    "            # print(outputs)\n",
    "            for j, label in enumerate(labels):\n",
    "              n = sum + label.sum().item()\n",
    "              # get the top n values of outputs\n",
    "              _, predicted = torch.topk(outputs[j], int(n)) #TODO: find best threshold and calc F1.\n",
    "              \n",
    "              correct_predictions = 0\n",
    "              for _, k in enumerate(predicted):\n",
    "                if(label[k.item()].item() == 1):\n",
    "                  correct_predictions += 1\n",
    "                some_accuracy_measure = correct_predictions/n\n",
    "            final_accuracy += some_accuracy_measure  \n",
    "            # the label with the highest energy will be our prediction\n",
    "            #_, predicted = torch.max(outputs.data, 1)\n",
    "            #print(\"Predicted: \", predicted)\n",
    "            total += labels.size(0)\n",
    "            \n",
    "            #accuracy += (predicted == labels).sum().item()\n",
    "    print(\"Total: \", total)\n",
    "    # compute the accuracy over all test images\n",
    "    #accuracy = (100 * accuracy / total)\n",
    "    accuracy = final_accuracy * 100 /total\n",
    "    print(\"Accuracy;: \", accuracy)\n",
    "    return(accuracy)\n",
    "\n",
    "\n",
    "# Training function. We simply have to loop over our data iterator and feed the inputs to the network and optimize.\n",
    "def train(num_epochs):\n",
    "    \n",
    "    best_accuracy = 0.0\n",
    "\n",
    "    # Define your execution device\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"The model will be running on\", device, \"device\")\n",
    "    # Convert model parameters and buffers to CPU or Cuda\n",
    "    model.to(device)\n",
    "\n",
    "    for epoch in range(num_epochs):  # loop over the dataset multiple times\n",
    "        running_loss = 0.0\n",
    "        running_acc = 0.0\n",
    "\n",
    "        for i, (images, labels) in enumerate(train_loader, 0):\n",
    "            \n",
    "            # get the inputs\n",
    "            images = Variable(images.to(device))\n",
    "            labels = Variable(labels.to(device))\n",
    "            #labels = labels.unsqueeze_(0)\n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "            # predict classes using images from the training set\n",
    "            outputs = model(images)\n",
    "            # compute the loss based on model output and real labels\n",
    "            loss = loss_fn(outputs, labels)\n",
    "            # backpropagate the loss\n",
    "            loss.backward()\n",
    "            # adjust parameters based on the calculated gradients\n",
    "            optimizer.step()\n",
    "\n",
    "            # Let's print statistics for every 1,000 images\n",
    "            running_loss += loss.item()     # extract the loss value\n",
    "            if i % 1000 == 0:    \n",
    "                # print every 1000 (twice per epoch) \n",
    "                #print('[%d, %5d] loss: %.3f' %\n",
    "                #      (epoch + 1, i + 1, running_loss / 1000))\n",
    "                # zero the loss\n",
    "                running_loss = 0.0\n",
    "\n",
    "        # Compute and print the average accuracy fo this epoch when tested over all 10000 test images\n",
    "        accuracy = testAccuracy() #BUG: why is this in for loop? \n",
    "        print('For epoch', epoch+1,'the test accuracy over the whole test set is %d %%' % (accuracy))\n",
    "        \n",
    "        # we want to save the model if the accuracy is the best\n",
    "        if accuracy > best_accuracy:\n",
    "            saveModel()\n",
    "            best_accuracy = accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t8V_GcgfnZLf"
   },
   "source": [
    "## Functions to display sample output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "bRs5CZQvevfK"
   },
   "outputs": [],
   "source": [
    "# Function to show the images\n",
    "def imageshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# Function to test the model with a batch of images and show the labels predictions\n",
    "def testBatch():\n",
    "    # get batch of images from the test DataLoader  \n",
    "    images, labels = next(iter(test_loader))\n",
    "\n",
    "    # show all images as one image grid\n",
    "    imageshow(torchvision.utils.make_grid(images))\n",
    "   \n",
    "    # Show the real labels on the screen \n",
    "    print('Real labels: ', ' '.join('%5s' % classes[labels[j]] \n",
    "                               for j in range(batch_size)))\n",
    "  \n",
    "    # Let's see what if the model identifiers the  labels of those example\n",
    "    outputs = model(images)\n",
    "    \n",
    "    # We got the probability for every 10 labels. The highest (max) probability should be correct label\n",
    "    _, predicted = torch.max(outputs, 1)\n",
    "    \n",
    "    # Let's show the predicted labels on the screen to compare with the real ones\n",
    "    print('Predicted: ', ' '.join('%5s' % classes[predicted[j]] \n",
    "                              for j in range(batch_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FUJJvd51rPzK"
   },
   "source": [
    "## Run Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gmm4YxLMevfL",
    "outputId": "86698cfb-20b2-470e-86a2-12c97055bd8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model will be running on cpu device\n",
      "model testing on cpu\n",
      "Total:  15872.0\n",
      "Accuracy;:  19.094947076613234\n",
      "For epoch 1 the test accuracy over the whole test set is 19 %\n"
     ]
    }
   ],
   "source": [
    "# for epoch in range(0, epochs):\n",
    "\n",
    "train(epochs)\n",
    "    #test()\n",
    "\n",
    "# accuracy = testAccuracy()\n",
    "# print('For epoch', epoch+1,'the test accuracy over the whole test set is %d %%' % (accuracy))\n",
    "\n",
    "# best_accuracy = 0\n",
    "# # we want to save the model if the accuracy is the best\n",
    "# if accuracy > best_accuracy:\n",
    "#     saveModel()\n",
    "#     best_accuracy = accuracy\n",
    "\n",
    "#TODO: plot learning curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zWazb0fHevfL"
   },
   "source": [
    "### Passing the images through object detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j3jEunoNevfL"
   },
   "outputs": [],
   "source": [
    "batch_size = 20\n",
    "epochs = 2\n",
    "training_size = 0.7\n",
    "learning_rate = 0.001\n",
    "img_rows, img_cols = 100, 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5I8RKlyEevfM"
   },
   "outputs": [],
   "source": [
    "# the data holders\n",
    "x_test_yolo = []\n",
    "x_train_yolo = []\n",
    "y_test_yolo = []\n",
    "y_train_yolo = []\n",
    "\n",
    "#images need to have the same size!!\n",
    "flist=glob.glob('./Movie_Poster_Dataset_Cropped_Once/*.jpg')\n",
    "\n",
    "#pretrained YOLOv5 model\n",
    "yolo_model = torch.hub.load('ultralytics/yolov5', 'yolov5s')\n",
    "\n",
    "length=int(len(flist)*training_size)\n",
    "i = 0\n",
    "\n",
    "#create lists with input data (object confidence vector) and output data (multi-hot encoded genre vectors)\n",
    "for filename in flist:\n",
    "        \n",
    "    imdb_id = filename[filename.index(\"tt\"):filename.index(\".jpg\")]\n",
    "      \n",
    "    if imdb_id in multihot_dict:\n",
    "        img = np.array(cv2.imread(filename))\n",
    "        img = np.swapaxes(img, 2,0)\n",
    "        img = np.swapaxes(img, 2,1)\n",
    "        \n",
    "        results = yolo_model(img, size = 100)\n",
    "            \n",
    "        #create an array for the 91 object categories and set initial confidence to 0\n",
    "        obj_arr = np.empty([91])\n",
    "        for x in range(91):\n",
    "            obj_arr[x] = 0.0\n",
    "\n",
    "        #update the confidence values according to the object detection results\n",
    "        for obj in results.pandas().xyxy[0]:\n",
    "            index =  results.pandas().xyxy[0]['class']\n",
    "            obj_arr[index] = obj_arr[index] + results.pandas().xyxy[0].confidence\n",
    "        \n",
    "        #create multi-hot encoded genre vector\n",
    "        genre_arr = np.empty([28])\n",
    "\n",
    "        for j in range(len(multihot_dict[imdb_id])):\n",
    "            genre_arr[j] = multihot_dict[imdb_id][j]\n",
    "        \n",
    "        if(i<length):                   \n",
    "            x_train_yolo.append(obj_arr)\n",
    "            y_train_yolo.append(genre_arr)\n",
    "        else:\n",
    "            x_test_yolo.append(obj_arr)\n",
    "            y_test_yolo.append(genre_arr)\n",
    "        \n",
    "        i +=1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pJ23NN21evfN"
   },
   "outputs": [],
   "source": [
    "#converting the data from lists to numpy arrays\n",
    "x_train_yolo=np.asarray(x_train_yolo,dtype=float)\n",
    "x_test_yolo=np.asarray(x_test_yolo,dtype=float)\n",
    "y_train_yolo=np.asarray(y_train_yolo,dtype=float)\n",
    "y_test_yolo=np.asarray(y_test_yolo,dtype=float)\n",
    "\n",
    "#printing stats about the features\n",
    "print('x_train shape:', x_train_yolo.shape)\n",
    "print(x_train_yolo.shape[0], 'train samples')\n",
    "print(x_test_yolo.shape[0], 'test samples')\n",
    "\n",
    "train_length = x_train_yolo.shape[0]\n",
    "\n",
    "x_train_yolo=torch.from_numpy(x_train_yolo)\n",
    "x_test_yolo=torch.from_numpy(x_test_yolo)\n",
    "y_train_yolo=torch.from_numpy(y_train_yolo)\n",
    "y_test_yolo=torch.from_numpy(y_test_yolo)\n",
    "\n",
    "train = data_utils.TensorDataset(x_train_yolo, y_train_yolo)\n",
    "train_loader = data_utils.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "test = data_utils.TensorDataset(x_test_yolo, y_test_yolo)\n",
    "test_loader = data_utils.DataLoader(test, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KnrUeKK2evfN"
   },
   "outputs": [],
   "source": [
    "#fully connected layer after object detection\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        #fully connected layer\n",
    "        self.fc1 = nn.Linear(91, 28)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        output = F.log_softmax(x, dim=1)\n",
    "        return output\n",
    "\n",
    "model = Net()\n",
    "\n",
    "result = model.train()\n",
    "loss_fn = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.RMSprop(model.parameters(), lr=learning_rate,\n",
    "            alpha=0.9, eps=1e-08, weight_decay=0.0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oPVp7s4bevfO"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "snqkcEj7eve1",
    "0Q8-JNQ4eve7",
    "VOd-ltaPevfB",
    "AkmXxvUBevfC",
    "OsZB9cOIevfE",
    "TXI9-JvvevfF",
    "L5iWf_9WpfaP",
    "tJ0kg2ICmmdq",
    "t8V_GcgfnZLf",
    "zWazb0fHevfL"
   ],
   "name": "genre_classification_Victoria.ipynb",
   "provenance": []
  },
  "interpreter": {
   "hash": "95e1d69f56d8e2a587436622babdf358ae56e3753491c465a4bfef20ca6b03f3"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}