final_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mYQC7co3nee1"
   },
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bEjE6sfVdl3Y",
    "outputId": "bae7ad32-5ed5-4ea9-ad1d-cb7976284963"
   },
   "outputs": [],
   "source": [
    "# Installing split-folders for dataset split process\n",
    "!pip install split-folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tkpIURzmjPmJ",
    "outputId": "cf142874-94f5-466a-98b4-3e01ad5fd746"
   },
   "outputs": [],
   "source": [
    "# Importing all the required libraries\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import splitfolders\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "tfk = tf.keras\n",
    "tfkl = tf.keras.layers\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rcDOdGc4nnGm"
   },
   "source": [
    "### Setting the random seed for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "av_eWxAenxhM"
   },
   "outputs": [],
   "source": [
    "# Random seed for reproducibility\n",
    "seed = 42\n",
    "\n",
    "random.seed(seed)\n",
    "os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "np.random.seed(seed)\n",
    "tf.random.set_seed(seed)\n",
    "tf.compat.v1.set_random_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9qVVJTU7j-O0"
   },
   "source": [
    "### PREPARING THE DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XKz5CoQfrOfG"
   },
   "outputs": [],
   "source": [
    "# TRAINING VARIABLES\n",
    "# Learning rate \n",
    "lr = 1e-3   # 0.0005 - 0.000001\n",
    "# Batch size\n",
    "batch_size = 64\n",
    "\n",
    "# Dataset folders, must include dataset into same folder as the notebook to run\n",
    "dataset_dir = './dataset'\n",
    "training_dir = os.path.join(dataset_dir, 'training')\n",
    "\n",
    "# Listing all the labels\n",
    "labels = ['Apple','Blueberry','Cherry','Corn','Grape','Orange','Peach','Pepper','Potato','Raspberry','Soybean','Squash','Strawberry','Tomato']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-yzGkVx-5iZj"
   },
   "source": [
    "### Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DcWi9Iat96uk",
    "outputId": "ab5a1c9a-83de-4fa4-cf0f-36d757364fb3"
   },
   "outputs": [],
   "source": [
    "# REMOVE COMMENT ONLY ON FIRST RUN, ONLY ONE SPLIT NEEDED! \n",
    "# Dividing dataset into training, validation and test to avoid overfitting..\n",
    "splitfolders.ratio(training_dir, output=\"output\", seed=seed, ratio=(.7,.2,.1), group_prefix=None)\n",
    "\n",
    "\n",
    "# Using ImageDataGenerator to exploit folder organization for reading\n",
    "\n",
    "train_dir = \"output/train\"\n",
    "test_dir = \"output/test\"\n",
    "valid_dir = \"output/val\"\n",
    "\n",
    "# Initializing the generators, training one will have data augmentation options\n",
    "# to generalize the results more\n",
    "train_data_gen = ImageDataGenerator(rotation_range=30, \n",
    "                                    width_shift_range=15,\n",
    "                                    height_shift_range=15,\n",
    "                                    zoom_range=0.3,\n",
    "                                    horizontal_flip=True,\n",
    "                                    fill_mode='nearest')\n",
    "valid_data_gen = ImageDataGenerator()\n",
    "test_data_gen = ImageDataGenerator()\n",
    "\n",
    "# Obtain a data generator with the 'ImageDataGenerator.flow_from_directory' method\n",
    "train_gen = train_data_gen.flow_from_directory(directory=train_dir,\n",
    "                                               target_size=(256,256),\n",
    "                                               color_mode='rgb',\n",
    "                                               classes=None, \n",
    "                                               batch_size=batch_size,\n",
    "                                               shuffle=True,\n",
    "                                               seed=seed,\n",
    ")\n",
    "valid_gen = valid_data_gen.flow_from_directory(directory=valid_dir,\n",
    "                                               target_size=(256,256),\n",
    "                                               color_mode='rgb',\n",
    "                                               classes=None,\n",
    "                                               batch_size=batch_size,\n",
    "                                               shuffle=False,\n",
    "                                               seed=seed,)\n",
    "test_gen = test_data_gen.flow_from_directory(directory=test_dir,\n",
    "                                              target_size=(256,256),\n",
    "                                              color_mode='rgb',\n",
    "                                              classes=None, \n",
    "                                              batch_size=batch_size,\n",
    "                                              shuffle=False,\n",
    "                                              seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PYPILXgSsilN",
    "outputId": "7186315b-8f6c-44a4-cd36-95b4a9cf7a48"
   },
   "outputs": [],
   "source": [
    "# Sanity check\n",
    "print(\"Assigned labels\")\n",
    "print(train_gen.class_indices)\n",
    "print()\n",
    "print(\"Target classes\")\n",
    "print(train_gen.classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GY8Qgjv3lneo"
   },
   "source": [
    "### TRANSFER LEARNING MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "0twHCq1ihJLd",
    "outputId": "0e00ae57-c46c-467e-8fb4-44454fbf980c"
   },
   "outputs": [],
   "source": [
    "# Download and visualizing the VGG16 model\n",
    "supernet = tfk.applications.VGG16(\n",
    "    include_top=False,\n",
    "    weights=\"imagenet\",\n",
    "    input_shape=(256,256,3)\n",
    ")\n",
    "\n",
    "supernet.summary()\n",
    "\n",
    "tfk.utils.plot_model(supernet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Uc6MlMIJlnHg"
   },
   "outputs": [],
   "source": [
    "def build_model(input_shape):\n",
    "    # Use the supernet as feature extractor\n",
    "    supernet.trainable = False\n",
    "    inputs = tfk.Input(shape=input_shape)\n",
    "    x = supernet(inputs)\n",
    "    \n",
    "    # Adding GAP + dropout for better feature extraction and less overfitting\n",
    "    x = tfkl.GlobalAveragePooling2D(name='GlobalPooling')(x)\n",
    "    x = tfkl.Dropout(0.3, seed=seed, name='GlobalPoolingDropout')(x)\n",
    "    \n",
    "    # Two dense layers with ReLU activation just like the real VGG16\n",
    "    x = tfkl.Dense(units=512, name='Classifier', kernel_initializer=tfk.initializers.GlorotUniform(seed), activation='relu')(x)\n",
    "    x = tfkl.Dense(units=512, name='Classifier2', kernel_initializer=tfk.initializers.GlorotUniform(seed), activation='relu')(x)\n",
    "\n",
    "    # Output layer for classification\n",
    "    outputs = tfkl.Dense(units=14, activation='softmax', kernel_initializer=tfk.initializers.GlorotUniform(seed), name='Output')(x)\n",
    "    \n",
    "    # Connect input and output through the Model class\n",
    "    model = tfk.Model(inputs=inputs, outputs=outputs, name='model')\n",
    "    \n",
    "    # Compiling the model\n",
    "    model.compile(loss=tfk.losses.CategoricalCrossentropy(), optimizer=tfk.optimizers.Adam(), metrics='accuracy')\n",
    "    \n",
    "    # Return the model\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2YiUz5vfmM4D",
    "outputId": "dd1ad601-e8ed-4508-bcb0-3d7dacd2d5fb"
   },
   "outputs": [],
   "source": [
    "input_shape = (256, 256, 3)\n",
    "epochs = 50\n",
    "\n",
    "model = build_model(input_shape)\n",
    "\n",
    "# Double checking the structure. In first training, no layer is trainable\n",
    "model.get_layer('vgg16').trainable = False\n",
    "for i, layer in enumerate(model.get_layer('vgg16').layers):\n",
    "       print(i, layer.name, layer.trainable)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1ukKt972nI2l",
    "outputId": "cc90cf21-5baa-4659-a2ca-02b14bad7071"
   },
   "outputs": [],
   "source": [
    "# Suppressing WARNING logs during the training\n",
    "tf.get_logger().setLevel('ERROR') \n",
    "\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    train_gen,\n",
    "    epochs = epochs,\n",
    "    validation_data = valid_gen,\n",
    "    callbacks = [tfk.callbacks.EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True),\n",
    "                 tfk.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5)]\n",
    ").history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DIg7wqHinFF4"
   },
   "outputs": [],
   "source": [
    "model.save('transfer_learning_vgg16_no_ft')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IflyHWpWKcp3"
   },
   "source": [
    "## FINE TUNING THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fU-kFcPaIP0D",
    "outputId": "43771dd2-442c-428f-9799-0b105387001a"
   },
   "outputs": [],
   "source": [
    "# Make all layers trainable at first\n",
    "model.get_layer('vgg16').trainable = True\n",
    "\n",
    "# Freeze all layers except last block\n",
    "for i, layer in enumerate(model.get_layer('vgg16').layers[:14]):\n",
    "      layer.trainable = False\n",
    "        \n",
    "# Recap of the network\n",
    "for i, layer in enumerate(model.get_layer('vgg16').layers):\n",
    "       print(i, layer.name, layer.trainable)\n",
    "        \n",
    "model.summary()\n",
    "\n",
    "# Recompiling the model and setting a low learning rate\n",
    "model.compile(loss=tfk.losses.CategoricalCrossentropy(), optimizer=tfk.optimizers.Adam(1e-4), metrics='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BN_9Y-_TJAda",
    "outputId": "3f737b18-eca8-4f04-873d-74df3dff24a7"
   },
   "outputs": [],
   "source": [
    "# Re-train the model\n",
    "history = model.fit(\n",
    "    train_gen,\n",
    "    epochs = epochs,\n",
    "    validation_data = valid_gen,\n",
    "    callbacks = [tfk.callbacks.EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True),\n",
    "                 tfk.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5)]\n",
    ").history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Wq8eWglAJGtr"
   },
   "outputs": [],
   "source": [
    "model.save('vgg16_tl_ft')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 471
    },
    "id": "35o-oUwAbwhf",
    "outputId": "33fd926a-b819-4f33-a06f-ccbe5c5f946d"
   },
   "outputs": [],
   "source": [
    "# Plotting loss and accuracy of the training\n",
    "plt.figure(figsize=(15,5))\n",
    "plt.plot(history['loss'], label='Training', alpha=.3, color='#ff7f0e', linestyle='--')\n",
    "plt.plot(history['val_loss'], label='Validation', alpha=.8, color='#ff7f0e')\n",
    "plt.legend(loc='upper left')\n",
    "plt.title('Categorical Crossentropy')\n",
    "plt.grid(alpha=.3)\n",
    "\n",
    "plt.figure(figsize=(15,5))\n",
    "plt.plot(history['accuracy'], label='Training', alpha=.8, color='#ff7f0e', linestyle='--')\n",
    "plt.plot(history['val_accuracy'], label='Validation', alpha=.8, color='#ff7f0e')\n",
    "plt.legend(loc='upper left')\n",
    "plt.title('Accuracy')\n",
    "plt.grid(alpha=.3)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GmGBkRX7bwhf",
    "outputId": "61faef5c-fc61-41a7-fc91-b9dd49cf5532"
   },
   "outputs": [],
   "source": [
    "# Testing the model on the test set \n",
    "model.evaluate(test_gen)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "mYQC7co3nee1",
    "rcDOdGc4nnGm"
   ],
   "name": "Copia di sub_084_transfer_learning_con_GAB_FINE_TUNING.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}