fundamentals_DL_init_intuition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Red Neuronal MultiCapa, OOP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Solo en caso de tener Jupyter Themes\n",
    "from jupyterthemes import jtplot\n",
    "jtplot.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_images import get_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST path\n",
    "mnist_path = './mnist_raw/'\n",
    "x_train_num, y_train_num, x_test_num, y_test_num = get_images(mnist_path)\n",
    "\n",
    "x_train = x_train_num[:50000].reshape(50000, -1).astype(np.float32)\n",
    "y_train = y_train_num[:50000].reshape(50000, 1)\n",
    "\n",
    "x_val = x_train_num[50000:].reshape(10000, -1).astype(np.float)\n",
    "y_val = y_train_num[50000:].reshape(10000, 1)\n",
    "\n",
    "x_test = x_test_num.copy().reshape(10000, -1).astype(np.float)\n",
    "y_test = y_test_num.copy().reshape(10000, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalise(x_mean, x_std, x_data):\n",
    "    return (x_data-x_mean)/x_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_t = x_train[:128]\n",
    "x_t = normalise(x_train[:128].mean(),x_train[:128].std(), x_train[:128])\n",
    "x_t.min(), x_t.max(), x_t.mean(), x_t.std()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graficar muestras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_number(image):\n",
    "    plt.figure(figsize=(5,5))\n",
    "    plt.imshow(image.squeeze(), cmap=plt.get_cmap('gray'))\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rnd_idx = np.random.randint(len(y_test))\n",
    "print(f'La imagen muestreada representa un: {y_test[rnd_idx][0]}')\n",
    "plot_number(x_test_num[rnd_idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inicialización de pesos sinápticos con Valores aleatorios, Xavier, y Kaiming He"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normal random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_params(neurons):\n",
    "    weights = {}\n",
    "    for i in range(len(neurons)-1):\n",
    "        weights['L' + str(i+1)] = np.random.randn(neurons[i+1], neurons[i]) * 0.001\n",
    "    \n",
    "    return weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Xavier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_xavier(neurons):\n",
    "    weights = {}\n",
    "    for i in range(len(neurons)-1):\n",
    "        weights['L' + str(i+1)] = np.random.randn(neurons[i+1], neurons[i]) / np.sqrt(neurons[i])\n",
    "    \n",
    "    return weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Kaiming He"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_kaiming(neurons):\n",
    "    weights = {}\n",
    "    for i in range(len(neurons)-1):\n",
    "        weights['L' + str(i+1)] = np.random.randn(neurons[i+1], neurons[i]) / np.sqrt(neurons[i]/2)  \n",
    "    return weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Definir algunas capas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neurons = [500] * 10\n",
    "neurons[0] = 784\n",
    "neurons[-1] = 10\n",
    "\n",
    "print(neurons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inicializar parámetros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = init_params(neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_x = init_xavier(neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_k = init_kaiming(neurons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_k.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pruebas con Random init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acts = {}\n",
    "z = x_t.T\n",
    "for (k, v) in weights.items():\n",
    "    z = v @ z\n",
    "#     z = np.maximum(0, z)\n",
    "    acts[k] = z\n",
    "\n",
    "print(acts.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "\n",
    "for i, (k, v) in enumerate(acts.items()):\n",
    "    print(f'La activación para la capa {k} \\\n",
    "tiene una media de {v.ravel().mean():.4} y una std de {v.ravel().std():.4}')\n",
    "\n",
    "for i, (k, v) in enumerate(acts.items(), 1):\n",
    "    plt.subplot(3, 3, i)\n",
    "    plt.hist(v.ravel(), bins=30, range=[-.003, .003])\n",
    "\n",
    "plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pruebas con Xavier init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acts_x = {}\n",
    "z= x_t.T\n",
    "for (k, v) in weights_x.items():\n",
    "    z = v @ z\n",
    "    z = np.maximum(0, z)\n",
    "#     z = np.tanh(z)\n",
    "#     z = 1/(1+np.exp(-z))\n",
    "    acts_x[k] = z\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "\n",
    "for (k, v) in acts_x.items():\n",
    "    print(f'La activación para la capa {k} \\\n",
    "tiene una media de {v.ravel().mean():.4} y una std de {v.ravel().std():.4}')\n",
    "\n",
    "for i, (k, v) in enumerate(acts_x.items(), 1):\n",
    "    plt.subplot(3, 3, i)\n",
    "    plt.hist(v.ravel(), bins=30, range=[-1, 1])\n",
    "    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pruebas con Kaiming He init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acts_k = {}\n",
    "z= x_t.T\n",
    "for (k, v) in weights_k.items():\n",
    "    z = v @ z\n",
    "    z = np.maximum(0, z) \n",
    "    acts_k[k] = z\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "for (k, v) in acts_k.items():\n",
    "    print(f'La activación para la capa {k} \\\n",
    "tiene una media de {v.ravel().mean():.4} y una std de {v.ravel().std():.4}')\n",
    "\n",
    "for i, (k, v) in enumerate(acts_k.items(), 1):\n",
    "    plt.subplot(3, 3, i)\n",
    "    plt.hist(v.ravel(), bins=20, range=[-3, 3])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}