fundamentos_DL_NN_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Red Neuronal (2 capas), desde cero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar librerías"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# Solamente para usuarios de Jupyter Themes\n",
    "from jupyterthemes import jtplot\n",
    "jtplot.style(grid=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importar MNIST "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"mnist.png\">\n",
    "<caption><left> Fig 1. Muestra MNIST </left></caption>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_images import get_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST path\n",
    "mnist_path = './mnist_raw/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_num, y_train_num, x_test_num, y_test_num = get_images(mnist_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_num.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convertir imágenes en vectores, y a float32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train_num[:50000].reshape(50000, -1).astype(np.float32)/255\n",
    "y_train = y_train_num[:50000].reshape(50000, 1)\n",
    "\n",
    "x_val = x_train_num[50000:].reshape(10000, -1).astype(np.float)/255\n",
    "y_val = y_train_num[50000:].reshape(10000, 1)\n",
    "\n",
    "x_test = x_test_num.copy().reshape(10000, -1).astype(np.float)/255\n",
    "y_test = y_test_num.copy().reshape(10000, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x_train.shape) #shape (50000, 784)\n",
    "print(y_train.shape) #shape (50000, 1)\n",
    "\n",
    "print(x_val.shape)\n",
    "print(y_val.shape)\n",
    "\n",
    "print(x_test.shape)\n",
    "print(y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graficar algunas imágenes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_number(image):\n",
    "    plt.imshow(image.squeeze(), cmap=plt.get_cmap('gray'))\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rnd_idx = np.random.randint(len(y_test))\n",
    "print(f'La imagen muestreada representa un: {y_test[rnd_idx]}')\n",
    "plot_number(x_test_num[rnd_idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ecuaciones para nuestro modelo\n",
    "\n",
    "\n",
    "$$z^1 = W^1 X + b^1$$\n",
    "\n",
    "$$a^1 = ReLU(z^1) $$\n",
    "\n",
    "$$z^2 = W^2 a^1 + b^2$$\n",
    "\n",
    "$$\\hat{y} = \\frac{e^{z^{2_k}}}{\\sum_j{e^{z_j}}}$$\n",
    "\n",
    "\n",
    "$$ \\mathcal{L}(\\hat{y}^{i}, y^{i}) =  - y^{i}  \\ln(\\hat{y}^{i}) = -\\ln(\\hat{y}^i)$$\n",
    "\n",
    "\n",
    "$$ \\mathcal{J}(w, b) =  \\frac{1}{num\\_samples} \\sum_{i=1}^{num\\_samples}-\\ln(\\hat{y}^{i})$$\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Red Neuronal\n",
    "### Dos capas 200-10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Crear 'Mini-batches'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_minibatches(mb_size, x, y, shuffle = True):\n",
    "    '''\n",
    "    x  #muestras, 784\n",
    "    y #muestras, 1\n",
    "    '''\n",
    "    assert x.shape[0] == y.shape[0], 'Error en cantidad de muestras'\n",
    "    total_data = x.shape[0]\n",
    "    if shuffle: \n",
    "        idxs = np.arange(total_data)\n",
    "        np.random.shuffle(idxs)\n",
    "        x = x[idxs]\n",
    "        y = y[idxs]\n",
    "        \n",
    "    return ((x[i:i+mb_size], y[i:i+mb_size]) for i in range(0, total_data, mb_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Init parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_parameters(input_size, neurons):\n",
    "    \n",
    "    '''\n",
    "    input_size -> elementos de entrada, 784\n",
    "    neurons -> list [200, 10] con cantidad de neuronas en cada capa\n",
    "    '''\n",
    "    \n",
    "    W1 = np.random.randn(neurons[0], input_size) * 0.001\n",
    "    b1 = np.zeros((neurons[0], 1))\n",
    "    \n",
    "    W2 = np.random.randn(neurons[1], neurons[0]) * 0.001\n",
    "    b2 = np.zeros((neurons[1], 1))\n",
    "    \n",
    "    return {'W1': W1, 'b1':b1, 'W2':W2, 'b2':b2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = init_parameters(28*28, [200, 10])\n",
    "print(parameters['W1'].shape)\n",
    "print(parameters['W2'].shape)\n",
    "print(parameters['b2'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(x):\n",
    "    return np.maximum(0, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scores(x, parameters, activation_fcn):\n",
    "    '''\n",
    "    x tiene la forma (#pixeles, num samples)\n",
    "    '''\n",
    "    z1 = parameters['W1'] @ x + parameters['b1']\n",
    "    a1 = activation_fcn(z1) # devuel fcn. de activa.\n",
    "    z2 = parameters['W2'] @ a1 + parameters['b2']\n",
    "    \n",
    "    return z2, z1, a1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores,z1, a1 = scores(x_train[:64].T, parameters, relu) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train[:64].T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    exp_scores = np.exp(x)\n",
    "    sum_exp_scores = np.sum(exp_scores, axis=0)\n",
    "    probs = exp_scores/sum_exp_scores\n",
    "    return probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def x_entropy(scores, y, batch_size=64):\n",
    "    probs = softmax(scores)\n",
    "    y_hat = probs[y.squeeze(), np.arange(batch_size)]\n",
    "    cost = np.sum(-np.log(y_hat)) / batch_size\n",
    "    \n",
    "    return probs, cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward(probs, x, y, z1, a1, scores, parameters, batch_size=64):\n",
    "    grads = {}\n",
    "    probs[y.squeeze(), np.arange(batch_size)] -= 1 # y-hat - y\n",
    "    dz2 = probs.copy()\n",
    "    \n",
    "    dW2 = dz2 @ a1.T / batch_size\n",
    "    db2 = np.sum(dz2, axis =1, keepdims=True) / batch_size\n",
    "    da1 = parameters['W2'].T @ dz2\n",
    "    \n",
    "    dz1 = da1.copy()\n",
    "    dz1[z1 <= 0 ] =0\n",
    "    \n",
    "    dW1 = dz1 @ x \n",
    "    db1 = np.sum(dz1, axis=1, keepdims=True) \n",
    "    \n",
    "    assert parameters['W1'].shape == dW1.shape, 'W1 no igual forma'\n",
    "    assert parameters['W2'].shape == dW2.shape, 'W2 no igual forma'\n",
    "    assert parameters['b1'].shape == db1.shape, 'b1 no igual forma'\n",
    "    assert parameters['b2'].shape == db2.shape, 'b2 no igual forma'\n",
    "    \n",
    "    grads = {'w1':dW1,  'b1':db1, 'W2':dW2, 'b2':db2}\n",
    "    \n",
    "    return grads\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat, cost = x_entropy(scores, y_train[:64])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grads = backward(y_hat, x_train[:64], y_train[:64],z1, a1, scores, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(x_data, y_data, mb_size=64):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for i, (x, y) in enumerate(create_minibatches(mb_size, x_data, y_data)):\n",
    "        scores2, z1, a1 = scores(x.T, parameters, relu)\n",
    "        y_hat, cost = x_entropy(scores2, y, batch_size=len(x))\n",
    "        \n",
    "        correct += np.sum(np.argmax(y_hat, axis=0) == y.squeeze())\n",
    "        total += y_hat.shape[1]\n",
    "    return correct/total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def train(epochs, parameters, mb_size=64, learning_rate = 1e-3):\n",
    "    for epoch in range(epochs):\n",
    "        for i, (x, y) in enumerate(create_minibatches(mb_size, x_train, y_train)):\n",
    "            scores2, z1, a1 = scores(x.T, parameters=parameters, activation_fcn=relu)\n",
    "            y_hat, cost = x_entropy(scores2, y, batch_size=len(x))\n",
    "            grads = backward(y_hat, x, y, z1, a1, scores2, parameters, batch_size=len(x))\n",
    "            \n",
    "            parameters['W1'] = parameters['W1'] - learning_rate*grads['w1']\n",
    "            parameters['b1'] = parameters['b1'] - learning_rate*grads['b1']\n",
    "            parameters['b2'] = parameters['b2'] - learning_rate*grads['b2']\n",
    "            parameters['W2'] = parameters['W2'] - learning_rate*grads['W2']\n",
    "            \n",
    "        print(f'costo es: {cost}, y accuracy: {accuracy(x_val, y_val, mb_size)}')\n",
    "    return parameters\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mb_size = 512\n",
    "learning_rate = 1e-2\n",
    "epochs = 20\n",
    "parameters = train(epochs=epochs, parameters=parameters, mb_size=mb_size, learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy(x_train, y_train, mb_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy(x_test, y_test, mb_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(x):\n",
    "    scores2, _, _ = scores(x, parameters, relu)\n",
    "    return np.argmax(scores2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test[0].reshape(-1, 1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = np.random.randint(len(y_test))\n",
    "plot_number(x_test_num[idx])\n",
    "pred = predict(x_test[idx].reshape(-1, 1))\n",
    "print(f'el valor predicho es: {pred}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}