nb04b_train_rcan-scale3_replica_original_continuacion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Debido a un corte de luz, el entrenamiento se detuvo a las 25 épocas\n",
    "\n",
    "Este notebook reinicia el entrenamiento a partir del último modelo guardado, ajustando el valor\n",
    "inicial del learning rate y el número de épocas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "irish-earthquake",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Physical GPUs, 1 Logical GPUs\n",
      "----------------------------------\n",
      "Num images for training: 800\n",
      "Num images for validating: 100\n",
      "----------------------------------\n",
      "Smallest image in train dataset: 1116\n",
      "Smallest image in validation dataset: 1356\n",
      "----------------------------------\n",
      "Biggest image in train dataset: 2040\n",
      "Biggest image in validation dataset: 2040\n",
      "----------------------------------\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import os\n",
    "\n",
    "# GPU memory limit\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        # Currently, memory growth needs to be the same across GPUs\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "            logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        # Memory growth must be set before GPUs have been initialized\n",
    "        print(e)\n",
    "\n",
    "# Import own modules\n",
    "import lib.custom_callbacks as callbacks\n",
    "import lib.PrepareDataset as dt\n",
    "import lib.constants as ctes\n",
    "import lib.RCAN as RCAN\n",
    "\n",
    "######################################################################################\n",
    "# DEFINE ARGUMENTS\n",
    "\n",
    "# Load datasets from CSV files\n",
    "TRAINING_DATASETS = [pd.read_csv('data/0_csvs/DIV2K_train_HR.csv')]\n",
    "VALIDATION_DATASETS = [pd.read_csv('data/0_csvs/DIV2K_valid_HR.csv')]\n",
    "\n",
    "# Define dataset parameters\n",
    "TRAINING_COLOR_MODE = ctes.COLOR_MODE.RGB\n",
    "TRAINING_SCALE = 3\n",
    "TRAINING_BATCH_SIZE = 16\n",
    "TRAINING_PATCH_SIZE = 48\n",
    "TRAINING_DATA_AUG = True\n",
    "TRAINING_SHUFFLE = True\n",
    "TRAINING_REPEAT = 200 # ----------------- Según la implementación en TF 1.13, se valida cada 10.000 training steps. Al ser 800 imágenes de entrenamiento y un batch de 16, hay 50 steps por época\n",
    "                      #                   Hay que realizar 10.000 iteraciones antes de validar. Estas son equivalentes a 200 épocas de entrenamiento sin validación, osea, repetir 200 veces el dataset\n",
    "TRAINING_NORMALIZE = False\n",
    "\n",
    "VALIDATION_COLOR_MODE = TRAINING_COLOR_MODE\n",
    "VALIDATION_SCALE = TRAINING_SCALE\n",
    "VALIDATION_BATCH_SIZE = 1\n",
    "VALIDATION_PATCH_SIZE = None\n",
    "VALIDATION_DATA_AUG = False\n",
    "VALIDATION_SHUFFLE = False\n",
    "VALIDATION_REPEAT = None\n",
    "VALIDATION_NORMALIZE = False\n",
    "\n",
    "FILENAME = 'nb04b_train_rcan-scale3_replica_original_continuacion'\n",
    "\n",
    "# Get model info from file name\n",
    "info_from_script_name = FILENAME.split('_') \n",
    "\n",
    "model_structure_name = info_from_script_name[2].upper() \n",
    "\n",
    "model_name = info_from_script_name[3:]            \n",
    "model_name = ('_'.join(['model'] + model_name)).split('.')[0]\n",
    "model_name_for_metrics = model_structure_name + '_' + model_name         \n",
    "\n",
    "BASE_MODEL_PATH = 'TRAINED_MODELS_NEW/'\n",
    "BASE_METRIC_PATH = 'TRAINING_METRIC_EVOLUTIONS_NEW/'\n",
    "\n",
    "if not os.path.exists(BASE_MODEL_PATH):\n",
    "    os.mkdir(BASE_MODEL_PATH)\n",
    "if not os.path.exists(BASE_METRIC_PATH):\n",
    "    os.mkdir(BASE_METRIC_PATH)\n",
    "\n",
    "save_path = BASE_MODEL_PATH + model_structure_name + '/'\n",
    "save_metrics_path = BASE_METRIC_PATH\n",
    "\n",
    "if not os.path.exists(save_path):\n",
    "    os.mkdir(save_path)\n",
    "\n",
    "# Load datasets as PrepareDataset objects\n",
    "train = dt.PrepareDataset(dataframes=TRAINING_DATASETS, \n",
    "                          channel_mode=TRAINING_COLOR_MODE, \n",
    "                          scale=TRAINING_SCALE,\n",
    "                          batch_size=TRAINING_BATCH_SIZE, \n",
    "                          patch_size=TRAINING_PATCH_SIZE,\n",
    "                          data_augmentation=TRAINING_DATA_AUG, \n",
    "                          shuffle=TRAINING_SHUFFLE, \n",
    "                          repeat=TRAINING_REPEAT,\n",
    "                          normalize=TRAINING_NORMALIZE)\n",
    "\n",
    "val = dt.PrepareDataset(dataframes=VALIDATION_DATASETS, \n",
    "                        channel_mode=VALIDATION_COLOR_MODE, \n",
    "                        scale=VALIDATION_SCALE, \n",
    "                        batch_size=VALIDATION_BATCH_SIZE,\n",
    "                        patch_size=VALIDATION_PATCH_SIZE, \n",
    "                        data_augmentation=VALIDATION_DATA_AUG,\n",
    "                        shuffle=VALIDATION_SHUFFLE,\n",
    "                        repeat=VALIDATION_REPEAT,\n",
    "                        normalize=VALIDATION_NORMALIZE)\n",
    "\n",
    "print('----------------------------------')\n",
    "print('Num images for training:', sum(x.shape[0] for x in TRAINING_DATASETS))\n",
    "print('Num images for validating:', sum(x.shape[0] for x in VALIDATION_DATASETS))\n",
    "print('----------------------------------')\n",
    "\n",
    "def aux_fn(datasets, condition_fn):\n",
    "    return condition_fn(condition_fn(cv2.imread(row.path).shape[1:-1]) for i in range(len(datasets)) for row in datasets[i].itertuples())\n",
    "\n",
    "print('Smallest image in train dataset:', aux_fn(TRAINING_DATASETS, min))\n",
    "smallest_validation_size = aux_fn(VALIDATION_DATASETS, min)\n",
    "print('Smallest image in validation dataset:', smallest_validation_size)\n",
    "print('----------------------------------')\n",
    "print('Biggest image in train dataset:', aux_fn(TRAINING_DATASETS, max))\n",
    "biggest_validation_image = aux_fn(VALIDATION_DATASETS, max)\n",
    "print('Biggest image in validation dataset:', biggest_validation_image)\n",
    "print('----------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "therapeutic-cemetery",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Num Conv layers: 814\n",
      "   Num ReLU layers: 200\n",
      "   Num Add layers: 211\n",
      "Total layers: 1225\n",
      "Num layers: 1631\n",
      "Num parameters: 15629283\n",
      "Num trainable variables: 1628\n"
     ]
    }
   ],
   "source": [
    "# Define model\n",
    "NUM_RESIDUAL_GROUPS = 10\n",
    "NUM_RESIDUAL_BLOCKS = 20\n",
    "NUM_FEATURES = 64\n",
    "KERNEL_SIZE = 3\n",
    "REDUCTION = 16\n",
    "NUM_IMAGE_CHANNELS = TRAINING_COLOR_MODE # 3\n",
    "SCALE = TRAINING_SCALE\n",
    "NORMALIZATION = False\n",
    "\n",
    "model = RCAN.get_RCAN(NUM_RESIDUAL_GROUPS,\n",
    "                        NUM_RESIDUAL_BLOCKS,\n",
    "                        NUM_FEATURES,\n",
    "                        KERNEL_SIZE,\n",
    "                        REDUCTION,\n",
    "                        NUM_IMAGE_CHANNELS,\n",
    "                        SCALE,\n",
    "                        NORMALIZATION)\n",
    "\n",
    "\n",
    "# Load weights from initial training\n",
    "# Best ssim was reached on epoch 24\n",
    "model.load_weights('TRAINED_MODELS_NEW/RCAN-SCALE3/model_replica_original_best_ssim.h5')\n",
    "\n",
    "# model.summary()\n",
    "types = {\n",
    "    'Conv': tf.keras.layers.Conv2D,\n",
    "    'ReLU': tf.keras.layers.ReLU,\n",
    "    'Add': tf.keras.layers.Add \n",
    "}\n",
    "\n",
    "total_num = 0\n",
    "\n",
    "for k in types:\n",
    "    num = len(list(filter(lambda x: type(x) == types[k], model.layers)))\n",
    "    print('   Num', k, 'layers:', num)\n",
    "    total_num += num\n",
    "print('Total layers:', total_num)\n",
    "\n",
    "print('Num layers:', len(model.layers))\n",
    "print('Num parameters:', model.count_params())\n",
    "\n",
    "print('Num trainable variables:', sum(len(l.trainable_variables) for l in model.layers))\n",
    "\n",
    "# Save model structure to JSON file\n",
    "if os.path.exists(save_path + model_name + '.json'):\n",
    "    os.remove(save_path + model_name + '.json')\n",
    "with open(save_path + model_name + '.json', 'w') as json_file:\n",
    "    json_file.write(model.to_json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "rough-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starts training\n",
      "Epoch 1/80\n",
      "10000/10000 [==============================] - 5349s 535ms/step - loss: 4.7768 - psnr: 34.3633 - ssim: 0.8803 - mse: 90.5763 - mae: 4.2510 - sobel_loss: 1891.0520 - val_loss: 4.6178 - val_psnr: 31.1934 - val_ssim: 0.8776 - val_mse: 81.8000 - val_mae: 4.0700 - val_sobel_loss: 1744.2400\n",
      "Epoch 2/80\n",
      "10000/10000 [==============================] - 5314s 531ms/step - loss: 4.7537 - psnr: 34.4229 - ssim: 0.8807 - mse: 89.8511 - mae: 4.2312 - sobel_loss: 1873.8337 - val_loss: 4.6168 - val_psnr: 31.1900 - val_ssim: 0.8777 - val_mse: 81.8000 - val_mae: 4.0700 - val_sobel_loss: 1744.6976\n",
      "Epoch 3/80\n",
      "10000/10000 [==============================] - 5316s 532ms/step - loss: 4.7579 - psnr: 34.4304 - ssim: 0.8808 - mse: 90.0542 - mae: 4.2358 - sobel_loss: 1877.5717 - val_loss: 4.6234 - val_psnr: 31.1871 - val_ssim: 0.8777 - val_mse: 82.0000 - val_mae: 4.0700 - val_sobel_loss: 1747.3477\n",
      "Epoch 4/80\n",
      "10000/10000 [==============================] - 5320s 532ms/step - loss: 4.7591 - psnr: 34.4387 - ssim: 0.8807 - mse: 89.9807 - mae: 4.2338 - sobel_loss: 1874.6318 - val_loss: 4.6216 - val_psnr: 31.1862 - val_ssim: 0.8778 - val_mse: 82.0100 - val_mae: 4.0700 - val_sobel_loss: 1748.3026\n",
      "Epoch 5/80\n",
      "10000/10000 [==============================] - 5331s 533ms/step - loss: 4.7687 - psnr: 34.4137 - ssim: 0.8806 - mse: 90.4216 - mae: 4.2482 - sobel_loss: 1885.6132 - val_loss: 4.6176 - val_psnr: 31.1883 - val_ssim: 0.8779 - val_mse: 81.8200 - val_mae: 4.0700 - val_sobel_loss: 1744.3628\n",
      "Epoch 6/80\n",
      "10000/10000 [==============================] - 5325s 533ms/step - loss: 4.7606 - psnr: 34.4171 - ssim: 0.8809 - mse: 90.0647 - mae: 4.2351 - sobel_loss: 1877.2720 - val_loss: 4.6117 - val_psnr: 31.2003 - val_ssim: 0.8779 - val_mse: 81.6800 - val_mae: 4.0700 - val_sobel_loss: 1741.2632\n",
      "Epoch 7/80\n",
      "10000/10000 [==============================] - 5323s 532ms/step - loss: 4.7554 - psnr: 34.4412 - ssim: 0.8809 - mse: 89.9937 - mae: 4.2323 - sobel_loss: 1873.9570 - val_loss: 4.6231 - val_psnr: 31.1931 - val_ssim: 0.8778 - val_mse: 81.7000 - val_mae: 4.0700 - val_sobel_loss: 1741.5565\n",
      "Epoch 8/80\n",
      "10000/10000 [==============================] - 5324s 532ms/step - loss: 4.7503 - psnr: 34.4179 - ssim: 0.8808 - mse: 89.4517 - mae: 4.2297 - sobel_loss: 1863.7268 - val_loss: 4.6223 - val_psnr: 31.1830 - val_ssim: 0.8781 - val_mse: 81.8800 - val_mae: 4.0700 - val_sobel_loss: 1744.6338\n",
      "Epoch 9/80\n",
      "10000/10000 [==============================] - 5327s 533ms/step - loss: 4.7457 - psnr: 34.4449 - ssim: 0.8809 - mse: 89.3859 - mae: 4.2242 - sobel_loss: 1860.7397 - val_loss: 4.6121 - val_psnr: 31.2021 - val_ssim: 0.8780 - val_mse: 81.6200 - val_mae: 4.0700 - val_sobel_loss: 1740.2080\n",
      "Epoch 10/80\n",
      "10000/10000 [==============================] - 5329s 533ms/step - loss: 4.7448 - psnr: 34.4275 - ssim: 0.8812 - mse: 89.3238 - mae: 4.2225 - sobel_loss: 1859.2570 - val_loss: 4.6159 - val_psnr: 31.1983 - val_ssim: 0.8779 - val_mse: 81.7800 - val_mae: 4.0700 - val_sobel_loss: 1743.1975\n",
      "Epoch 11/80\n",
      "10000/10000 [==============================] - 5332s 533ms/step - loss: 4.7398 - psnr: 34.4564 - ssim: 0.8811 - mse: 89.1420 - mae: 4.2150 - sobel_loss: 1854.2710 - val_loss: 4.6307 - val_psnr: 31.2056 - val_ssim: 0.8781 - val_mse: 81.6300 - val_mae: 4.0800 - val_sobel_loss: 1738.4165\n",
      "Epoch 12/80\n",
      "10000/10000 [==============================] - 5334s 533ms/step - loss: 4.7471 - psnr: 34.4394 - ssim: 0.8815 - mse: 89.5639 - mae: 4.2231 - sobel_loss: 1863.7778 - val_loss: 4.6053 - val_psnr: 31.2073 - val_ssim: 0.8782 - val_mse: 81.6200 - val_mae: 4.0600 - val_sobel_loss: 1739.3643\n",
      "Epoch 13/80\n",
      "10000/10000 [==============================] - 5333s 533ms/step - loss: 4.7355 - psnr: 34.4457 - ssim: 0.8813 - mse: 88.8088 - mae: 4.2120 - sobel_loss: 1848.8951 - val_loss: 4.6192 - val_psnr: 31.1952 - val_ssim: 0.8780 - val_mse: 81.7700 - val_mae: 4.0700 - val_sobel_loss: 1743.3423\n",
      "Epoch 14/80\n",
      "10000/10000 [==============================] - 5336s 534ms/step - loss: 4.7354 - psnr: 34.4483 - ssim: 0.8814 - mse: 89.0289 - mae: 4.2131 - sobel_loss: 1851.0996 - val_loss: 4.6271 - val_psnr: 31.1910 - val_ssim: 0.8779 - val_mse: 81.8300 - val_mae: 4.0700 - val_sobel_loss: 1742.9624\n",
      "Epoch 15/80\n",
      "10000/10000 [==============================] - 5346s 535ms/step - loss: 4.7450 - psnr: 34.4405 - ssim: 0.8814 - mse: 89.5074 - mae: 4.2184 - sobel_loss: 1859.4041 - val_loss: 4.6115 - val_psnr: 31.1973 - val_ssim: 0.8778 - val_mse: 81.7200 - val_mae: 4.0700 - val_sobel_loss: 1740.7744\n",
      "Epoch 16/80\n",
      "10000/10000 [==============================] - 5332s 533ms/step - loss: 4.7425 - psnr: 34.4508 - ssim: 0.8811 - mse: 89.0447 - mae: 4.2199 - sobel_loss: 1851.8951 - val_loss: 4.6087 - val_psnr: 31.2139 - val_ssim: 0.8780 - val_mse: 81.5500 - val_mae: 4.0700 - val_sobel_loss: 1738.4604\n",
      "Epoch 17/80\n",
      "10000/10000 [==============================] - 5336s 534ms/step - loss: 4.7343 - psnr: 34.4647 - ssim: 0.8815 - mse: 89.0290 - mae: 4.2081 - sobel_loss: 1850.2754 - val_loss: 4.6035 - val_psnr: 31.2156 - val_ssim: 0.8783 - val_mse: 81.4900 - val_mae: 4.0700 - val_sobel_loss: 1736.3303\n",
      "Epoch 18/80\n",
      "10000/10000 [==============================] - 5335s 533ms/step - loss: 4.7273 - psnr: 34.4699 - ssim: 0.8817 - mse: 88.6378 - mae: 4.2039 - sobel_loss: 1844.0248 - val_loss: 4.6257 - val_psnr: 31.1952 - val_ssim: 0.8780 - val_mse: 81.8200 - val_mae: 4.0700 - val_sobel_loss: 1742.4113\n",
      "Epoch 19/80\n",
      "10000/10000 [==============================] - 5336s 534ms/step - loss: 4.7327 - psnr: 34.4701 - ssim: 0.8817 - mse: 89.0693 - mae: 4.2056 - sobel_loss: 1850.4938 - val_loss: 4.6102 - val_psnr: 31.2125 - val_ssim: 0.8781 - val_mse: 81.4200 - val_mae: 4.0700 - val_sobel_loss: 1735.5028\n",
      "Epoch 20/80\n",
      "10000/10000 [==============================] - 5337s 534ms/step - loss: 4.7091 - psnr: 34.5092 - ssim: 0.8821 - mse: 88.1065 - mae: 4.1899 - sobel_loss: 1829.5978 - val_loss: 4.6211 - val_psnr: 31.1853 - val_ssim: 0.8781 - val_mse: 81.9000 - val_mae: 4.0700 - val_sobel_loss: 1745.1606\n",
      "Epoch 0020 --- Learning rate reduced to: 2.499999936844688e-05\n",
      "Epoch 21/80\n",
      "10000/10000 [==============================] - 5344s 534ms/step - loss: 4.6917 - psnr: 34.5393 - ssim: 0.8827 - mse: 87.6069 - mae: 4.1707 - sobel_loss: 1817.6458 - val_loss: 4.5916 - val_psnr: 31.2384 - val_ssim: 0.8787 - val_mse: 81.0800 - val_mae: 4.0500 - val_sobel_loss: 1726.5686\n",
      "Epoch 22/80\n",
      "10000/10000 [==============================] - 5348s 535ms/step - loss: 4.6897 - psnr: 34.5412 - ssim: 0.8829 - mse: 87.6196 - mae: 4.1709 - sobel_loss: 1818.4528 - val_loss: 4.5901 - val_psnr: 31.2448 - val_ssim: 0.8787 - val_mse: 81.0600 - val_mae: 4.0500 - val_sobel_loss: 1725.3949\n",
      "Epoch 23/80\n",
      "10000/10000 [==============================] - 5347s 535ms/step - loss: 4.6868 - psnr: 34.5560 - ssim: 0.8828 - mse: 87.2739 - mae: 4.1629 - sobel_loss: 1809.9139 - val_loss: 4.5976 - val_psnr: 31.2366 - val_ssim: 0.8786 - val_mse: 81.0600 - val_mae: 4.0500 - val_sobel_loss: 1724.9819\n",
      "Epoch 24/80\n",
      "10000/10000 [==============================] - 5357s 536ms/step - loss: 4.6872 - psnr: 34.5516 - ssim: 0.8828 - mse: 87.2416 - mae: 4.1684 - sobel_loss: 1810.4956 - val_loss: 4.5887 - val_psnr: 31.2398 - val_ssim: 0.8787 - val_mse: 81.1300 - val_mae: 4.0600 - val_sobel_loss: 1726.7479\n",
      "Epoch 25/80\n",
      "10000/10000 [==============================] - 5365s 536ms/step - loss: 4.6852 - psnr: 34.5505 - ssim: 0.8828 - mse: 87.4097 - mae: 4.1657 - sobel_loss: 1812.2354 - val_loss: 4.5913 - val_psnr: 31.2457 - val_ssim: 0.8787 - val_mse: 80.9700 - val_mae: 4.0400 - val_sobel_loss: 1723.3966\n",
      "Epoch 26/80\n",
      "10000/10000 [==============================] - 5375s 538ms/step - loss: 4.6805 - psnr: 34.5624 - ssim: 0.8828 - mse: 86.9941 - mae: 4.1608 - sobel_loss: 1803.5682 - val_loss: 4.5885 - val_psnr: 31.2495 - val_ssim: 0.8789 - val_mse: 80.9100 - val_mae: 4.0400 - val_sobel_loss: 1722.6481\n",
      "Epoch 27/80\n",
      "10000/10000 [==============================] - 5387s 539ms/step - loss: 4.6694 - psnr: 34.5747 - ssim: 0.8831 - mse: 86.6897 - mae: 4.1522 - sobel_loss: 1797.5148 - val_loss: 4.5893 - val_psnr: 31.2461 - val_ssim: 0.8787 - val_mse: 81.0000 - val_mae: 4.0600 - val_sobel_loss: 1724.0280\n",
      "Epoch 28/80\n",
      "10000/10000 [==============================] - 5413s 541ms/step - loss: 4.6951 - psnr: 34.5586 - ssim: 0.8826 - mse: 87.6707 - mae: 4.1790 - sobel_loss: 1816.7676 - val_loss: 4.5900 - val_psnr: 31.2366 - val_ssim: 0.8787 - val_mse: 81.0800 - val_mae: 4.0600 - val_sobel_loss: 1726.3883\n",
      "Epoch 29/80\n",
      "10000/10000 [==============================] - 5443s 544ms/step - loss: 4.6786 - psnr: 34.5594 - ssim: 0.8831 - mse: 86.8881 - mae: 4.1598 - sobel_loss: 1801.1357 - val_loss: 4.5934 - val_psnr: 31.2343 - val_ssim: 0.8787 - val_mse: 81.1200 - val_mae: 4.0400 - val_sobel_loss: 1726.6077\n",
      "Epoch 30/80\n",
      "10000/10000 [==============================] - 5457s 546ms/step - loss: 4.6805 - psnr: 34.5542 - ssim: 0.8829 - mse: 86.9856 - mae: 4.1644 - sobel_loss: 1802.1328 - val_loss: 4.5848 - val_psnr: 31.2495 - val_ssim: 0.8788 - val_mse: 80.9000 - val_mae: 4.0400 - val_sobel_loss: 1721.4606\n",
      "Epoch 31/80\n",
      "10000/10000 [==============================] - 5456s 546ms/step - loss: 4.6868 - psnr: 34.5732 - ssim: 0.8829 - mse: 87.3940 - mae: 4.1623 - sobel_loss: 1811.0450 - val_loss: 4.5883 - val_psnr: 31.2449 - val_ssim: 0.8788 - val_mse: 81.1100 - val_mae: 4.0500 - val_sobel_loss: 1725.8679\n",
      "Epoch 32/80\n",
      "10000/10000 [==============================] - 5471s 547ms/step - loss: 4.6785 - psnr: 34.5649 - ssim: 0.8828 - mse: 86.8586 - mae: 4.1617 - sobel_loss: 1798.1285 - val_loss: 4.5902 - val_psnr: 31.2428 - val_ssim: 0.8788 - val_mse: 81.0200 - val_mae: 4.0500 - val_sobel_loss: 1724.3400\n",
      "Epoch 33/80\n",
      "10000/10000 [==============================] - 5482s 548ms/step - loss: 4.6749 - psnr: 34.5797 - ssim: 0.8833 - mse: 86.9708 - mae: 4.1540 - sobel_loss: 1802.3658 - val_loss: 4.5866 - val_psnr: 31.2525 - val_ssim: 0.8788 - val_mse: 80.9000 - val_mae: 4.0400 - val_sobel_loss: 1722.1539\n",
      "Epoch 34/80\n",
      "10000/10000 [==============================] - 5493s 549ms/step - loss: 4.6884 - psnr: 34.5240 - ssim: 0.8827 - mse: 87.0561 - mae: 4.1683 - sobel_loss: 1802.8444 - val_loss: 4.5870 - val_psnr: 31.2395 - val_ssim: 0.8788 - val_mse: 81.0100 - val_mae: 4.0600 - val_sobel_loss: 1724.3861\n",
      "Epoch 35/80\n",
      "10000/10000 [==============================] - 5505s 551ms/step - loss: 4.6714 - psnr: 34.5807 - ssim: 0.8834 - mse: 86.6600 - mae: 4.1494 - sobel_loss: 1795.8239 - val_loss: 4.5877 - val_psnr: 31.2437 - val_ssim: 0.8788 - val_mse: 81.0600 - val_mae: 4.0500 - val_sobel_loss: 1725.5640\n",
      "Epoch 36/80\n",
      "10000/10000 [==============================] - 5523s 552ms/step - loss: 4.6666 - psnr: 34.5688 - ssim: 0.8833 - mse: 86.4571 - mae: 4.1465 - sobel_loss: 1790.8772 - val_loss: 4.5897 - val_psnr: 31.2435 - val_ssim: 0.8789 - val_mse: 81.1300 - val_mae: 4.0600 - val_sobel_loss: 1725.8882\n",
      "Epoch 37/80\n",
      "10000/10000 [==============================] - 5547s 555ms/step - loss: 4.6499 - psnr: 34.6116 - ssim: 0.8838 - mse: 86.0464 - mae: 4.1278 - sobel_loss: 1782.9836 - val_loss: 4.5922 - val_psnr: 31.2426 - val_ssim: 0.8788 - val_mse: 81.0100 - val_mae: 4.0400 - val_sobel_loss: 1723.9827\n",
      "Epoch 38/80\n",
      "10000/10000 [==============================] - 5422s 542ms/step - loss: 4.6620 - psnr: 34.6012 - ssim: 0.8834 - mse: 86.4248 - mae: 4.1379 - sobel_loss: 1789.5981 - val_loss: 4.5831 - val_psnr: 31.2548 - val_ssim: 0.8790 - val_mse: 80.7900 - val_mae: 4.0500 - val_sobel_loss: 1719.4491\n",
      "Epoch 39/80\n",
      "10000/10000 [==============================] - 5341s 534ms/step - loss: 4.6810 - psnr: 34.5458 - ssim: 0.8830 - mse: 86.8820 - mae: 4.1606 - sobel_loss: 1798.2224 - val_loss: 4.5869 - val_psnr: 31.2529 - val_ssim: 0.8787 - val_mse: 80.7600 - val_mae: 4.0500 - val_sobel_loss: 1719.1644\n",
      "Epoch 40/80\n",
      "10000/10000 [==============================] - 5346s 535ms/step - loss: 4.6698 - psnr: 34.5815 - ssim: 0.8832 - mse: 86.4986 - mae: 4.1498 - sobel_loss: 1790.8322 - val_loss: 4.5881 - val_psnr: 31.2464 - val_ssim: 0.8788 - val_mse: 81.0200 - val_mae: 4.0400 - val_sobel_loss: 1724.3428\n",
      "Epoch 0040 --- Learning rate reduced to: 1.249999968422344e-05\n",
      "Epoch 41/80\n",
      "10000/10000 [==============================] - 5353s 535ms/step - loss: 4.6563 - psnr: 34.6198 - ssim: 0.8839 - mse: 86.4497 - mae: 4.1365 - sobel_loss: 1789.3762 - val_loss: 4.5759 - val_psnr: 31.2636 - val_ssim: 0.8792 - val_mse: 80.7400 - val_mae: 4.0300 - val_sobel_loss: 1717.6587\n",
      "Epoch 42/80\n",
      "10000/10000 [==============================] - 5359s 536ms/step - loss: 4.6392 - psnr: 34.6356 - ssim: 0.8840 - mse: 85.6049 - mae: 4.1204 - sobel_loss: 1770.4113 - val_loss: 4.5761 - val_psnr: 31.2643 - val_ssim: 0.8791 - val_mse: 80.7300 - val_mae: 4.0400 - val_sobel_loss: 1716.8660\n",
      "Epoch 43/80\n",
      "10000/10000 [==============================] - 5365s 537ms/step - loss: 4.6348 - psnr: 34.6372 - ssim: 0.8841 - mse: 85.4513 - mae: 4.1213 - sobel_loss: 1767.9728 - val_loss: 4.5790 - val_psnr: 31.2588 - val_ssim: 0.8792 - val_mse: 80.9200 - val_mae: 4.0300 - val_sobel_loss: 1720.1456\n",
      "Epoch 44/80\n",
      "10000/10000 [==============================] - 5378s 538ms/step - loss: 4.6440 - psnr: 34.6161 - ssim: 0.8837 - mse: 85.6603 - mae: 4.1235 - sobel_loss: 1770.9261 - val_loss: 4.5768 - val_psnr: 31.2602 - val_ssim: 0.8790 - val_mse: 80.7700 - val_mae: 4.0400 - val_sobel_loss: 1718.1506\n",
      "Epoch 45/80\n",
      "10000/10000 [==============================] - 5393s 539ms/step - loss: 4.6470 - psnr: 34.6142 - ssim: 0.8839 - mse: 85.9489 - mae: 4.1282 - sobel_loss: 1777.9288 - val_loss: 4.5770 - val_psnr: 31.2624 - val_ssim: 0.8792 - val_mse: 80.7700 - val_mae: 4.0300 - val_sobel_loss: 1718.6620\n",
      "Epoch 46/80\n",
      "10000/10000 [==============================] - 5425s 543ms/step - loss: 4.6447 - psnr: 34.6461 - ssim: 0.8841 - mse: 85.8751 - mae: 4.1247 - sobel_loss: 1775.4650 - val_loss: 4.5791 - val_psnr: 31.2605 - val_ssim: 0.8792 - val_mse: 80.8100 - val_mae: 4.0400 - val_sobel_loss: 1718.6948\n",
      "Epoch 47/80\n",
      "10000/10000 [==============================] - 5458s 546ms/step - loss: 4.6400 - psnr: 34.6183 - ssim: 0.8840 - mse: 85.5641 - mae: 4.1231 - sobel_loss: 1769.7675 - val_loss: 4.5750 - val_psnr: 31.2693 - val_ssim: 0.8791 - val_mse: 80.6800 - val_mae: 4.0300 - val_sobel_loss: 1715.8618\n",
      "Epoch 48/80\n",
      "10000/10000 [==============================] - 5497s 550ms/step - loss: 4.6429 - psnr: 34.6208 - ssim: 0.8838 - mse: 85.6593 - mae: 4.1265 - sobel_loss: 1770.3820 - val_loss: 4.5788 - val_psnr: 31.2632 - val_ssim: 0.8792 - val_mse: 80.7500 - val_mae: 4.0300 - val_sobel_loss: 1717.1882\n",
      "Epoch 49/80\n",
      "10000/10000 [==============================] - 5548s 555ms/step - loss: 4.6457 - psnr: 34.6215 - ssim: 0.8841 - mse: 85.8981 - mae: 4.1283 - sobel_loss: 1776.6237 - val_loss: 4.5783 - val_psnr: 31.2621 - val_ssim: 0.8792 - val_mse: 80.7900 - val_mae: 4.0300 - val_sobel_loss: 1718.8348\n",
      "Epoch 50/80\n",
      "10000/10000 [==============================] - 5586s 559ms/step - loss: 4.6455 - psnr: 34.6198 - ssim: 0.8840 - mse: 85.6900 - mae: 4.1309 - sobel_loss: 1772.2253 - val_loss: 4.5781 - val_psnr: 31.2610 - val_ssim: 0.8791 - val_mse: 80.8200 - val_mae: 4.0400 - val_sobel_loss: 1720.2947\n",
      "Epoch 51/80\n",
      "10000/10000 [==============================] - 5627s 563ms/step - loss: 4.6392 - psnr: 34.6640 - ssim: 0.8841 - mse: 85.7593 - mae: 4.1240 - sobel_loss: 1772.4152 - val_loss: 4.5807 - val_psnr: 31.2561 - val_ssim: 0.8792 - val_mse: 80.9000 - val_mae: 4.0400 - val_sobel_loss: 1720.9028\n",
      "Epoch 52/80\n",
      "10000/10000 [==============================] - 5657s 566ms/step - loss: 4.6377 - psnr: 34.6257 - ssim: 0.8842 - mse: 85.5479 - mae: 4.1240 - sobel_loss: 1769.4430 - val_loss: 4.5762 - val_psnr: 31.2609 - val_ssim: 0.8792 - val_mse: 80.8600 - val_mae: 4.0400 - val_sobel_loss: 1720.3633\n",
      "Epoch 53/80\n",
      "10000/10000 [==============================] - 5666s 567ms/step - loss: 4.6470 - psnr: 34.6205 - ssim: 0.8840 - mse: 85.8914 - mae: 4.1298 - sobel_loss: 1775.2848 - val_loss: 4.5757 - val_psnr: 31.2625 - val_ssim: 0.8792 - val_mse: 80.8100 - val_mae: 4.0400 - val_sobel_loss: 1719.1713\n",
      "Epoch 54/80\n",
      "10000/10000 [==============================] - 5676s 568ms/step - loss: 4.6542 - psnr: 34.5927 - ssim: 0.8838 - mse: 85.9621 - mae: 4.1388 - sobel_loss: 1778.2208 - val_loss: 4.5771 - val_psnr: 31.2605 - val_ssim: 0.8791 - val_mse: 80.8400 - val_mae: 4.0400 - val_sobel_loss: 1719.7766\n",
      "Epoch 55/80\n",
      "10000/10000 [==============================] - 5632s 563ms/step - loss: 4.6390 - psnr: 34.6272 - ssim: 0.8841 - mse: 85.5272 - mae: 4.1227 - sobel_loss: 1767.2390 - val_loss: 4.5753 - val_psnr: 31.2611 - val_ssim: 0.8791 - val_mse: 80.7800 - val_mae: 4.0400 - val_sobel_loss: 1719.2153\n",
      "Epoch 56/80\n",
      "10000/10000 [==============================] - 5417s 542ms/step - loss: 4.6384 - psnr: 34.6273 - ssim: 0.8842 - mse: 85.5556 - mae: 4.1211 - sobel_loss: 1768.9781 - val_loss: 4.5764 - val_psnr: 31.2617 - val_ssim: 0.8792 - val_mse: 80.7900 - val_mae: 4.0400 - val_sobel_loss: 1718.0261\n",
      "Epoch 57/80\n",
      "10000/10000 [==============================] - 5447s 545ms/step - loss: 4.6152 - psnr: 34.6742 - ssim: 0.8846 - mse: 84.7493 - mae: 4.0935 - sobel_loss: 1752.0367 - val_loss: 4.5771 - val_psnr: 31.2613 - val_ssim: 0.8792 - val_mse: 80.8000 - val_mae: 4.0400 - val_sobel_loss: 1718.7908\n",
      "Epoch 58/80\n",
      "10000/10000 [==============================] - 5478s 548ms/step - loss: 4.6291 - psnr: 34.6429 - ssim: 0.8844 - mse: 85.0872 - mae: 4.1154 - sobel_loss: 1758.9464 - val_loss: 4.5784 - val_psnr: 31.2626 - val_ssim: 0.8792 - val_mse: 80.7300 - val_mae: 4.0400 - val_sobel_loss: 1717.5233\n",
      "Epoch 59/80\n",
      "10000/10000 [==============================] - 5515s 552ms/step - loss: 4.6348 - psnr: 34.6141 - ssim: 0.8843 - mse: 85.2330 - mae: 4.1139 - sobel_loss: 1762.9890 - val_loss: 4.5759 - val_psnr: 31.2564 - val_ssim: 0.8792 - val_mse: 80.8600 - val_mae: 4.0300 - val_sobel_loss: 1720.6676\n",
      "Epoch 60/80\n",
      "10000/10000 [==============================] - 5555s 555ms/step - loss: 4.6267 - psnr: 34.6379 - ssim: 0.8843 - mse: 84.9720 - mae: 4.1060 - sobel_loss: 1757.0560 - val_loss: 4.5792 - val_psnr: 31.2579 - val_ssim: 0.8791 - val_mse: 80.8200 - val_mae: 4.0400 - val_sobel_loss: 1718.7961\n",
      "Epoch 0060 --- Learning rate reduced to: 6.24999984211172e-06\n",
      "Epoch 61/80\n",
      "10000/10000 [==============================] - 5594s 559ms/step - loss: 4.6330 - psnr: 34.6427 - ssim: 0.8843 - mse: 85.2969 - mae: 4.1152 - sobel_loss: 1761.3680 - val_loss: 4.5762 - val_psnr: 31.2646 - val_ssim: 0.8792 - val_mse: 80.7900 - val_mae: 4.0400 - val_sobel_loss: 1717.6505\n",
      "Epoch 62/80\n",
      "10000/10000 [==============================] - 5655s 566ms/step - loss: 4.6305 - psnr: 34.6538 - ssim: 0.8844 - mse: 85.3697 - mae: 4.1162 - sobel_loss: 1763.0120 - val_loss: 4.5735 - val_psnr: 31.2656 - val_ssim: 0.8792 - val_mse: 80.7300 - val_mae: 4.0400 - val_sobel_loss: 1717.6904\n",
      "Epoch 63/80\n",
      "10000/10000 [==============================] - 5379s 538ms/step - loss: 4.6283 - psnr: 34.6407 - ssim: 0.8846 - mse: 85.1834 - mae: 4.1097 - sobel_loss: 1760.4886 - val_loss: 4.5738 - val_psnr: 31.2662 - val_ssim: 0.8792 - val_mse: 80.7200 - val_mae: 4.0400 - val_sobel_loss: 1716.3885\n",
      "Epoch 64/80\n",
      "10000/10000 [==============================] - 5356s 536ms/step - loss: 4.6247 - psnr: 34.6561 - ssim: 0.8845 - mse: 85.0224 - mae: 4.1030 - sobel_loss: 1757.2184 - val_loss: 4.5760 - val_psnr: 31.2658 - val_ssim: 0.8792 - val_mse: 80.7600 - val_mae: 4.0400 - val_sobel_loss: 1717.2532\n",
      "Epoch 65/80\n",
      "10000/10000 [==============================] - 5343s 534ms/step - loss: 4.6176 - psnr: 34.6568 - ssim: 0.8848 - mse: 84.8567 - mae: 4.1029 - sobel_loss: 1752.1053 - val_loss: 4.5748 - val_psnr: 31.2635 - val_ssim: 0.8792 - val_mse: 80.7700 - val_mae: 4.0400 - val_sobel_loss: 1717.2714\n",
      "Epoch 66/80\n",
      "10000/10000 [==============================] - 5352s 535ms/step - loss: 4.6323 - psnr: 34.6328 - ssim: 0.8845 - mse: 85.4187 - mae: 4.1173 - sobel_loss: 1764.5824 - val_loss: 4.5725 - val_psnr: 31.2671 - val_ssim: 0.8792 - val_mse: 80.7400 - val_mae: 4.0400 - val_sobel_loss: 1717.2666\n",
      "Epoch 67/80\n",
      "10000/10000 [==============================] - 5361s 536ms/step - loss: 4.6290 - psnr: 34.6565 - ssim: 0.8845 - mse: 85.2263 - mae: 4.1144 - sobel_loss: 1759.9976 - val_loss: 4.5743 - val_psnr: 31.2637 - val_ssim: 0.8792 - val_mse: 80.8100 - val_mae: 4.0400 - val_sobel_loss: 1718.7472\n",
      "Epoch 68/80\n",
      "10000/10000 [==============================] - 5370s 537ms/step - loss: 4.6341 - psnr: 34.6285 - ssim: 0.8844 - mse: 85.3822 - mae: 4.1184 - sobel_loss: 1763.9652 - val_loss: 4.5743 - val_psnr: 31.2673 - val_ssim: 0.8793 - val_mse: 80.7400 - val_mae: 4.0400 - val_sobel_loss: 1716.2985\n",
      "Epoch 69/80\n",
      "10000/10000 [==============================] - 5385s 538ms/step - loss: 4.6316 - psnr: 34.6393 - ssim: 0.8844 - mse: 85.2578 - mae: 4.1177 - sobel_loss: 1760.9596 - val_loss: 4.5733 - val_psnr: 31.2671 - val_ssim: 0.8792 - val_mse: 80.7100 - val_mae: 4.0400 - val_sobel_loss: 1715.8721\n",
      "Epoch 70/80\n",
      "10000/10000 [==============================] - 5409s 541ms/step - loss: 4.6240 - psnr: 34.6444 - ssim: 0.8845 - mse: 84.9595 - mae: 4.1085 - sobel_loss: 1754.4728 - val_loss: 4.5750 - val_psnr: 31.2628 - val_ssim: 0.8792 - val_mse: 80.7800 - val_mae: 4.0400 - val_sobel_loss: 1717.5835\n",
      "Epoch 71/80\n",
      "10000/10000 [==============================] - 5437s 544ms/step - loss: 4.6101 - psnr: 34.6763 - ssim: 0.8850 - mse: 84.5369 - mae: 4.0899 - sobel_loss: 1747.0114 - val_loss: 4.5752 - val_psnr: 31.2624 - val_ssim: 0.8792 - val_mse: 80.8300 - val_mae: 4.0400 - val_sobel_loss: 1719.2837\n",
      "Epoch 72/80\n",
      "10000/10000 [==============================] - 5407s 541ms/step - loss: 4.6153 - psnr: 34.6567 - ssim: 0.8848 - mse: 84.6263 - mae: 4.1020 - sobel_loss: 1747.3840 - val_loss: 4.5754 - val_psnr: 31.2629 - val_ssim: 0.8792 - val_mse: 80.8100 - val_mae: 4.0400 - val_sobel_loss: 1718.4229\n",
      "Epoch 73/80\n",
      "10000/10000 [==============================] - 5352s 535ms/step - loss: 4.6258 - psnr: 34.6421 - ssim: 0.8847 - mse: 85.1469 - mae: 4.1108 - sobel_loss: 1758.7050 - val_loss: 4.5752 - val_psnr: 31.2647 - val_ssim: 0.8793 - val_mse: 80.7800 - val_mae: 4.0400 - val_sobel_loss: 1717.2329\n",
      "Epoch 74/80\n",
      "10000/10000 [==============================] - 5359s 536ms/step - loss: 4.6098 - psnr: 34.6772 - ssim: 0.8848 - mse: 84.4802 - mae: 4.0924 - sobel_loss: 1745.5508 - val_loss: 4.5739 - val_psnr: 31.2667 - val_ssim: 0.8792 - val_mse: 80.7200 - val_mae: 4.0400 - val_sobel_loss: 1716.8420\n",
      "Epoch 75/80\n",
      "10000/10000 [==============================] - 5373s 537ms/step - loss: 4.6193 - psnr: 34.6607 - ssim: 0.8846 - mse: 84.9257 - mae: 4.0999 - sobel_loss: 1754.7754 - val_loss: 4.5714 - val_psnr: 31.2680 - val_ssim: 0.8793 - val_mse: 80.7000 - val_mae: 4.0300 - val_sobel_loss: 1715.9062\n",
      "Epoch 76/80\n",
      "10000/10000 [==============================] - 5387s 539ms/step - loss: 4.6186 - psnr: 34.6649 - ssim: 0.8847 - mse: 85.0355 - mae: 4.0993 - sobel_loss: 1755.3722 - val_loss: 4.5739 - val_psnr: 31.2639 - val_ssim: 0.8792 - val_mse: 80.7700 - val_mae: 4.0400 - val_sobel_loss: 1718.0958\n",
      "Epoch 77/80\n",
      "10000/10000 [==============================] - 5410s 541ms/step - loss: 4.6214 - psnr: 34.6594 - ssim: 0.8847 - mse: 85.0401 - mae: 4.1004 - sobel_loss: 1754.8090 - val_loss: 4.5725 - val_psnr: 31.2672 - val_ssim: 0.8793 - val_mse: 80.7200 - val_mae: 4.0400 - val_sobel_loss: 1717.0944\n",
      "Epoch 78/80\n",
      "10000/10000 [==============================] - 5439s 544ms/step - loss: 4.6174 - psnr: 34.6653 - ssim: 0.8847 - mse: 84.9065 - mae: 4.1023 - sobel_loss: 1753.7844 - val_loss: 4.5735 - val_psnr: 31.2618 - val_ssim: 0.8793 - val_mse: 80.8100 - val_mae: 4.0400 - val_sobel_loss: 1718.7590\n",
      "Epoch 79/80\n",
      "10000/10000 [==============================] - 5496s 550ms/step - loss: 4.6133 - psnr: 34.6681 - ssim: 0.8848 - mse: 84.6919 - mae: 4.0965 - sobel_loss: 1747.1514 - val_loss: 4.5747 - val_psnr: 31.2626 - val_ssim: 0.8792 - val_mse: 80.8200 - val_mae: 4.0400 - val_sobel_loss: 1718.0797\n",
      "Epoch 80/80\n",
      "10000/10000 [==============================] - 5504s 550ms/step - loss: 4.6018 - psnr: 34.6880 - ssim: 0.8850 - mse: 84.1460 - mae: 4.0833 - sobel_loss: 1738.0726 - val_loss: 4.5740 - val_psnr: 31.2628 - val_ssim: 0.8793 - val_mse: 80.7700 - val_mae: 4.0400 - val_sobel_loss: 1718.6260\n",
      "Ends training\n"
     ]
    }
   ],
   "source": [
    "# Metrics for model evaluation\n",
    "METRICS = [\n",
    "    ctes.METRIC_FUNCTIONS.PSNR,\n",
    "    ctes.METRIC_FUNCTIONS.SSIM,\n",
    "    ctes.METRIC_FUNCTIONS.SSIM_MS, # ---- https://github.com/tensorflow/tensorflow/issues/33840\n",
    "                                   #      MODIFIED ARGUMENTS IN SSIM MULTISCALE (lib.custom_metric_functions.py)\n",
    "    ctes.METRIC_FUNCTIONS.MSE,\n",
    "    ctes.METRIC_FUNCTIONS.MAE,\n",
    "    ctes.METRIC_FUNCTIONS.SOBEL\n",
    "]\n",
    "\n",
    "# Metrics for saving model's checkpoints\n",
    "METRICS_CHECKPOINTS = [\n",
    "    (ctes.METRICS_ALL.PSNR, ctes.METRICS_ALL.VAL_PSNR, 'max'),\n",
    "    (ctes.METRICS_ALL.SSIM, ctes.METRICS_ALL.VAL_SSIM, 'max'),\n",
    "    # (ctes.METRICS_ALL.SOBEL, ctes.METRICS_ALL.VAL_SOBEL, 'min')\n",
    "]\n",
    "\n",
    "# Define training hyperparameters\n",
    "# Como el entrenamiento se está reanudando a partir de la época 24 y cada 20 épocas el learning rate\n",
    "# se reduce 1/2, aquí tiene como valor inicial 5e-5 y no 1e-4\n",
    "INITIAL_LEARNING_RATE = 0.00005\n",
    "\n",
    "# Quitamos 20 épocas (lo correcto sería quiar 24, pero por evitar cambios en la implementación se dejó así)\n",
    "NUM_EPOCHS = 80 # Nº máximo de iteraciones de entrenamiento: 1.000.000 -- Como antes definimos una época como 10.000 iteraciones (repetir 200 veces el dataset de 800 imágenes dividio en batches de 16)\n",
    "                 # 1.000.000 de iteraciones serán 100 épocas\n",
    "\n",
    "OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=INITIAL_LEARNING_RATE)\n",
    "LOSS_FUNCTION = train.get_loss_function(ctes.LOSS_FUNCTIONS.MAE)\n",
    "\n",
    "# Define callbacks\n",
    "\n",
    "# Se indica en el paper original, y se aplica en la implementación en TF 1.13, que cada 200.000 iteraciones, el learning rate se debe reducir a la mitad\n",
    "# Estableciendo que una época se compone de 200 repeticiones del dataset de entrenamiento para conseguir 10.000 iteraciones, cada 20 'épocas' se deberá reducir a la mitad el learning rate\n",
    "\n",
    "learning_rate_scheduler_callback = tf.keras.callbacks.LearningRateScheduler(callbacks.create_scheduler_function(20, 0.5)) # Cada 20 épocas, multiplicar lr por 0.5\n",
    "\n",
    "# Model checkpoints callbacks\n",
    "checkpoint_callbacks = [tf.keras.callbacks.ModelCheckpoint(save_path + model_name + '_best_' + mtr[0] + '.h5',\n",
    "                                                           monitor=mtr[1],\n",
    "                                                           save_best_only=True,\n",
    "                                                           mode=mtr[2],\n",
    "                                                           save_weights_only=True) for mtr in METRICS_CHECKPOINTS]\n",
    "\n",
    "# Save training metrics evolution callback\n",
    "metrics_evolution_callback = callbacks.Save_Training_Evolution(save_metrics_path + model_name_for_metrics + '_evolution.csv')\n",
    "\n",
    "\n",
    "CBACKS = [learning_rate_scheduler_callback, checkpoint_callbacks, metrics_evolution_callback] \n",
    "\n",
    "\n",
    "# TRAIN\n",
    "model.compile(optimizer=OPTIMIZER,\n",
    "              loss=LOSS_FUNCTION,\n",
    "              # SSIM_MS needs greater images than training patches, so ignore this metric on training\n",
    "              metrics=[train.get_metric_function(x) for x in METRICS if x != ctes.METRIC_FUNCTIONS.SSIM_MS])\n",
    "\n",
    "print('Starts training')\n",
    "model.fit(train.dataset, epochs=NUM_EPOCHS, verbose=1, validation_data=val.dataset, callbacks=CBACKS)\n",
    "print('Ends training')\n",
    "\n",
    "\n",
    "with open('ended_scripts.txt', 'a') as f:\n",
    "    f.write(FILENAME + '\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "language": "python",
   "name": "python38564bit6c4dec2e42734bc298ce0c3bfcfccb75"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}