CIFAR10_Estimator_Conv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import os \n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_path = os.getcwd()\n",
    "dir_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)\n",
    "savedir = \"models/Tensor_CIFAR_Sparse_model/\" + \"run-{}/\".format(now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(savedir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unpickle(file):\n",
    "    import pickle\n",
    "    with open(file, 'rb') as fo:\n",
    "        dict = pickle.load(fo, encoding='bytes')\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_1 = unpickle('datasets/cifar-10/data_batch_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_dataset_folder_path = 'datasets/cifar-10/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(cifar10_1.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cifar10_1[b'data']),len(cifar10_1[b'labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_1[b'batch_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar10_Label_names = unpickle('datasets/cifar-10/batches.meta')\n",
    "cifar10_Label_names[b'label_names']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = []\n",
    "for x in cifar10_Label_names[b'label_names']:\n",
    "    label = x.decode()\n",
    "    label_names.append(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cfar10_batch(cifar10_dataset_folder_path, batch_id):\n",
    "    with open(cifar10_dataset_folder_path + '/data_batch_' + str(batch_id), mode='rb') as file:\n",
    "        # note the encoding type is 'latin1'\n",
    "        batch = pickle.load(file, encoding='latin1')\n",
    "        \n",
    "    features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1)\n",
    "    labels = batch['labels']\n",
    "        \n",
    "    return features, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_stats(cifar10_dataset_folder_path, batch_id, sample_id):\n",
    "    features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_id)\n",
    "    \n",
    "    if not (0 <= sample_id < len(features)):\n",
    "        print('{} samples in batch {}.  {} is out of range.'.format(len(features), batch_id, sample_id))\n",
    "        return None\n",
    "\n",
    "    print('\\nStats of batch #{}:'.format(batch_id))\n",
    "    print('# of Samples: {}\\n'.format(len(features)))\n",
    "    \n",
    "    label_counts = dict(zip(*np.unique(labels, return_counts=True)))\n",
    "    for key, value in label_counts.items():\n",
    "        print('Label Counts of [{}]({}) : {}'.format(key, label_names[key].upper(), value))\n",
    "    \n",
    "    sample_image = features[sample_id]\n",
    "    sample_label = labels[sample_id]\n",
    "    \n",
    "    print('\\nExample of Image {}:'.format(sample_id))\n",
    "    print('Image - Min Value: {} Max Value: {}'.format(sample_image.min(), sample_image.max()))\n",
    "    print('Image - Shape: {}'.format(sample_image.shape))\n",
    "    print('Label - Label Id: {} Name: {}'.format(sample_label, label_names[sample_label]))\n",
    "    \n",
    "    plt.imshow(sample_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "# Explore the dataset\n",
    "batch_id = 3\n",
    "sample_id = 7000\n",
    "display_stats(cifar10_dataset_folder_path, batch_id, sample_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(x):\n",
    "\n",
    "    min_val = np.min(x)\n",
    "    max_val = np.max(x)\n",
    "    x = (x-min_val) / (max_val-min_val)\n",
    "    \n",
    "    x = x.astype('float32')\n",
    "    \n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(x):\n",
    "\n",
    "    encoded = np.zeros((len(x), 10))\n",
    "    \n",
    "    for idx, val in enumerate(x):\n",
    "        encoded[idx][val] = 1\n",
    "    \n",
    "    return encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _preprocess_and_save(normalize, one_hot_encode, features, labels, filename):\n",
    "    features = normalize(features)\n",
    "    labels = one_hot_encode(labels)\n",
    "    \n",
    "    pickle.dump((features, labels), open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode):\n",
    "    n_batches = 5\n",
    "    valid_features = []\n",
    "    valid_labels = []\n",
    "    all_features = []\n",
    "    all_labels = []\n",
    "\n",
    "    for batch_i in range(1, n_batches + 1):\n",
    "        features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_i)\n",
    "        \n",
    "        # find index to be the point as validation data in the whole dataset of the batch (10%)\n",
    "        index_of_validation = int(len(features) * 0.1)\n",
    "\n",
    "        valid_features.extend(features[-index_of_validation:])\n",
    "        valid_labels.extend(labels[-index_of_validation:])\n",
    "        all_features.extend(features[:-index_of_validation])\n",
    "        all_labels.extend(labels[:-index_of_validation])\n",
    "\n",
    "    # preprocess the all stacked validation dataset\n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(valid_features), np.array(valid_labels),\n",
    "                         'datasets/preprocess_validation.p')\n",
    "                                 \n",
    "    _preprocess_and_save(normalize, one_hot_encode,\n",
    "                         np.array(all_features), np.array(all_labels),\n",
    "                         'datasets/preprocess_all.p')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_features, valid_labels = pickle.load(open('datasets/preprocess_validation.p', mode='rb'))\n",
    "print(valid_features.dtype)\n",
    "print(valid_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, train_labels = pickle.load(open('datasets/preprocess_all.p', mode='rb'))\n",
    "print(train_features.dtype)\n",
    "print(train_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_model_fn(features, labels, mode):\n",
    "    \n",
    "    with tf.name_scope('model_input') as scope:\n",
    "        input_layer = tf.reshape(features, [-1, 32, 32, 3], name=\"input\")\n",
    "        \n",
    "    with tf.name_scope('model_conv1') as scope:    \n",
    "        conv1 = tf.layers.conv2d(inputs=input_layer, filters=64, kernel_size=[5, 5], \n",
    "                                 padding=\"same\", activation=tf.nn.relu6, \n",
    "                                 trainable=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=(2,2))\n",
    "        \n",
    "    with tf.name_scope('model_conv2') as scope:\n",
    "        conv2 = tf.layers.conv2d(inputs=pool1, filters=128, kernel_size=[5, 5], \n",
    "                                 padding=\"same\", activation=tf.nn.relu6, \n",
    "                                 trainable=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=(2,2))\n",
    "    \n",
    "    with tf.name_scope('model_dense') as scope:\n",
    "        pool2_flat = tf.reshape(pool2, [-1, 8 * 8 * 128]) \n",
    "    \n",
    "        dense = tf.layers.dense(inputs=pool2_flat, units=1024, \n",
    "                                activation=tf.nn.relu6, \n",
    "                                trainable=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        \n",
    "        dropout = tf.layers.dropout(inputs=dense, rate=0.25, training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        \n",
    "    with tf.name_scope('model_output') as scope:    \n",
    "        logits = tf.layers.dense(inputs=dropout, units=10, trainable=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        # shape should be:[-1, 10]\n",
    "    \n",
    "    predictions = {\n",
    "      \"classes\": tf.argmax(input=logits, axis=1),\n",
    "      \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "    }\n",
    "    \n",
    "    # PREDICT mode\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        \n",
    "        export_outputs = {\n",
    "            'predict_output': tf.estimator.export.PredictOutput(predictions)\n",
    "        }\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, export_outputs=export_outputs)\n",
    "    \n",
    "    # Calculate Loss (for both TRAIN and EVAL modes)\n",
    "    loss = tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=logits)\n",
    "    \n",
    "    \n",
    "    # TRAIN mode\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        g = tf.get_default_graph()\n",
    "        tf.contrib.quantize.create_training_graph(input_graph=g, quant_delay=6000) \n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "        train_op = optimizer.minimize(\n",
    "            loss=loss,\n",
    "            global_step=tf.train.get_global_step())\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "    # EVAL mode\n",
    "    g = tf.get_default_graph()\n",
    "    tf.contrib.quantize.create_eval_graph(input_graph=g)\n",
    "    labels = tf.argmax(labels, axis=1)\n",
    "    eval_metric_ops = {\n",
    "      \"accuracy\": tf.metrics.accuracy(labels=labels, predictions=predictions[\"classes\"])\n",
    "    }\n",
    "    return tf.estimator.EstimatorSpec(\n",
    "      mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#see for options here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto\n",
    "#sess_config = tf.ConfigProto(gpu_options=gpu_options)\n",
    "\n",
    "run_cfg = tf.estimator.RunConfig( \n",
    "    model_dir=savedir,\n",
    "    tf_random_seed=2,\n",
    "    save_summary_steps=2,\n",
    "    #session_config = sess_config,\n",
    "    save_checkpoints_steps=100,\n",
    "    keep_checkpoint_max=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the Estimator\n",
    "cifar10_classifier = tf.estimator.Estimator(\n",
    "        model_fn=cnn_model_fn, config=run_cfg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up logging for predictions\n",
    "tensors_to_log = {\"probabilities\": \"softmax_tensor\"}\n",
    "\n",
    "logging_hook = tf.train.LoggingTensorHook(\n",
    "    tensors=tensors_to_log, every_n_iter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_all_batches(xEpochs):\n",
    "    # Train the model\n",
    "    train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x=train_features,\n",
    "        y=train_labels,\n",
    "        batch_size=32,\n",
    "        num_epochs=xEpochs,\n",
    "        shuffle=False)\n",
    "        \n",
    "    # train one step and display the probabilties\n",
    "    cifar10_classifier.train(\n",
    "        input_fn=train_input_fn,\n",
    "        steps=None, \n",
    "        hooks=[logging_hook]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fit_all_batches(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "defaultGraph = tf.get_default_graph()\n",
    "    \n",
    "eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x=valid_features,\n",
    "    y=valid_labels,\n",
    "    num_epochs=1,\n",
    "    shuffle=False)\n",
    "\n",
    "eval_results = cifar10_classifier.evaluate(input_fn=eval_input_fn,\n",
    "                                          hooks=[logging_hook])\n",
    "            \n",
    "print(eval_results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_image = valid_features[3:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions(nfeature):\n",
    "    \n",
    "    pred_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x=valid_features[nfeature:nfeature+1],\n",
    "        y=valid_labels[nfeature:nfeature+1],\n",
    "        num_epochs=1,\n",
    "        shuffle=False)\n",
    "    \n",
    "    pred_results = cifar10_classifier.predict(\n",
    "            input_fn=pred_input_fn,\n",
    "            hooks=[logging_hook])\n",
    "    \n",
    "    results = list(pred_results)\n",
    "    results = results[0]\n",
    " \n",
    "    y_pred_flat = results['probabilities']\n",
    "    \n",
    "    y_pred = np.column_stack((label_names, y_pred_flat))\n",
    "    print(y_pred)\n",
    "    plt.imshow(valid_features[nfeature])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[print(n.name) for n in defaultGraph.as_graph_def().node]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def serving_input_receiver_fn():\n",
    "\n",
    "    feature_tensor = tf.placeholder(tf.float32, [None, 32,32,3])\n",
    "    return tf.estimator.export.TensorServingInputReceiver(feature_tensor, {'input': feature_tensor})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cifar10_classifier.export_savedmodel(savedir, serving_input_receiver_fn,\n",
    "#    strip_default_attrs=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_receiver_fn_map={\n",
    "    tf.estimator.ModeKeys.PREDICT: serving_input_receiver_fn,\n",
    "    }\n",
    "\n",
    "cifar10_classifier.experimental_export_all_saved_models(\n",
    "    savedir, \n",
    "    input_receiver_fn_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}