Automatic Mode Imputation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled3.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "MC2q4G2wP9Jp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 448
        },
        "outputId": "7f0159d2-39b4-40f0-a0fa-b7c2cad2955c"
      },
      "source": [
        "!apt-get update\r\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\r\n",
        "!wget -q htt  https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!tar xf spark-2.3.1-bin-hadoop2.7.tgz\r\n",
        "!pip install -q findspark\r\n",
        "\r\n",
        "import os\r\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\r\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.3.1-bin-hadoop2.7\"\r\n",
        "\r\n",
        "import findspark\r\n",
        "findspark.init()\r\n",
        "from pyspark.sql import SparkSession\r\n",
        "spark = SparkSession.builder.getOrCreate()\r\n",
        "spark"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Ign:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:6 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Hit:7 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:8 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:11 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Fetched 252 kB in 2s (122 kB/s)\n",
            "Reading package lists... Done\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://acaf6fed03c0:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v2.3.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7f4e087ada20>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "na3dsBE35N_r",
        "outputId": "8e316875-67ce-480e-fa0e-741a5f0909bc"
      },
      "source": [
        "df=spark.read.format(\"csv\").option(\"header\",\"true\").option(\"inferschema\",\"true\").load(\"/content/Josna.csv\");\r\n",
        "df.show()"
      ],
      "execution_count": 179,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---+------+---------+----+-------+\n",
            "| ID|  City|   Degree| Age|Married|\n",
            "+---+------+---------+----+-------+\n",
            "|  1|Lisbon|     null|  25|      0|\n",
            "|  2|Berlin| Bachelor|  25|      1|\n",
            "|  3|Lisbon|     null|  30|      1|\n",
            "|  4|Lisbon| Bachelor|  30|      1|\n",
            "|  5|Berlin| Bachelor|  18|      1|\n",
            "|  6|Lisbon| Bachelor|null|      0|\n",
            "|  7|Berlin|  Masters|  30|      1|\n",
            "|  8|Berlin|No Degree|null|      0|\n",
            "|  9|Berlin|  Masters|  25|      1|\n",
            "| 10|Madrid|  Masters|  25|      1|\n",
            "+---+------+---------+----+-------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GBv27qX051rn",
        "outputId": "a0745643-d9ce-4490-b564-6cd524306cdb"
      },
      "source": [
        "df.printSchema()"
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- ID: integer (nullable = true)\n",
            " |-- City: string (nullable = true)\n",
            " |-- Degree: string (nullable = true)\n",
            " |-- Age: integer (nullable = true)\n",
            " |-- Married: integer (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Td16kqQ-V5Mj"
      },
      "source": [
        "# **Get the Number of Null values for Individual Columns**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XH1P3Ns2Ym22"
      },
      "source": [
        "ms=df.columns;\r\n",
        "gs=[];\r\n",
        "for i in ms:\r\n",
        "  gs.append(df.filter(df[i].isNull()).count());"
      ],
      "execution_count": 155,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kh9ZkI56ZOzj",
        "outputId": "92093859-77d8-4800-ddfa-165f4a552189"
      },
      "source": [
        "gs"
      ],
      "execution_count": 156,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0, 0, 2, 2, 0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TcZvKLWs8BnX"
      },
      "source": [
        "from pyspark.sql.functions import *"
      ],
      "execution_count": 157,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Qdn5ohyV_5k"
      },
      "source": [
        "# **Separate all the Number Type & String type columns**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8r_R3oIu69DA"
      },
      "source": [
        "ms=df.columns;\r\n",
        "input_columns_Number_Type=[];\r\n",
        "input_columns_String_Type=[];\r\n",
        "for i in ms:\r\n",
        "  if str(df.schema[i].dataType) in ('IntegerType','DoubleType'):\r\n",
        "    input_columns_Number_Type.append(i);\r\n",
        "  elif str(df.schema[i].dataType)=='StringType':\r\n",
        "    input_columns_String_Type.append(i);"
      ],
      "execution_count": 158,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xGKDTK9v8r0L",
        "outputId": "a868b38e-7a3a-42d6-db6a-affc95a15666"
      },
      "source": [
        "print(input_columns_Number_Type)\r\n",
        "print(input_columns_String_Type)"
      ],
      "execution_count": 159,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['ID', 'Age', 'Married']\n",
            "['City', 'Degree']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "StAtBCerWIJH"
      },
      "source": [
        "# **Automatic Mean Imputation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b58QcsI58tiV"
      },
      "source": [
        "def fill_using_mean(dfa,cola):\r\n",
        "  for c in dfa.columns:\r\n",
        "    if c in cola:\r\n",
        "      ms=dfa.select(mean(c).alias(\"Ammao\")).collect()[0].asDict()['Ammao'];\r\n",
        "      dfa=dfa.na.fill({c:ms});\r\n",
        "  return dfa;"
      ],
      "execution_count": 160,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B9wbpFmsWYYE"
      },
      "source": [
        "dfa=fill_using_mean(df,input_columns_Number_Type)"
      ],
      "execution_count": 161,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3mFULBL-bbfs",
        "outputId": "0cb84b07-c1a8-4028-f689-3e1cf90ff95f"
      },
      "source": [
        "dfa.show()"
      ],
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---+------+---------+---+-------+\n",
            "| ID|  City|   Degree|Age|Married|\n",
            "+---+------+---------+---+-------+\n",
            "|  1|Lisbon|     null| 25|      0|\n",
            "|  2|Berlin| Bachelor| 25|      1|\n",
            "|  3|Lisbon|     null| 30|      1|\n",
            "|  4|Lisbon| Bachelor| 30|      1|\n",
            "|  5|Berlin| Bachelor| 18|      1|\n",
            "|  6|Lisbon| Bachelor| 26|      0|\n",
            "|  7|Berlin|  Masters| 30|      1|\n",
            "|  8|Berlin|No Degree| 26|      0|\n",
            "|  9|Berlin|  Masters| 25|      1|\n",
            "| 10|Madrid|  Masters| 25|      1|\n",
            "+---+------+---------+---+-------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yur0nmUyWNbb"
      },
      "source": [
        "# **Automatic Mode Imputation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A3bksP6EmY7k"
      },
      "source": [
        "def fill_using_mode(dfa,cola):\r\n",
        "  for c in dfa.columns:\r\n",
        "    if c in cola:\r\n",
        "      msao=dfa.groupBy(c).agg(count(c).alias('ms')).orderBy('ms',ascending=False).collect()[0].asDict()[c];\r\n",
        "      dfa=dfa.na.fill({c:msao});\r\n",
        "  return dfa;"
      ],
      "execution_count": 163,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgosFnP0sSrJ"
      },
      "source": [
        "dfa=fill_using_mode(dfa,input_columns_String_Type)"
      ],
      "execution_count": 164,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RrrI1uaWsi_N",
        "outputId": "8d2d1622-4cdb-4967-961f-fa9f1495de4c"
      },
      "source": [
        "dfa.show()"
      ],
      "execution_count": 165,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---+------+---------+---+-------+\n",
            "| ID|  City|   Degree|Age|Married|\n",
            "+---+------+---------+---+-------+\n",
            "|  1|Lisbon| Bachelor| 25|      0|\n",
            "|  2|Berlin| Bachelor| 25|      1|\n",
            "|  3|Lisbon| Bachelor| 30|      1|\n",
            "|  4|Lisbon| Bachelor| 30|      1|\n",
            "|  5|Berlin| Bachelor| 18|      1|\n",
            "|  6|Lisbon| Bachelor| 26|      0|\n",
            "|  7|Berlin|  Masters| 30|      1|\n",
            "|  8|Berlin|No Degree| 26|      0|\n",
            "|  9|Berlin|  Masters| 25|      1|\n",
            "| 10|Madrid|  Masters| 25|      1|\n",
            "+---+------+---------+---+-------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}