Ex_02_Narrow_Transformation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ex-02-Narrow Transformation.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_0y0i6xAfXiE"
      },
      "source": [
        "### Sanjoy Biswas"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1dNjOuZITmvJ"
      },
      "source": [
        "#### Setup Hadoop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LnTS_GV50yl3"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://downloads.apache.org/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vYgGaoSTG8tq",
        "outputId": "dba8ea70-b4ea-44ab-c499-07f718018ef4"
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.7/dist-packages (3.1.2)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.7/dist-packages (from pyspark) (0.10.9)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "44SmOaWfT4lv"
      },
      "source": [
        "#### Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67C8z90tO0pP"
      },
      "source": [
        "from pyspark import SparkContext, SparkConf"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17KqWp8pQrLI"
      },
      "source": [
        "conf = SparkConf().setAppName('Youtube').setMaster('local')\n",
        "sc = SparkContext(conf = conf)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XXufDIO8cYzU"
      },
      "source": [
        "#### Apply Map Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6st918ToQ-rf",
        "outputId": "509ec88b-889e-4a11-fb58-25c87eb52a53"
      },
      "source": [
        "num = sc.parallelize([2,5,7,9,8])\n",
        "num.collect()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 5, 7, 9, 8]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2YH4MvqKQ-uo",
        "outputId": "acd0a50b-0f5e-4992-a387-f0dd65f9b98b"
      },
      "source": [
        "num.map(lambda a : a*2).collect()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[4, 10, 14, 18, 16]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9TYihNIVQ-w_",
        "outputId": "cab189d7-4039-4e01-c80c-ff50afe2b604"
      },
      "source": [
        "num.map(lambda a : pow(a,3)).collect()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[8, 125, 343, 729, 512]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zf08sQ77Q-33",
        "outputId": "f24e6e35-95e0-49e0-a9ce-fa5de6be08f1"
      },
      "source": [
        "nm = sc.parallelize(['abc','xyz','mnp'])\n",
        "nm.collect()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['abc', 'xyz', 'mnp']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8OfTxlMlQ-8H",
        "outputId": "83b92ed1-e2c1-4b1c-90b8-6936734cb783"
      },
      "source": [
        "nm.map(lambda a : \"Mr. \"+ a).collect()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Mr. abc', 'Mr. xyz', 'Mr. mnp']"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "332oK7q1egl9"
      },
      "source": [
        "#### Apply FlatMap"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NZyyIXVsQ-_w",
        "outputId": "365399dc-1d2a-4a9a-fcee-df74773c059a"
      },
      "source": [
        "rdd = sc.parallelize([2,3,4])\n",
        "rdd.collect()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 3, 4]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CwewW7RhhPx1",
        "outputId": "cc8a3ac1-1ec0-4d75-ec7d-d184c57649d7"
      },
      "source": [
        "a = range(1,3)\n",
        "for i in a:\n",
        "  print(i)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1\n",
            "2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bibwy_h_hP1D",
        "outputId": "267b14d8-0590-49c3-8e54-5f7fb23493d0"
      },
      "source": [
        "rdd.flatMap(lambda x: range(1,x)).collect()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 1, 2, 1, 2, 3]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUj4kbp_hP37"
      },
      "source": [
        "a = sc.parallelize([1,2,3])"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j69k-QzzhP6M",
        "outputId": "f9113c2e-be50-46ec-eb15-157148ca74fd"
      },
      "source": [
        "a.flatMap(lambda x: (x, x*10,57)).collect()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 10, 57, 2, 20, 57, 3, 30, 57]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y46ZIlYAemQ7"
      },
      "source": [
        "#### Apply Filter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgJP0CBOhP9L",
        "outputId": "26eb6c62-3fcf-40aa-801f-15c97fa672be"
      },
      "source": [
        "num.collect()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 5, 7, 9, 8]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8CHyQqtOhQAj",
        "outputId": "a65799a6-0bb8-4e32-ac1f-d8aadf29dd5b"
      },
      "source": [
        "num.filter(lambda x: x%2==0).collect()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 8]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "InqVJ7lAjwZt",
        "outputId": "31a821c2-49e6-459e-f9cf-8dd343e34b08"
      },
      "source": [
        "nm.collect()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['abc', 'xyz', 'mnp']"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HM1eIgAxjwck",
        "outputId": "f009e0c4-b887-4e83-c048-40b2a30d4e0f"
      },
      "source": [
        "nm.filter(lambda x : \"b\" in x).collect()"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['abc']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jqn2XCm4esJs"
      },
      "source": [
        "#### Apply Union"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YyMQhJDujwhq",
        "outputId": "00a468c3-9a58-48f6-80d2-5824af1ff6b3"
      },
      "source": [
        "num1 = sc.parallelize([2,3,4,5])\n",
        "num1.collect()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 3, 4, 5]"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lw1FJ0Qajwkj",
        "outputId": "25704758-446d-4cc5-bcda-aea3976357f5"
      },
      "source": [
        "num2 = sc.parallelize([6,7,8,9,3,4])\n",
        "num2.collect()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[6, 7, 8, 9, 3, 4]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1u0Cu029jwo6",
        "outputId": "bb8d3597-06d6-43e4-c916-21eb40eba75e"
      },
      "source": [
        "num1.union(num2).collect()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 3, 4, 5, 6, 7, 8, 9, 3, 4]"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kYIbatHExrHc",
        "outputId": "45ee8bae-7219-495c-e519-b6b850f6478e"
      },
      "source": [
        "parallel = sc.parallelize(range(1,10))\n",
        "parallel.collect()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 5, 6, 7, 8, 9]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N1MI6vA6tHmK",
        "outputId": "8f4be9a6-dd23-42a4-ac3e-616ccb852981"
      },
      "source": [
        "parallel.sample(True,.2).collect()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[6]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IrB5E3uatHpY",
        "outputId": "d42c21c7-0b05-4c84-a74e-17752042ec33"
      },
      "source": [
        "parallel.sample(False,.2,seed = 23).collect()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[4, 5]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fPDC4aV7fSkk"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}