WSD_ML_RA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "madu.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "z55CtLfCwr0K",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Testing of various classification algorithms for the word madu.\n",
        "1.  load dataset and create train/test split\n",
        "2. find baseline by training a supervised model\n",
        "3. remove labels and test various unsupervised algorithms"
      ]
    },
    {
      "metadata": {
        "id": "nWVW4u59wHgq",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mfFcR2rWnZcR",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Read input list uploaded into colab fs from local drive. Note: file need to be uploaded everytime colab notebook is started since colab doesn't store the file in disk once session ends."
      ]
    },
    {
      "metadata": {
        "id": "190mR6f8nf8Q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = pd.read_excel('madu-sense1-sense2-context-96sentences.xlsx')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ENxYBm2boXGK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "get plain bag of words from all sentences"
      ]
    },
    {
      "metadata": {
        "id": "ltU6PWuCobAh",
        "colab_type": "code",
        "outputId": "aff8a55a-2988-4274-d8b0-d59889009568",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "words = []\n",
        "for line in df['SENTENCE']:\n",
        "  for w in re.split('\\s+',line):\n",
        "    words.append(w)\n",
        "    \n",
        "print(len(words))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1688\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "IEMIAabMpTR6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Visualize top words used in list of sentences"
      ]
    },
    {
      "metadata": {
        "id": "DWDR6AWmpXe6",
        "colab_type": "code",
        "outputId": "0af98344-28b2-46e4-b392-b7e405db384f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 15964
        }
      },
      "cell_type": "code",
      "source": [
        "def wordcount(inlist):\n",
        "    #count unique words in list\n",
        "    #ignore case\n",
        "    list = [item.lower() for item in inlist]\n",
        "    wc = {}\n",
        "    for item in set(list):\n",
        "        wc[item] = list.count(item)\n",
        "    #wc_sorted = sorted(wc.items(), key=lambda x: x[1], reverse=True)\n",
        "    return wc\n",
        "\n",
        "wordcount(words)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'': 18,\n",
              " '\"bagi': 2,\n",
              " '\"bahan': 1,\n",
              " '\"dari': 1,\n",
              " '\"jadi,': 1,\n",
              " '\"kami': 1,\n",
              " '\"kita': 1,\n",
              " '\"lebah': 1,\n",
              " '\"mahkota': 1,\n",
              " '\"neetar\"': 1,\n",
              " '\"saya': 1,\n",
              " '\"sekiranya': 1,\n",
              " '\"selain': 1,\n",
              " '\"sudah': 1,\n",
              " \"'lamaran'\": 1,\n",
              " '(1989),': 1,\n",
              " '(akhbar': 1,\n",
              " '(hadis': 1,\n",
              " '(kiri)': 1,\n",
              " '(madu)': 1,\n",
              " '(prk)': 1,\n",
              " '(ukm)': 1,\n",
              " ',': 5,\n",
              " '-': 1,\n",
              " '.': 1,\n",
              " '..': 1,\n",
              " '1,001': 2,\n",
              " '11': 1,\n",
              " '20': 1,\n",
              " '2018': 1,\n",
              " '2019.': 1,\n",
              " '25': 1,\n",
              " '29,': 1,\n",
              " '30': 1,\n",
              " '32,': 2,\n",
              " '33,': 1,\n",
              " '36': 1,\n",
              " '50': 1,\n",
              " '8': 1,\n",
              " '95': 1,\n",
              " '97': 1,\n",
              " ':': 1,\n",
              " '?': 1,\n",
              " 'a,': 1,\n",
              " 'abdul': 2,\n",
              " 'abu…': 1,\n",
              " 'ada': 7,\n",
              " 'adakah': 1,\n",
              " 'adalah': 3,\n",
              " 'adam': 1,\n",
              " 'adik': 1,\n",
              " 'adira': 2,\n",
              " 'aduhai': 1,\n",
              " 'afifah,': 1,\n",
              " 'ahli': 3,\n",
              " 'ahmad': 1,\n",
              " 'aida': 2,\n",
              " 'air': 2,\n",
              " 'ajak': 1,\n",
              " 'akan': 7,\n",
              " 'akel': 1,\n",
              " 'aktres': 1,\n",
              " 'aku': 3,\n",
              " 'al-quran': 1,\n",
              " 'alahai': 1,\n",
              " 'alami': 1,\n",
              " 'alasan': 1,\n",
              " 'alimentarius': 1,\n",
              " 'alkohol': 1,\n",
              " 'allah': 1,\n",
              " 'alternatif': 1,\n",
              " 'amalan': 1,\n",
              " 'an-nahl:69': 1,\n",
              " 'anak': 2,\n",
              " 'anak..': 1,\n",
              " 'angkara': 1,\n",
              " 'annapurna': 1,\n",
              " 'antara': 1,\n",
              " 'antikeradangan,': 1,\n",
              " 'antimikrob,': 1,\n",
              " 'antipengoksida': 1,\n",
              " 'anugerah': 1,\n",
              " 'apabila': 4,\n",
              " 'apatah': 1,\n",
              " 'api': 1,\n",
              " 'april': 1,\n",
              " 'arahan': 1,\n",
              " 'arena': 1,\n",
              " 'aroma,': 1,\n",
              " 'asli': 3,\n",
              " 'asmara': 1,\n",
              " 'aspek': 1,\n",
              " 'asyraff': 1,\n",
              " 'atas': 2,\n",
              " 'atau': 15,\n",
              " 'australia': 1,\n",
              " 'awal': 1,\n",
              " 'ayah': 1,\n",
              " 'azminda.': 1,\n",
              " 'b,': 1,\n",
              " 'badan': 1,\n",
              " 'badan,': 1,\n",
              " 'bagi': 6,\n",
              " 'bagus..': 1,\n",
              " 'bahagian': 1,\n",
              " 'bahan': 1,\n",
              " 'baharudin': 1,\n",
              " 'bahasa': 1,\n",
              " 'bahawa': 1,\n",
              " 'bahwa': 1,\n",
              " 'baik': 1,\n",
              " 'bali,': 1,\n",
              " 'bancuhan': 1,\n",
              " 'bangi,': 1,\n",
              " 'banyak': 3,\n",
              " 'barah.': 1,\n",
              " 'barang': 1,\n",
              " 'baru': 1,\n",
              " 'baru-baru': 1,\n",
              " 'batu-bata': 1,\n",
              " 'begitu': 1,\n",
              " 'begitu,': 1,\n",
              " 'bekalan': 1,\n",
              " 'bekerja': 1,\n",
              " 'belajar': 5,\n",
              " 'belanda,': 1,\n",
              " 'beliau': 2,\n",
              " 'berat': 1,\n",
              " 'berbagai': 1,\n",
              " 'berbuat': 1,\n",
              " 'berbulan': 2,\n",
              " 'bercerai': 1,\n",
              " 'berdiam': 1,\n",
              " 'berdosa?': 1,\n",
              " 'berfikir': 1,\n",
              " 'bergantung': 2,\n",
              " 'berhasrat': 1,\n",
              " 'beriman': 1,\n",
              " 'berjalan': 1,\n",
              " 'berkahwin': 1,\n",
              " 'berkaitan': 2,\n",
              " 'berkata,': 4,\n",
              " 'berkenaan': 3,\n",
              " 'berkenalan': 1,\n",
              " 'berkongsi': 4,\n",
              " 'berkongsi,': 1,\n",
              " 'bermacam-macam': 1,\n",
              " 'bermadu': 24,\n",
              " 'bermadu.': 3,\n",
              " 'bermadu.\"': 1,\n",
              " 'bermadu..': 1,\n",
              " 'bermaduu.': 1,\n",
              " 'berminat': 1,\n",
              " 'berniaga': 1,\n",
              " 'bernilai': 1,\n",
              " 'berpoligami.': 1,\n",
              " 'berpura-pura': 1,\n",
              " 'bersama': 2,\n",
              " 'bersetuju,': 1,\n",
              " 'berstatus': 1,\n",
              " 'bertajuk': 1,\n",
              " 'bertindak': 2,\n",
              " 'berupa': 1,\n",
              " 'berusia': 2,\n",
              " 'berwarna': 1,\n",
              " 'besar.': 1,\n",
              " 'besi': 2,\n",
              " 'bh': 1,\n",
              " 'bhd,': 1,\n",
              " 'biarlah': 1,\n",
              " 'biarpun': 1,\n",
              " 'biasa': 2,\n",
              " 'boleh': 5,\n",
              " 'botol': 1,\n",
              " 'buah-buahan': 1,\n",
              " 'budi': 1,\n",
              " 'budidaya.': 1,\n",
              " 'bukan': 2,\n",
              " 'bulan': 1,\n",
              " 'bumiputera': 1,\n",
              " 'bunga': 3,\n",
              " 'bunga\"': 1,\n",
              " 'bunga.': 2,\n",
              " 'c,': 1,\n",
              " 'cabaran': 2,\n",
              " 'cairan': 2,\n",
              " 'calamansi': 1,\n",
              " 'calon': 1,\n",
              " 'campur': 1,\n",
              " 'campuran': 3,\n",
              " 'cantik': 3,\n",
              " 'cemburu': 1,\n",
              " 'cemburu.': 1,\n",
              " 'cerai': 1,\n",
              " 'cinta': 1,\n",
              " 'ciri': 1,\n",
              " 'cuba': 1,\n",
              " 'cubaan': 1,\n",
              " 'cuka': 2,\n",
              " 'cukup': 1,\n",
              " 'daesh': 1,\n",
              " 'dah': 1,\n",
              " 'dahsyatnya': 1,\n",
              " 'dahuri': 1,\n",
              " 'dalam': 15,\n",
              " 'dalamnya': 2,\n",
              " 'dan': 28,\n",
              " 'dapat': 3,\n",
              " 'darah': 1,\n",
              " 'dari': 10,\n",
              " 'daripada': 6,\n",
              " 'datin': 1,\n",
              " 'demikian': 1,\n",
              " 'dengan': 15,\n",
              " 'depan.': 1,\n",
              " 'derus': 1,\n",
              " 'destinasi': 1,\n",
              " 'di': 24,\n",
              " 'dia': 7,\n",
              " 'dia...': 1,\n",
              " 'diabetes.': 1,\n",
              " 'diakui': 2,\n",
              " 'diberikan': 1,\n",
              " 'dibina': 1,\n",
              " 'dicampur': 1,\n",
              " 'diceraikan..?': 1,\n",
              " 'didefinisikan': 1,\n",
              " 'dielak.': 1,\n",
              " 'diet': 1,\n",
              " 'digabungkan': 1,\n",
              " 'digunakan': 3,\n",
              " 'dihasilkan': 5,\n",
              " 'dihisap': 1,\n",
              " 'dikategorikan': 1,\n",
              " 'dikenali': 1,\n",
              " 'dikenang': 1,\n",
              " 'dikumpulkan': 1,\n",
              " 'dilakukan': 2,\n",
              " 'dilekat': 1,\n",
              " 'dimadukan': 2,\n",
              " 'dimadukan,': 2,\n",
              " 'dimadukan.': 1,\n",
              " 'dimadukan...jadi': 1,\n",
              " 'dimadukan..?': 1,\n",
              " 'dimadukan?': 1,\n",
              " 'diminum': 1,\n",
              " 'diolah': 1,\n",
              " 'dipadamkan': 1,\n",
              " 'diperakui': 1,\n",
              " 'diperlukan': 1,\n",
              " 'diragui': 1,\n",
              " 'dirampas': 1,\n",
              " 'diri': 3,\n",
              " 'diri,': 1,\n",
              " 'diri.': 1,\n",
              " 'dirinya': 4,\n",
              " 'disebabkan': 1,\n",
              " 'disebut': 1,\n",
              " 'diserap': 1,\n",
              " 'disimpan': 1,\n",
              " 'disyaki': 1,\n",
              " 'disyariatkan': 1,\n",
              " 'ditahan': 2,\n",
              " 'diterima': 2,\n",
              " 'diternak': 1,\n",
              " 'ditinggalkan...?': 1,\n",
              " 'ditunda.': 1,\n",
              " 'diubah,': 1,\n",
              " 'diuji': 1,\n",
              " 'diusahakan': 1,\n",
              " 'dua': 5,\n",
              " 'duka': 1,\n",
              " 'dungun,': 1,\n",
              " 'durian': 1,\n",
              " 'empat': 1,\n",
              " 'enzim': 1,\n",
              " 'enzim,': 1,\n",
              " 'erma': 2,\n",
              " 'eropah': 1,\n",
              " 'erti': 1,\n",
              " 'facebook': 1,\n",
              " 'faedahnya,malah': 1,\n",
              " 'fatima': 2,\n",
              " 'fitrah': 1,\n",
              " 'fizikal': 1,\n",
              " 'fosfat.': 1,\n",
              " 'fruktosa': 1,\n",
              " 'gajah': 1,\n",
              " 'garam,': 1,\n",
              " 'glukosa.': 1,\n",
              " 'gula': 8,\n",
              " 'gula,': 2,\n",
              " 'hadis': 1,\n",
              " 'hakikat': 1,\n",
              " 'hakikat!': 1,\n",
              " 'halal,': 1,\n",
              " 'hampir': 1,\n",
              " 'hanya': 1,\n",
              " 'haram': 1,\n",
              " 'hari': 1,\n",
              " 'hartanah': 1,\n",
              " 'haruman': 1,\n",
              " 'harus': 1,\n",
              " 'hasil': 2,\n",
              " 'hasrat': 1,\n",
              " 'hati': 3,\n",
              " 'hebat': 1,\n",
              " 'hendak': 1,\n",
              " 'hendaklah': 1,\n",
              " 'hidup': 13,\n",
              " 'hingga': 1,\n",
              " 'hubungan': 1,\n",
              " 'hubungi': 1,\n",
              " 'hukum': 1,\n",
              " 'iaitu': 5,\n",
              " 'ialah': 8,\n",
              " 'ibarat': 1,\n",
              " 'ibu': 1,\n",
              " 'ilmu': 1,\n",
              " 'indah': 1,\n",
              " 'indah..': 1,\n",
              " 'indahnya': 1,\n",
              " 'indonesia': 2,\n",
              " 'industri': 1,\n",
              " 'ingin': 1,\n",
              " 'ini': 9,\n",
              " 'ini,': 2,\n",
              " 'ini..': 1,\n",
              " 'insan': 1,\n",
              " 'intan': 3,\n",
              " 'isteri': 8,\n",
              " 'isterinya': 1,\n",
              " 'isterinya,': 1,\n",
              " 'itik,': 1,\n",
              " 'itu': 13,\n",
              " 'itu,': 2,\n",
              " 'itu.': 1,\n",
              " 'izreen': 2,\n",
              " 'j33': 1,\n",
              " 'jadi': 4,\n",
              " 'jadi.': 2,\n",
              " 'janda': 1,\n",
              " 'jangan': 1,\n",
              " 'jelly.': 1,\n",
              " 'jenis': 7,\n",
              " 'jernih,': 1,\n",
              " 'jika': 1,\n",
              " 'juga': 3,\n",
              " 'jumaat': 1,\n",
              " 'jururawat': 1,\n",
              " 'jus': 1,\n",
              " 'justeru': 1,\n",
              " 'kabung': 1,\n",
              " 'kadang-kadang,': 1,\n",
              " 'kadar': 1,\n",
              " 'kafe': 1,\n",
              " 'kajian': 2,\n",
              " 'kaka': 1,\n",
              " 'kakaknya,': 1,\n",
              " 'kalau': 2,\n",
              " 'kalium,': 1,\n",
              " 'kalsium,': 1,\n",
              " 'kami': 2,\n",
              " 'kamu': 1,\n",
              " 'kandis,': 1,\n",
              " 'kandungan': 1,\n",
              " 'kanser': 1,\n",
              " 'kapur,': 1,\n",
              " 'kardiovaskular.': 1,\n",
              " 'karier': 1,\n",
              " 'kasihan': 1,\n",
              " 'kata': 1,\n",
              " 'katanya': 1,\n",
              " 'katanya,': 3,\n",
              " 'katanya.': 3,\n",
              " 'kawasan': 1,\n",
              " 'ke': 4,\n",
              " 'keasliannya.': 1,\n",
              " 'kebaikan': 1,\n",
              " 'kebaikan,”': 1,\n",
              " 'kebangsaan': 1,\n",
              " 'kebenaran': 1,\n",
              " 'kebencian': 1,\n",
              " 'kebunnya': 1,\n",
              " 'kecantikan,': 1,\n",
              " 'kecerdasan': 1,\n",
              " 'kecewa': 1,\n",
              " 'kecil': 2,\n",
              " 'kedua': 1,\n",
              " 'kedua.': 1,\n",
              " 'keemasan': 1,\n",
              " 'kelakuan': 1,\n",
              " 'kelantan': 1,\n",
              " 'kelebihan': 1,\n",
              " 'keluar': 2,\n",
              " 'keluaran': 1,\n",
              " 'kelulut': 2,\n",
              " 'kemas': 1,\n",
              " 'kempen': 1,\n",
              " 'kemudian': 1,\n",
              " 'kenapa': 3,\n",
              " 'kepada': 8,\n",
              " 'keputusannya': 1,\n",
              " 'kerana': 4,\n",
              " 'kereta': 1,\n",
              " 'kering': 1,\n",
              " 'kerosakan': 1,\n",
              " 'keruh,\"': 1,\n",
              " 'kesalahan': 1,\n",
              " 'kesan': 1,\n",
              " 'kesehatan,': 1,\n",
              " 'kesihatan': 5,\n",
              " 'kesulitan.': 1,\n",
              " 'ketika': 2,\n",
              " 'ketumpatan': 1,\n",
              " 'kewartawanan': 1,\n",
              " 'kewujudan': 1,\n",
              " 'khan': 1,\n",
              " 'khan,': 1,\n",
              " 'khasiat': 2,\n",
              " 'kicap': 1,\n",
              " 'kimia': 1,\n",
              " 'kini': 2,\n",
              " 'kisah': 1,\n",
              " 'kita': 2,\n",
              " 'kitalah': 1,\n",
              " 'kodeks': 1,\n",
              " 'koktel': 1,\n",
              " 'komposisi': 2,\n",
              " 'kongsi': 1,\n",
              " 'kota': 1,\n",
              " 'kotak': 1,\n",
              " 'kuat': 1,\n",
              " 'kukuh,': 1,\n",
              " 'kurma.': 1,\n",
              " 'lagi': 4,\n",
              " 'lagi,': 1,\n",
              " 'lain': 2,\n",
              " 'lain,': 1,\n",
              " 'lainnya.': 1,\n",
              " 'lalu': 2,\n",
              " 'lalu,': 1,\n",
              " 'lalu.': 3,\n",
              " 'laman': 1,\n",
              " 'lapan': 2,\n",
              " 'lebah': 15,\n",
              " 'lebah,': 3,\n",
              " 'lebah:': 1,\n",
              " 'lebih': 3,\n",
              " 'lelah': 1,\n",
              " 'lelaki': 2,\n",
              " 'lengkap': 1,\n",
              " 'liana': 3,\n",
              " 'liar': 1,\n",
              " 'lihatlah': 1,\n",
              " 'lilin': 1,\n",
              " 'limau': 1,\n",
              " 'london': 1,\n",
              " 'luahan': 1,\n",
              " 'luar': 1,\n",
              " 'mac': 1,\n",
              " 'macam': 1,\n",
              " 'madu': 64,\n",
              " \"madu'\": 1,\n",
              " 'madu,': 5,\n",
              " 'madu-madunya': 1,\n",
              " 'madu.': 6,\n",
              " 'madu.\"': 1,\n",
              " 'madu:': 1,\n",
              " 'madunya,': 1,\n",
              " 'magnesium,': 1,\n",
              " 'mahu': 2,\n",
              " 'maka': 2,\n",
              " 'makan,': 1,\n",
              " 'makan.': 1,\n",
              " 'makanan': 6,\n",
              " 'makmal': 1,\n",
              " 'malah': 1,\n",
              " 'malam': 1,\n",
              " 'malaysia': 2,\n",
              " 'malaysia,\"': 1,\n",
              " 'mampu': 2,\n",
              " 'mana': 2,\n",
              " 'mana-mana': 1,\n",
              " 'manakala': 3,\n",
              " 'mangga,': 1,\n",
              " 'mangsa': 2,\n",
              " 'manis': 4,\n",
              " 'manis,': 1,\n",
              " 'manis.': 1,\n",
              " 'manusia': 1,\n",
              " 'manusia\",': 1,\n",
              " 'manusia.': 2,\n",
              " 'masa': 1,\n",
              " 'masalah': 1,\n",
              " 'masih': 2,\n",
              " 'masing-masing': 1,\n",
              " 'masuk': 1,\n",
              " 'matang.': 1,\n",
              " 'meja': 1,\n",
              " 'melaka.': 1,\n",
              " 'melakukan': 1,\n",
              " 'melarikan': 1,\n",
              " 'melayu)': 1,\n",
              " 'melenting': 1,\n",
              " 'meletakkan': 2,\n",
              " 'melihat': 1,\n",
              " 'meluas': 1,\n",
              " 'memakai': 1,\n",
              " 'memaklumkan': 1,\n",
              " 'memaklumkan,': 1,\n",
              " 'memandang': 1,\n",
              " 'memandangkan': 1,\n",
              " 'memang': 2,\n",
              " 'memasuki': 1,\n",
              " 'membawa': 3,\n",
              " 'membuat': 3,\n",
              " 'membuktikan': 1,\n",
              " 'memekakkan': 1,\n",
              " 'memenangi': 1,\n",
              " 'memilih': 4,\n",
              " 'memiliki': 3,\n",
              " 'meminta': 1,\n",
              " 'meminum': 2,\n",
              " 'mempelbagaikan': 1,\n",
              " 'memperoleh': 1,\n",
              " 'mempunyai': 2,\n",
              " 'memuji': 1,\n",
              " 'memusuhinya.': 1,\n",
              " 'menakutkan.': 1,\n",
              " 'menantu': 1,\n",
              " 'menaruh': 1,\n",
              " 'mencampurkan': 1,\n",
              " 'menceburi': 1,\n",
              " 'mencegah': 1,\n",
              " 'menceraikan': 1,\n",
              " 'mencium': 1,\n",
              " 'mendaki': 1,\n",
              " 'mendalami': 1,\n",
              " 'mendapat': 1,\n",
              " 'menemukan': 1,\n",
              " 'menerima': 2,\n",
              " 'menerusi': 2,\n",
              " 'meneruskan': 1,\n",
              " 'mengaku': 1,\n",
              " 'mengakui': 1,\n",
              " 'mengalami': 1,\n",
              " 'mengamalkan': 1,\n",
              " 'mengambil': 1,\n",
              " 'mengandung': 1,\n",
              " 'mengandungi': 2,\n",
              " 'menganggap': 1,\n",
              " 'mengecam': 1,\n",
              " 'mengeluarkan': 1,\n",
              " 'mengenal': 1,\n",
              " 'mengenali': 1,\n",
              " 'mengenyangkan': 1,\n",
              " 'mengetahui': 1,\n",
              " 'menggunakan': 2,\n",
              " 'menghalang': 1,\n",
              " 'menghilangkan': 1,\n",
              " 'mengizinkan': 1,\n",
              " 'mengoleskan': 1,\n",
              " 'menguasai': 1,\n",
              " 'mengundur': 1,\n",
              " 'mengutamakan': 1,\n",
              " 'mengutip': 1,\n",
              " 'meningkat.': 1,\n",
              " 'menipu': 1,\n",
              " 'menjadi': 9,\n",
              " 'menjadikan': 2,\n",
              " 'menjalinkan': 1,\n",
              " 'menjejaskan': 1,\n",
              " 'menolak': 3,\n",
              " 'menunjuk': 1,\n",
              " 'menurut': 1,\n",
              " 'menyamar': 1,\n",
              " 'menyebutkan': 1,\n",
              " 'menyembuh': 1,\n",
              " 'menyembuhkan': 1,\n",
              " 'menyertai': 1,\n",
              " 'menyerupai': 1,\n",
              " 'menyihatkan': 1,\n",
              " 'menyumbangkan': 1,\n",
              " 'merah': 1,\n",
              " 'merasa': 1,\n",
              " 'merawat': 1,\n",
              " 'merdeka,': 1,\n",
              " 'mereka': 5,\n",
              " 'merupakan': 1,\n",
              " 'mesej': 1,\n",
              " 'mestilah': 1,\n",
              " 'milik': 1,\n",
              " 'mimpi': 1,\n",
              " 'mineral': 1,\n",
              " 'mineral,': 2,\n",
              " 'minuman': 3,\n",
              " 'mirin,': 1,\n",
              " 'miskin,\"': 1,\n",
              " 'mohd': 2,\n",
              " 'monoflora': 1,\n",
              " 'monoflora,': 1,\n",
              " 'mudah': 1,\n",
              " 'mughni,': 1,\n",
              " 'muhamad': 1,\n",
              " 'muhammad': 3,\n",
              " 'mukjizat': 1,\n",
              " 'mula': 1,\n",
              " 'multiflora': 1,\n",
              " 'munasabah': 1,\n",
              " 'mungkin': 2,\n",
              " 'musnah': 1,\n",
              " 'nak': 2,\n",
              " 'nama': 1,\n",
              " 'namun': 1,\n",
              " 'nangka': 1,\n",
              " 'natrium,': 1,\n",
              " 'negara.': 1,\n",
              " 'negeri': 2,\n",
              " 'nektar': 4,\n",
              " 'nepal,': 1,\n",
              " 'netizen': 1,\n",
              " 'ngeri': 1,\n",
              " 'ni': 1,\n",
              " 'ni..': 1,\n",
              " 'nikmat': 1,\n",
              " 'nunjuk': 1,\n",
              " 'nur': 1,\n",
              " 'nusaraya': 1,\n",
              " 'oat': 1,\n",
              " 'oksidasi': 1,\n",
              " 'oleh': 6,\n",
              " 'or': 1,\n",
              " 'orang': 4,\n",
              " 'orang.': 1,\n",
              " 'pada': 2,\n",
              " 'pagi': 1,\n",
              " 'pakaian': 1,\n",
              " 'pakar': 1,\n",
              " 'paku': 1,\n",
              " 'panggangmu,': 1,\n",
              " 'pasangan': 2,\n",
              " 'pasaran': 1,\n",
              " 'pasir,': 1,\n",
              " 'pasti': 3,\n",
              " 'pelajar': 1,\n",
              " 'pelakon': 2,\n",
              " 'pelakon,': 2,\n",
              " 'pelancaran': 1,\n",
              " 'pelbagai': 4,\n",
              " 'pemakanan,': 1,\n",
              " 'pemanis': 2,\n",
              " 'pemantauan': 1,\n",
              " 'pemasaran': 1,\n",
              " 'pemeriksaan': 1,\n",
              " 'peminat': 1,\n",
              " 'pemuzik,': 1,\n",
              " 'pendapatan': 1,\n",
              " 'penelitian': 1,\n",
              " 'peneraju': 1,\n",
              " 'pengakuan': 2,\n",
              " 'pengalaman': 2,\n",
              " 'pengalamannya': 1,\n",
              " 'pengeluar': 1,\n",
              " 'penghujung': 1,\n",
              " 'penguraian': 1,\n",
              " 'pengurus': 1,\n",
              " 'pengusaha': 1,\n",
              " 'peniaga': 1,\n",
              " 'penjual': 1,\n",
              " 'penting.': 1,\n",
              " 'penyakit': 2,\n",
              " 'penyakit,': 1,\n",
              " 'penyakit.': 1,\n",
              " 'penyanyi,': 1,\n",
              " 'penyembuh': 1,\n",
              " 'peratus': 2,\n",
              " 'perdana': 1,\n",
              " 'perempuan': 1,\n",
              " 'perhatian': 2,\n",
              " 'perkahwinan': 1,\n",
              " 'perkara': 2,\n",
              " 'perkenal': 1,\n",
              " 'perkongsian': 1,\n",
              " 'perkumuhan': 1,\n",
              " 'permainanmu': 1,\n",
              " 'pernah': 1,\n",
              " 'perniagaan': 2,\n",
              " 'persediaan\"': 1,\n",
              " 'pertama': 1,\n",
              " 'pertama,': 1,\n",
              " 'perubatan': 1,\n",
              " 'perut': 1,\n",
              " 'pesakit': 1,\n",
              " 'peternak': 1,\n",
              " 'pihak': 1,\n",
              " 'pilihan': 2,\n",
              " 'pinocembrin.': 1,\n",
              " 'pkr': 1,\n",
              " 'pokok': 1,\n",
              " 'poliflora': 1,\n",
              " 'poliflora.': 1,\n",
              " 'poligami.': 1,\n",
              " 'produk': 4,\n",
              " 'produk.': 1,\n",
              " 'produknya': 1,\n",
              " 'proses': 1,\n",
              " 'pula': 1,\n",
              " 'pun': 1,\n",
              " 'pun.': 1,\n",
              " 'punca': 1,\n",
              " 'purba': 1,\n",
              " 'putih': 1,\n",
              " 'putih.': 1,\n",
              " 'radikal': 1,\n",
              " 'radzi': 2,\n",
              " 'rakyat': 1,\n",
              " 'ramai': 2,\n",
              " 'ramai.': 1,\n",
              " 'rasa': 1,\n",
              " 'rasa,': 1,\n",
              " 'rasulullah': 1,\n",
              " 'rata-rata': 1,\n",
              " 'rawatan': 1,\n",
              " 'raya': 1,\n",
              " 'reda.': 1,\n",
              " 'redha': 1,\n",
              " 'rela': 3,\n",
              " 'rembesan': 1,\n",
              " 'rencana': 1,\n",
              " 'resources': 1,\n",
              " 'riwayat': 1,\n",
              " 'roselle': 1,\n",
              " 'roti': 1,\n",
              " 'royal': 1,\n",
              " 'rudjini': 1,\n",
              " 'ruhainies,': 1,\n",
              " 'rumah': 3,\n",
              " 'sabar': 1,\n",
              " 'sah': 1,\n",
              " 'sahaja': 1,\n",
              " 'sahur.': 1,\n",
              " 'saintifik': 1,\n",
              " 'saja.': 1,\n",
              " 'sake': 1,\n",
              " 'sakit,': 1,\n",
              " 'salah': 1,\n",
              " 'salah,': 1,\n",
              " 'saleh,': 3,\n",
              " 'sama': 1,\n",
              " 'sana': 1,\n",
              " 'sangat': 2,\n",
              " 'sanggup': 2,\n",
              " 'sarang': 1,\n",
              " 'sari': 1,\n",
              " 'sari,': 1,\n",
              " 'sasqia': 1,\n",
              " 'saya': 12,\n",
              " 'saya,': 2,\n",
              " 'saya.': 1,\n",
              " 'sdn': 1,\n",
              " 'sebab': 1,\n",
              " 'sebagai': 10,\n",
              " 'sebanyak': 1,\n",
              " 'sebelum': 6,\n",
              " 'sebenarnya': 1,\n",
              " 'seboleh': 1,\n",
              " 'sebuah': 3,\n",
              " 'secara': 3,\n",
              " 'sedang': 1,\n",
              " 'sedemikian': 1,\n",
              " 'sedikit': 2,\n",
              " 'sehingga': 3,\n",
              " 'sejak': 3,\n",
              " 'sejenis': 1,\n",
              " 'sekadar': 1,\n",
              " 'sekiranya': 1,\n",
              " 'selain': 1,\n",
              " 'selama': 1,\n",
              " 'selepas': 2,\n",
              " 'seluruh': 1,\n",
              " 'semua': 1,\n",
              " 'semula': 4,\n",
              " 'senang': 1,\n",
              " 'seni,': 1,\n",
              " 'seorang': 4,\n",
              " 'sepatutnya': 1,\n",
              " 'sepenuhnya.': 1,\n",
              " 'seperti': 4,\n",
              " 'seragam': 1,\n",
              " 'serangga': 1,\n",
              " 'seri': 1,\n",
              " 'sering': 1,\n",
              " 'serong': 1,\n",
              " 'serta': 4,\n",
              " 'sesetengah': 1,\n",
              " 'sesuai': 1,\n",
              " 'sesuatu': 3,\n",
              " 'sesudu': 1,\n",
              " 'seterusnya': 1,\n",
              " 'shahmira': 1,\n",
              " 'sheila,': 1,\n",
              " 'shuib': 1,\n",
              " 'siapa': 1,\n",
              " 'sifat': 2,\n",
              " 'sihulk': 1,\n",
              " 'sindiket': 1,\n",
              " 'singapura,': 1,\n",
              " 'sirup': 1,\n",
              " 'soal': 1,\n",
              " 'spesifik': 2,\n",
              " 'status': 1,\n",
              " 'stokis': 1,\n",
              " 'suami': 4,\n",
              " 'suami,': 1,\n",
              " 'suami.': 1,\n",
              " 'suaminya': 2,\n",
              " 'suatu': 1,\n",
              " 'suhaimi,': 1,\n",
              " 'suka': 2,\n",
              " 'sulfur,': 1,\n",
              " 'sumber': 1,\n",
              " 'sungai': 1,\n",
              " 'surah': 1,\n",
              " 'suruhanjaya': 1,\n",
              " 'suspek': 2,\n",
              " 'suspek,': 1,\n",
              " 'susulan': 1,\n",
              " 'swt': 1,\n",
              " 'syarak.': 1,\n",
              " 'syarikat': 3,\n",
              " 'syria.': 1,\n",
              " 'syurga': 1,\n",
              " 'syurga.\"': 1,\n",
              " 'tabib.': 1,\n",
              " 'tadi': 2,\n",
              " 'tahu': 3,\n",
              " 'tahun': 5,\n",
              " 'tahun.': 1,\n",
              " 'tak': 3,\n",
              " 'tambah': 1,\n",
              " 'tangga': 1,\n",
              " 'tanpa': 2,\n",
              " 'tapi': 1,\n",
              " 'tarikh': 1,\n",
              " 'telinga': 1,\n",
              " 'telur': 1,\n",
              " 'tempatan': 2,\n",
              " 'tenaga': 2,\n",
              " 'tepung': 1,\n",
              " 'terbabit': 1,\n",
              " 'terbaik': 1,\n",
              " 'terbesar': 1,\n",
              " 'terbuat': 1,\n",
              " 'terdapat': 3,\n",
              " 'terhadap': 2,\n",
              " 'terhalanglah': 1,\n",
              " 'terima': 1,\n",
              " 'terjadi': 1,\n",
              " 'terkandung': 2,\n",
              " 'terkini,': 1,\n",
              " 'termasuk': 2,\n",
              " 'terpaksa': 1,\n",
              " 'tersendiri.': 1,\n",
              " 'tetapi': 2,\n",
              " 'tiada': 3,\n",
              " 'tidak': 15,\n",
              " 'tidur.\"': 1,\n",
              " 'tiga': 1,\n",
              " 'tiruan': 2,\n",
              " 'tkpm,': 1,\n",
              " 'tonik': 1,\n",
              " 'tradisional': 1,\n",
              " 'tualang': 3,\n",
              " 'tubuh': 1,\n",
              " 'tujuh': 1,\n",
              " 'tukar': 1,\n",
              " 'tumbuhan,': 2,\n",
              " 'tunai.': 1,\n",
              " 'tunda': 1,\n",
              " 'tunggal': 1,\n",
              " 'turun': 1,\n",
              " 'turut': 2,\n",
              " 'ubat': 3,\n",
              " 'ude': 1,\n",
              " 'ude,': 1,\n",
              " 'ujian': 1,\n",
              " 'umie': 1,\n",
              " 'umie,': 1,\n",
              " 'umpama': 1,\n",
              " 'umum,': 1,\n",
              " 'umumnya': 1,\n",
              " 'universiti': 1,\n",
              " 'unsur': 1,\n",
              " 'untuk': 13,\n",
              " 'uruskan': 1,\n",
              " 'usaha': 1,\n",
              " 'utama': 2,\n",
              " 'utk': 1,\n",
              " 'vitamin': 4,\n",
              " 'vitamin,': 1,\n",
              " 'wahid': 1,\n",
              " 'wahid,': 1,\n",
              " 'wahyu': 1,\n",
              " 'walaupun': 4,\n",
              " 'wang': 1,\n",
              " 'wanita': 13,\n",
              " 'wanita,': 1,\n",
              " 'warna,': 1,\n",
              " 'warnanya,': 1,\n",
              " 'wartawan': 1,\n",
              " 'yaitu': 1,\n",
              " 'yang': 62,\n",
              " 'yg': 2,\n",
              " 'zainal,': 1,\n",
              " 'zainuddin': 1,\n",
              " 'zaman': 1,\n",
              " 'zarina': 1,\n",
              " 'zawawi': 1,\n",
              " '‘justgreat’,': 1}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "n7q_6spuBM4X",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Some symbols and digits found in the words list that should be removed. Also, there are words that share the same root and can be merged. Will attempt naive bayes with and without stemming."
      ]
    },
    {
      "metadata": {
        "id": "HlHDOTQMC9L2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "words_cleaned = []\n",
        "#symbols_to_remove = re.compile('[/(){}\\[\\]\\.\\-\\\"|@,; ]')\n",
        "symbols_to_remove = re.compile('[^\\w]')\n",
        "for line in df['SENTENCE']:\n",
        "  for w in re.split('\\s+',line):\n",
        "    #won't try to correct non-words that have digits\n",
        "    if not re.match('.*[0-9].*',w):\n",
        "      w = re.sub(symbols_to_remove,'',w)\n",
        "      if len(w)>0:\n",
        "        words_cleaned.append(w)\n",
        "    \n",
        "words_cleaned_dict = wordcount(words_cleaned)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ab2Ssi0ly226",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Use malaya library to stem the malay words and create a new words list"
      ]
    },
    {
      "metadata": {
        "id": "ywAr_uKhzGVZ",
        "colab_type": "code",
        "outputId": "65be2782-6c62-45c9-e844-d80a39597a96",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1303
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install malaya\n",
        "import malaya"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting malaya\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/df/ed/ae9f6889296e7a874fb768d20df18b0c31bb908c4ee64fb21eb092dc65ac/malaya-1.9.1.2-py3-none-any.whl (1.5MB)\n",
            "\u001b[K    100% |████████████████████████████████| 1.5MB 15.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from malaya) (2.18.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from malaya) (1.1.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.6/dist-packages (from malaya) (0.9.0)\n",
            "Collecting PySastrawi (from malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/61/84/b0a5454a040f81e81e6a95a5d5635f20ad43cc0c288f8b4966b339084962/PySastrawi-1.2.0-py2.py3-none-any.whl (210kB)\n",
            "\u001b[K    100% |████████████████████████████████| 215kB 30.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from malaya) (1.14.6)\n",
            "Collecting fuzzywuzzy (from malaya)\n",
            "  Downloading https://files.pythonhosted.org/packages/d8/f1/5a267addb30ab7eaa1beab2b9323073815da4551076554ecc890a3595ec9/fuzzywuzzy-0.17.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from malaya) (4.28.1)\n",
            "Collecting python-levenshtein (from malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/a9/d1785c85ebf9b7dfacd08938dd028209c34a0ea3b1bcdb895208bd40a67d/python-Levenshtein-0.12.0.tar.gz (48kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 10.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow in /usr/local/lib/python3.6/dist-packages (from malaya) (1.13.1)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from malaya) (0.0)\n",
            "Requirement already satisfied: unidecode in /usr/local/lib/python3.6/dist-packages (from malaya) (1.0.23)\n",
            "Collecting scikit-learn==0.19.1 (from malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3d/2d/9fbc7baa5f44bc9e88ffb7ed32721b879bfa416573e85031e16f52569bc9/scikit_learn-0.19.1-cp36-cp36m-manylinux1_x86_64.whl (12.4MB)\n",
            "\u001b[K    100% |████████████████████████████████| 12.4MB 2.5MB/s \n",
            "\u001b[?25hCollecting xgboost==0.80 (from malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8f/15/606f81a2b8a8e82eaa10683cb3f3074905ec65d3bcef949e3f0909f165a5/xgboost-0.80-py2.py3-none-manylinux1_x86_64.whl (15.8MB)\n",
            "\u001b[K    100% |████████████████████████████████| 15.8MB 2.6MB/s \n",
            "\u001b[?25hCollecting sklearn-crfsuite (from malaya)\n",
            "  Downloading https://files.pythonhosted.org/packages/25/74/5b7befa513482e6dee1f3dd68171a6c9dfc14c0eaa00f885ffeba54fe9b0/sklearn_crfsuite-0.3.6-py2.py3-none-any.whl\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->malaya) (1.22)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->malaya) (2.6)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->malaya) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->malaya) (2018.11.29)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from python-levenshtein->malaya) (40.8.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (0.7.1)\n",
            "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.13.0)\n",
            "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (0.7.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.11.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (3.6.1)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (0.33.1)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.15.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (0.2.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.0.7)\n",
            "Collecting tensorboard<1.14.0,>=1.13.0 (from tensorflow->malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fa/7b/3ee06856ec30d5136cd2002408df1d111fcff269f3691147dbf3b8dc0ba2/tensorboard-1.13.0-py3-none-any.whl (3.2MB)\n",
            "\u001b[K    100% |████████████████████████████████| 3.2MB 11.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow->malaya) (1.0.9)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from sklearn-crfsuite->malaya) (0.8.3)\n",
            "Collecting python-crfsuite>=0.8.3 (from sklearn-crfsuite->malaya)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2f/86/cfcd71edca9d25d3d331209a20f6314b6f3f134c29478f90559cee9ce091/python_crfsuite-0.9.6-cp36-cp36m-manylinux1_x86_64.whl (754kB)\n",
            "\u001b[K    100% |████████████████████████████████| 757kB 16.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: mock>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow->malaya) (2.0.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow->malaya) (2.8.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow->malaya) (3.0.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow->malaya) (0.14.1)\n",
            "Requirement already satisfied: pbr>=0.11 in /usr/local/lib/python3.6/dist-packages (from mock>=2.0.0->tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow->malaya) (5.1.2)\n",
            "Building wheels for collected packages: python-levenshtein\n",
            "  Building wheel for python-levenshtein (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/de/c2/93/660fd5f7559049268ad2dc6d81c4e39e9e36518766eaf7e342\n",
            "Successfully built python-levenshtein\n",
            "\u001b[31myellowbrick 0.9.1 has requirement scikit-learn>=0.20, but you'll have scikit-learn 0.19.1 which is incompatible.\u001b[0m\n",
            "\u001b[31mimbalanced-learn 0.4.3 has requirement scikit-learn>=0.20, but you'll have scikit-learn 0.19.1 which is incompatible.\u001b[0m\n",
            "Installing collected packages: PySastrawi, fuzzywuzzy, python-levenshtein, scikit-learn, xgboost, python-crfsuite, sklearn-crfsuite, malaya, tensorboard\n",
            "  Found existing installation: scikit-learn 0.20.2\n",
            "    Uninstalling scikit-learn-0.20.2:\n",
            "      Successfully uninstalled scikit-learn-0.20.2\n",
            "  Found existing installation: xgboost 0.7.post4\n",
            "    Uninstalling xgboost-0.7.post4:\n",
            "      Successfully uninstalled xgboost-0.7.post4\n",
            "  Found existing installation: tensorboard 1.12.2\n",
            "    Uninstalling tensorboard-1.12.2:\n",
            "      Successfully uninstalled tensorboard-1.12.2\n",
            "Successfully installed PySastrawi-1.2.0 fuzzywuzzy-0.17.0 malaya-1.9.1.2 python-crfsuite-0.9.6 python-levenshtein-0.12.0 scikit-learn-0.19.1 sklearn-crfsuite-0.3.6 tensorboard-1.13.0 xgboost-0.80\n",
            "not found any version, deleting previous version models..\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bYSZK9k0y9Fh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def clean_and_stem(sentence_list):\n",
        "  #function to clean and stem all words\n",
        "  #returns new list of sentences and words_dict{word:count}\n",
        "  symbols_to_remove = re.compile('[^\\w]')\n",
        "  words_cleaned_stemmed = []\n",
        "  words_dict = {}\n",
        "  new_sentences =[]\n",
        "  #stemmer = malaya.stem.deep_model(model)\n",
        "  \n",
        "  for sentence in sentence_list:\n",
        "    nstr = ''\n",
        "    sentence = malaya.stem.sastrawi(sentence)\n",
        "    for w in re.split('\\s+', sentence.lower()):\n",
        "      #only include words that don't have digits and remove symbols\n",
        "      if not re.match('.*[0-9].*',w):\n",
        "        w = re.sub(symbols_to_remove,'',w)\n",
        "        if len(w)>0:\n",
        "          words_cleaned_stemmed.append(w)\n",
        "          nstr = nstr+' '+w\n",
        "    new_sentences.append(nstr)\n",
        "  \n",
        "  words_dict = wordcount(words_cleaned_stemmed)\n",
        "  return new_sentences, words_dict\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4TRi81yr2zaX",
        "colab_type": "code",
        "outputId": "beeb263a-49c9-4f67-842c-e5df926b4ede",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "cell_type": "code",
      "source": [
        "ilist=['MADU ialah makanan yang sangat bernilai yang digunakan sejak zaman purba sebagai sumber makanan tenaga yang penting.','Menurut Suruhanjaya Alimentarius Kodeks (1989), madu didefinisikan sebagai pemanis semula jadi dihasilkan lebah madu dari sari bunga atau nektar atau dari rembesan bahagian hidup tumbuhan, atau perkumuhan tumbuhan, dikumpulkan oleh lebah madu dan diubah, serta digabungkan dengan bahan spesifik dalam lebah madu sehingga matang.']\n",
        "\n",
        "x,y = clean_and_stem(ilist)\n",
        "x\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[' madu ialah makan yang sangat nila yang guna sejak zaman purba bagai sumber makan tenaga yang penting',\n",
              " ' turut suruhanjaya alimentarius kodeks madu definisi bagai man mula jadi hasil lebah madu dari sari bunga atau nektar atau dari rembes bahagian hidup tumbuh atau kumuh tumbuh kumpul oleh lebah madu dan ubah serta gabung dengan bahan spesifik dalam lebah madu sehingga matang']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "ltQ6jicuHa7X",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "The cleaned list of words had 805 instances. Will use a CBOW representation for first trial."
      ]
    },
    {
      "metadata": {
        "id": "MpNYkTqDJk7m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def BOW(text, words_to_index, dict_size):\n",
        "    \"\"\"\n",
        "        text: a string\n",
        "        dict_size: size of the dictionary\n",
        "        \n",
        "        return a vector which is a bag-of-words representation of 'text'\n",
        "    \"\"\"\n",
        "    symbols_to_remove = re.compile('[^\\w]')\n",
        "    result_vector = np.zeros(dict_size)\n",
        "\n",
        "    l=re.split('\\s+',text)\n",
        "    for w in l:\n",
        "      w = re.sub(symbols_to_remove,'',w)\n",
        "      if w in words_to_index.keys():\n",
        "        result_vector[words_to_index[w]]=1\n",
        "    return result_vector"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "twWHPsXHjmxU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Convert all sentences to BOW"
      ]
    },
    {
      "metadata": {
        "id": "hMkAR6AznBhb",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Encode label 'manisan'=0 and 'manusia'=1"
      ]
    },
    {
      "metadata": {
        "id": "fZOYq2Ls15s7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "y_encoded = []\n",
        "for line in df['CATEGORY & POS']:\n",
        "  if re.match('manisan',line):\n",
        "    y_encoded.append('0')\n",
        "  else:\n",
        "    y_encoded.append('1')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eu1jsWcgiMHU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Create train-test split"
      ]
    },
    {
      "metadata": {
        "id": "2ZAR8rvfjQUk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ae4nhEipC6ID",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "plot ROC curve"
      ]
    },
    {
      "metadata": {
        "id": "9WUwihm7DT8E",
        "colab_type": "code",
        "outputId": "8e5d5df6-3219-4a64-a7c4-538d8fd5b4cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install scikit-plot"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting scikit-plot\n",
            "  Downloading https://files.pythonhosted.org/packages/7c/47/32520e259340c140a4ad27c1b97050dd3254fdc517b1d59974d47037510e/scikit_plot-0.3.7-py3-none-any.whl\n",
            "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.6/dist-packages (from scikit-plot) (0.19.1)\n",
            "Requirement already satisfied: joblib>=0.10 in /usr/local/lib/python3.6/dist-packages (from scikit-plot) (0.13.2)\n",
            "Requirement already satisfied: scipy>=0.9 in /usr/local/lib/python3.6/dist-packages (from scikit-plot) (1.1.0)\n",
            "Requirement already satisfied: matplotlib>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-plot) (3.0.2)\n",
            "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from scipy>=0.9->scikit-plot) (1.14.6)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.0->scikit-plot) (2.3.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.0->scikit-plot) (2.5.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.0->scikit-plot) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.4.0->scikit-plot) (1.0.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib>=1.4.0->scikit-plot) (1.11.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib>=1.4.0->scikit-plot) (40.8.0)\n",
            "Installing collected packages: scikit-plot\n",
            "Successfully installed scikit-plot-0.3.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "B7CD0T_CC9JL",
        "colab_type": "code",
        "outputId": "6853a077-0597-4d4e-8d19-26fdc3847e49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import scikitplot as skplt\n",
        "probs=classifier.predict_proba(X_test)\n",
        "preds = probs[:,1]\n",
        "skplt.metrics.plot_roc_curve(y_test, probs)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-1d97182f9e4c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mscikitplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mskplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprobs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mskplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_roc_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'classifier' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "1K6uLTnSjRd_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Get F1 score"
      ]
    },
    {
      "metadata": {
        "id": "LXm2x_pPjUGz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import f1_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZqdDOJMlmV1S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "check for F1 score trend with different vocab size"
      ]
    },
    {
      "metadata": {
        "id": "RcbBDV8UlIVa",
        "colab_type": "code",
        "outputId": "50be4658-48be-407c-f85e-49e46f3cf2d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        }
      },
      "cell_type": "code",
      "source": [
        "#check F1 score for different vocab sizes using different classifiers\n",
        "vocabsize=[10,20,50,100]\n",
        "nbresult=[]\n",
        "lrresult=[]\n",
        "svmresult=[]\n",
        "\n",
        "from sklearn.naive_bayes import BernoulliNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "\n",
        "for i in vocabsize:\n",
        "  DICT_SIZE = i\n",
        "  WORDS_TO_INDEX = {}\n",
        "  top_words=sorted(words_cleaned_dict.items(), key=lambda x: x[1], reverse=True)[:DICT_SIZE]\n",
        "  for x in range(DICT_SIZE): WORDS_TO_INDEX.update({top_words[x][0]:x})\n",
        "  #INDEX_TO_WORDS = {v:k for k,v in WORDS_TO_INDEX.items()}\n",
        "  #ALL_WORDS = WORDS_TO_INDEX.keys()\n",
        "\n",
        "  X_BOW = [BOW(sentence,WORDS_TO_INDEX, i) for sentence in df['SENTENCE']]\n",
        "  X_train, X_test, y_train, y_test  = train_test_split(X_BOW, y_encoded, test_size=0.2, random_state=10)\n",
        "  y_train=np.asarray(y_train, dtype=int)\n",
        "  y_test=np.asarray(y_test, dtype=int)\n",
        "  #naive bayes\n",
        "  bnb = BernoulliNB()\n",
        "  nbclassifier = bnb.fit(X_train, y_train)\n",
        "  y_pred_nb = nbclassifier.predict(X_test)\n",
        "  nbscore=f1_score(y_test, y_pred_nb)\n",
        "  nbresult.append(\"{:.2f}\".format(nbscore))\n",
        "  \n",
        "  result=[]\n",
        "  #LogisticRegression\n",
        "  lr = LogisticRegression()\n",
        "  lrclassifier = lr.fit(X_train, y_train)\n",
        "  y_pred_lr = lrclassifier.predict(X_test)\n",
        "  lrscore=f1_score(y_test, y_pred_lr)\n",
        "  lrresult.append(\"{:.2f}\".format(lrscore))\n",
        "  #SVM\n",
        "  svmclassifier=SVC().fit(X_train, y_train)\n",
        "  y_pred_svm = svmclassifier.predict(X_test)\n",
        "  svmscore = f1_score(y_test, y_pred_svm)\n",
        "  svmresult.append(\"{:.2f}\".format(svmscore))\n",
        "  \n",
        "print(top_words[:10])\n",
        "print(\"naive bayes\")\n",
        "print(vocabsize)\n",
        "print(nbresult)\n",
        "print(\"logistic regression\")\n",
        "print(vocabsize)\n",
        "print(lrresult)\n",
        "print(\"SVM\")\n",
        "print(vocabsize)\n",
        "print(svmresult)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('madu', 79), ('yang', 62), ('bermadu', 29), ('dan', 28), ('di', 24), ('lebah', 20), ('itu', 16), ('saya', 16), ('tidak', 15), ('dengan', 15)]\n",
            "naive bayes\n",
            "[10, 20, 50, 100]\n",
            "['0.93', '0.93', '1.00', '1.00']\n",
            "logistic regression\n",
            "[10, 20, 50, 100]\n",
            "['0.93', '0.93', '1.00', '1.00']\n",
            "SVM\n",
            "[10, 20, 50, 100]\n",
            "['0.93', '0.93', '1.00', '0.60']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bHsqCcWgssCM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Checking to see with 100 words vocab how many of the sentence data had all '0' array"
      ]
    },
    {
      "metadata": {
        "id": "AqPRU1xY47LG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_BOW = [BOW(sentence,WORDS_TO_INDEX, 200) for sentence in df['SENTENCE']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PW1OGAxO5bxW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#count how many entries in X_BOW equal 0\n",
        "counter=0\n",
        "for i in range(0,len(X_BOW)):\n",
        "  if sum(X_BOW[i])==0:\n",
        "    print(df['SENTENCE'][i])\n",
        "    counter+=1\n",
        "print(counter)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YQ-0hkAo6Qa9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "test models with stences that are stemmed"
      ]
    },
    {
      "metadata": {
        "id": "WH4GfS4L6Pbd",
        "colab_type": "code",
        "outputId": "c2ab003a-152c-4736-fd62-65f448453090",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        }
      },
      "cell_type": "code",
      "source": [
        "new_sentences, words_cleaned_stemmed = clean_and_stem(df['SENTENCE'])\n",
        "\n",
        "#check F1 score for different vocab sizes using different classifiers\n",
        "vocabsize=[10,20,50,100]\n",
        "nbresult=[]\n",
        "lrresult=[]\n",
        "svmresult=[]\n",
        "\n",
        "from sklearn.naive_bayes import BernoulliNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "\n",
        "for i in vocabsize:\n",
        "  DICT_SIZE = i\n",
        "  WORDS_TO_INDEX = {}\n",
        "  top_words=sorted(words_cleaned_stemmed.items(), key=lambda x: x[1], reverse=True)[:DICT_SIZE]\n",
        "  for x in range(DICT_SIZE): WORDS_TO_INDEX.update({top_words[x][0]:x})\n",
        "  #INDEX_TO_WORDS = {v:k for k,v in WORDS_TO_INDEX.items()}\n",
        "  #ALL_WORDS = WORDS_TO_INDEX.keys()\n",
        "  \n",
        "\n",
        "  X_BOW = [BOW(sentence,WORDS_TO_INDEX, i) for sentence in new_sentences]\n",
        "  X_train, X_test, y_train, y_test  = train_test_split(X_BOW, y_encoded, test_size=0.2, random_state=10)\n",
        "  y_train=np.asarray(y_train, dtype=int)\n",
        "  y_test=np.asarray(y_test, dtype=int)\n",
        "  #naive bayes\n",
        "  bnb = BernoulliNB()\n",
        "  nbclassifier = bnb.fit(X_train, y_train)\n",
        "  y_pred_nb = nbclassifier.predict(X_test)\n",
        "  nbscore=f1_score(y_test, y_pred_nb)\n",
        "  nbresult.append(\"{:.2f}\".format(nbscore))\n",
        "  \n",
        "  result=[]\n",
        "  #LogisticRegression\n",
        "  lr = LogisticRegression()\n",
        "  lrclassifier = lr.fit(X_train, y_train)\n",
        "  y_pred_lr = lrclassifier.predict(X_test)\n",
        "  lrscore=f1_score(y_test, y_pred_lr)\n",
        "  lrresult.append(\"{:.2f}\".format(lrscore))\n",
        "  #SVM\n",
        "  svmclassifier=SVC().fit(X_train, y_train)\n",
        "  y_pred_svm = svmclassifier.predict(X_test)\n",
        "  svmscore = f1_score(y_test, y_pred_svm)\n",
        "  svmresult.append(\"{:.2f}\".format(svmscore))\n",
        "  \n",
        "print(top_words[:10])\n",
        "print(\"naive bayes\")\n",
        "print(vocabsize)\n",
        "print(nbresult)\n",
        "print(\"logistic regression\")\n",
        "print(vocabsize)\n",
        "print(lrresult)\n",
        "print(\"SVM\")\n",
        "print(vocabsize)\n",
        "print(svmresult)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('madu', 118), ('yang', 62), ('dan', 28), ('di', 24), ('jadi', 20), ('lebah', 20), ('dalam', 18), ('itu', 16), ('saya', 16), ('tidak', 15)]\n",
            "naive bayes\n",
            "[10, 20, 50, 100]\n",
            "['0.78', '0.88', '0.88', '0.82']\n",
            "logistic regression\n",
            "[10, 20, 50, 100]\n",
            "['0.63', '0.82', '0.88', '0.82']\n",
            "SVM\n",
            "[10, 20, 50, 100]\n",
            "['0.52', '0.92', '0.44', '0.00']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}