income_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python: 3.7.3 (default, Apr 24 2019, 15:29:51) [MSC v.1915 64 bit (AMD64)]\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import math\n",
    "import sklearn\n",
    "import statistics\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "%config IPcompletor.greedy = True\n",
    "print(\"Python: {}\".format(sys.version))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = pd.read_csv(\"tcd ml 2019-20 income prediction training (with labels).csv\", index_col = 0)\n",
    "test_set = pd.read_csv(\"tcd ml 2019-20 income prediction test (without labels).csv\", index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set[train_set.columns]=train_set[train_set.columns].fillna(train_set.mode().iloc[0])\n",
    "test_set[test_set.columns]=test_set[test_set.columns].fillna(train_set.mode().iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding\n",
    " --for categorical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = train_set.Profession.value_counts()\n",
    "p = t[t <10].index.tolist()\n",
    "p.extend(set(test_set[\"Profession\"].values) - set(t.index.tolist()))\n",
    "t = train_set.Country.value_counts()\n",
    "c = t[t <10].index.tolist()\n",
    "c.extend(set(test_set[\"Country\"].values) - set(t.index.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = dict((k,\"Rare\") for k in p)\n",
    "train_set[\"Profession\"] = train_set[\"Profession\"].replace(t)\n",
    "test_set[\"Profession\"] = test_set[\"Profession\"].replace(t)\n",
    "c = dict((k,\"Rare\") for k in c)\n",
    "train_set[\"Country\"] = train_set[\"Country\"].replace(c)\n",
    "test_set[\"Country\"] = test_set[\"Country\"].replace(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Outlier Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# res = train_set.groupby(\"Profession\")[\"Income in EUR\"].quantile([0.01, 0.99]).unstack(level=1)\n",
    "# train_set = train_set.loc[((res.loc[train_set.Profession, 0.01] < train_set[\"Income in EUR\"].values) & (train_set[\"Income in EUR\"].values < res.loc[train_set.Profession, 0.99])).values]\n",
    "# temp.shape\n",
    "# res = train_set.groupby(\"Country\")[\"Income in EUR\"].quantile([0.001, 0.999]).unstack(level=1)\n",
    "# train_set = train_set.loc[((res.loc[train_set.Country, 0.001] < train_set[\"Income in EUR\"].values) & (train_set[\"Income in EUR\"].values < res.loc[train_set.Country, 0.999])).values]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "prof = train_set.groupby('Profession').agg({\"Income in EUR\": 'mean'})\n",
    "prof = dict(zip(prof.index, prof[\"Income in EUR\"]))\n",
    "country = train_set.groupby('Country').agg({\"Income in EUR\": 'mean'})\n",
    "country = dict(zip(country.index, country[\"Income in EUR\"]))\n",
    "degree = train_set.groupby('University Degree').agg({\"Income in EUR\": 'mean'})\n",
    "degree = dict(zip(degree.index, degree[\"Income in EUR\"]))\n",
    "gender = train_set.groupby('Gender').agg({\"Income in EUR\": 'mean'})\n",
    "gender = dict(zip(gender.index, gender[\"Income in EUR\"]))\n",
    "color = train_set.groupby('Hair Color').agg({\"Income in EUR\": 'mean'})\n",
    "color = dict(zip(color.index, color[\"Income in EUR\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vansh\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "train_set[\"train\"] = 1\n",
    "test_set[\"train\"] = 0\n",
    "train_db = pd.concat([train_set,test_set])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db[\"Profession\"] = train_db[\"Profession\"].replace(prof)\n",
    "train_db[\"Country\"] = train_db[\"Country\"].replace(country)\n",
    "train_db[\"University Degree\"] = train_db[\"University Degree\"].replace(degree)\n",
    "train_db[\"Gender\"] = train_db[\"Gender\"].replace(gender)\n",
    "train_db[\"Hair Color\"] = train_db[\"Hair Color\"].replace(color)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning\n",
    "1. Removing NULL values\n",
    "2. Noise\n",
    "3. Inconsistent Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db = train_db.rename(columns= {\"Body Height [cm]\":  \"Body Height\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_db[\"Year of Record\"] = train_db[\"Year of Record\"].fillna(statistics.median(train_db['Year of Record'])).astype('int')\n",
    "train_db[\"Age\"] = train_db[\"Age\"].fillna(statistics.median(train_db['Age'])).astype('int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_db = train_db.drop(columns=[\"Gender\", \"Hair Color\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train_db[train_db[\"train\"] ==1].drop(columns=[\"Income in EUR\",\"Income\", \"train\"], axis =1)\n",
    "test = train_db[train_db[\"train\"] ==0].drop(columns=[\"Income in EUR\",\"Income\", \"train\"], axis =1)\n",
    "train_income = train_db[train_db[\"train\"] ==1][\"Income in EUR\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age                  0\n",
       "Body Height          0\n",
       "Country              0\n",
       "Gender               0\n",
       "Hair Color           0\n",
       "Profession           0\n",
       "Size of City         0\n",
       "University Degree    0\n",
       "Wears Glasses        0\n",
       "Year of Record       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MODEL CREATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit = pd.read_csv(\"tcd ml 2019-20 income prediction submission file.csv\", index_col= 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, train_income, test_size=0.2, random_state=122)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Polynomial Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "poly = PolynomialFeatures(degree=3, interaction_only = True)\n",
    "X_ = poly.fit_transform(X_train)\n",
    "test_ = poly.fit_transform(X_test)\n",
    "new_train = poly.fit_transform(train)\n",
    "new_test = poly.fit_transform(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = linear_model.LinearRegression()\n",
    "lmodel = model.fit(X_, y_train)\n",
    "\n",
    "y_pred = lmodel.predict(test_)\n",
    "math.sqrt(mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = linear_model.LinearRegression()\n",
    "lmodel = model.fit(train, train_income)\n",
    "\n",
    "y_pred = lmodel.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVR\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "begin = time()\n",
    "model = SVR(kernel = 'linear', gamma='scale',epsilon=0.1)\n",
    "lmodel = model.fit(X_train, y_train)\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lmodel.predict(X_test)\n",
    "math.sqrt(mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "begin = time()\n",
    "model = RandomForestRegressor(random_state=42, n_estimators=400, warm_start = True)\n",
    "lmodel = model.fit(X_train, y_train)\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done 400 out of 400 | elapsed:    4.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "36024.65920344338"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = lmodel.predict(X_test)\n",
    "math.sqrt(mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time elapsed: 494.89382147789\n"
     ]
    }
   ],
   "source": [
    "begin = time()\n",
    "model = RandomForestRegressor(random_state=42, n_estimators=400, warm_start = True)\n",
    "lmodel = model.fit(train, train_income)\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))\n",
    "y_pred = lmodel.predict(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost\n",
    "--with cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dmatrix = xgb.DMatrix(data=train,label=train_income)\n",
    "params = {\"objective\": \"reg:linear\", \"random_state\": 123, \"learning_rate\": 0.15, \"max_depth\" : 7, \"subsample\": 0.75, \n",
    "                         \"colsample_bytree\" : 0.7, \"booster\" : 'gbtree'}\n",
    "cv_results = xgb.cv(dtrain=data_dmatrix, params=params, nfold=4,\n",
    "                    num_boost_round=100,early_stopping_rounds=10,metrics=\"rmse\", as_pandas=True, seed=223)\n",
    "cv_results.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13:34:40] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAEWCAYAAADrfqfPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV1bn/8c8XUESgei1DUapIQUVBEBTEIsYizopSK1KsIlprW6v+nC639lq1g1i1TrUDOICoiGgrDq2K4hG1TqAMgqJW48URaa0CUpme3x97J56EhOxAwskJ3/frlVf2WXvttZ+VA3my1t5nL0UEZmZmtn5NCh2AmZlZMXDCNDMzy8AJ08zMLAMnTDMzswycMM3MzDJwwjQzM8vACdPM6pSkP0r630LHYVbX5M9hmjUMkkqB9sCavOJdIuL9jWizBLg9IjpuXHTFSdJ44N2I+FmhY7Hi5xGmWcNyVES0yvva4GRZFyQ1K+T5N4akpoWOwRoXJ0yzIiBpX0l/l/RvSXPSkWPZvlMkvSppqaS3JP0gLW8J/A3YXtKy9Gt7SeMl/TLv+BJJ7+a9LpX035LmAsslNUuPu1fSx5LelnTWemItb7+sbUkXSlos6QNJx0g6XNLrkv4l6ad5x14i6R5Jk9P+vCSpZ97+bpJy6c9hvqSjK533D5L+Kmk5cCowArgw7fsDab3Rkv6Rtr9A0rF5bYyU9LSkqyR9kvb1sLz920m6VdL76f778vYdKWl2GtvfJe2Z+Q22ouCEadbASdoBeAj4JbAdcD5wr6S2aZXFwJHAV4BTgGsk9Y6I5cBhwPsbMGIdDhwBbAusBR4A5gA7AIOAcyQdkrGtrwFbpcdeDIwDTgT6APsDF0vqnFd/CDAl7eudwH2StpC0RRrHo0A74CfAHZJ2zTv2u8CvgNbAbcAdwG/Svh+V1vlHet5tgEuB2yV1yGujH7AQaAP8BrhZktJ9E4GtgT3SGK4BkNQbuAX4AfBV4E/A/ZKaZ/wZWRFwwjRrWO5LRyj/zhu9nAj8NSL+GhFrI2IaMBM4HCAiHoqIf0TiSZKEsv9GxnF9RCyKiBXAPkDbiLgsIlZGxFskSe+EjG2tAn4VEauAu0gS0XURsTQi5gPzgfzR2KyIuCet/1uSZLtv+tUKGJPGMR14kCS5l5kaEc+kP6f/VBVMREyJiPfTOpOBN4C+eVXeiYhxEbEGmAB0ANqnSfUw4IyI+CQiVqU/b4DvA3+KiOcjYk1ETAC+SGO2RqJor0+YNVLHRMRjlcp2Ar4j6ai8si2AJwDSKcOfA7uQ/BG8NTBvI+NYVOn820v6d15ZU+CpjG39M00+ACvS7x/l7V9BkgjXOXdErE2ni7cv2xcRa/PqvkMycq0q7ipJOgk4F+iUFrUiSeJlPsw7/+fp4LIVyYj3XxHxSRXN7gScLOkneWVb5sVtjYATplnDtwiYGBHfr7wjnfK7FziJZHS1Kh2Zlk0hVnUb/HKSpFrma1XUyT9uEfB2RHTdkOA3wNfLNiQ1AToCZVPJX5fUJC9p7gi8nnds5f5WeC1pJ5LR8SDg2YhYI2k2X/681mcRsJ2kbSPi31Xs+1VE/CpDO1akPCVr1vDdDhwl6RBJTSVtld5M05FkFNMc+BhYnY42D8479iPgq5K2ySubDRye3sDyNeCcGs7/AvBZeiNQizSG7pL2qbMeVtRH0tD0Dt1zSKY2nwOeJ0n2F6bXNEuAo0imeavzEZB/fbQlSRL9GJIbpoDuWYKKiA9IbqL6vaT/SmMYmO4eB5whqZ8SLSUdIal1xj5bEXDCNGvgImIRyY0wPyX5Rb8IuABoEhFLgbOAu4FPSG56uT/v2NeAScBb6XXR7UluXJkDlJJc75xcw/nXkCSmXsDbwBLgJpKbZurDVGAYSX++BwxNrxeuBI4muY64BPg9cFLax+rcDOxedk04IhYAVwPPkiTTHsAztYjteyTXZF8judnqHICImElyHfN3adxvAiNr0a4VAT+4wMwaDEmXAF0i4sRCx2JWmUeYZmZmGThhmpmZZeApWTMzsww8wjQzM8vAn8NsxLbddtvo0qVLocOoc8uXL6dly5aFDqPOuV/Fxf0qPln6NmvWrCUR0baqfU6YjVj79u2ZOXNmocOoc7lcjpKSkkKHUefcr+LifhWfLH2T9E51+zwla2ZmloETppmZWQZOmGZmZhk4YZqZmWXghGlmZpaBE6aZmVkGTphmZmYZOGGamZll4IRpZmaWgROmmZlZBk6YZmZmGThhmpmZZeCEaWZmloETppmZWQZOmGZmZhk4YZqZmWXghGlmZg3WokWLOPDAA+nWrRt77LEH1113HQCzZ89m3333pVevXuy999688MILALz22mv079+f5s2bc9VVV9VpLM3qtLVGTNLXgGuBfYAvgFLgnIh4vY7aLwFWRsTf66I9M7PGoFmzZlx99dX07t2bpUuX0qdPHwYPHsyFF17Iz3/+cw477DD++te/cuGFF5LL5dhuu+24/vrrue++++o+ljpvsRGSJOAvwISIOCEt6wW0B+okYQIlwDJgnYQpqVlErK5tgytWraHT6IfqILSG5bweqxnpfhUN96u4NJR+lY45AoAOHTrQoUMHAFq3bk23bt147733kMRnn30GwKeffsr2228PQLt27WjXrh0PPVT3fXDCzOZAYFVE/LGsICJmK3ElcBgQwC8jYnI6Wjw/Io4EkPQ7YGZEjJdUCkwAjgK2AL4D/Ac4A1gj6UTgJ8CpwL+AvYDZko4E9ouIjyU1IUnU+0bEkk3QfzOzgistLeXll1+mX79+XHvttRxyyCGcf/75rF27lr//vf4n55wws+kOzKqifCjQC+gJtAFelDQjQ3tLIqK3pB+RJNbTJP0RWBYRVwFIOhXYBTgoItZI+jcwgmRa+CBgTlXJUtLpwOkAbdq05eIetR6YNnjtWyR/BTc27ldxcb/qVy6Xq/B6xYoVnH322Zx22mm89NJLXH/99Zx66qkccMABPPHEEwwdOpSrr766vH5paSktWrSo0M6yZcvWabc2nDA3zgBgUkSsAT6S9CTJNc7Pajjuz+n3WSRJtzpT0rYBbgGmkiTMUcCtVR0QEWOBsQA7du4SV89rfG/xeT1W434VD/eruDSUfpWOKCnfXrVqFUceeSRnnHEG5557LgBDhgzh3nvvRRIHHHAA11xzDSUlXx6Ty+Vo1arVOmX5r2ur8D+V4jAfOK6KclVTfzUV70DeqtL+L9Lva1j/e7C8bCMiFkn6SNK3gH4ko831arFFUxam1wEak1wuV+E/U2PhfhUX92vTiAhOPfVUunXrVp4sAbbffnuefPJJSkpKmD59Ol27dq33WJwws5kO/FrS9yNiHICkfYBPgGGSJgDbAQOBC0iuTe4uqTlJshwEPF3DOZYCX6mhzk3A7cDEvJGnmVmj9cwzzzBx4kR69OhBr169APj1r3/NuHHjOPvss1m9ejVbbbUVY8eOBeDDDz9k77335rPPPqNJkyZce+21LFiwgK98paZfrzVzwswgIkLSscC1kkaT3KRTCpwDtALmkNz0c2FEfAgg6W5gLvAG8HKG0zwA3CNpCMlNP1W5n2QqtsrpWDOzxmbAgAFERJX7Zs1a99aSr33ta7z77rv1EosTZkYR8T5wfBW7Lki/Kte/ELiwivJOedszST5OQvp5zj3zqj5Vxbl6ktzs81otQjczszrghFkk0pHtD8lw7dLMzOqeH41XJCJiTETsFBE1XQs1M7N64IRpZmaWgROmmZlZBk6YZmZmGThhmpmZZeCEaWZmloETppmZWQZOmGZmZhk4YZqZmWXghGlmZpaBE6aZmVkGTphmZtbgLFq0iAMPPJBu3bqxxx57cN111wEwbNgwevXqRa9evejUqVP5kl8rV67klFNOoUePHvTs2ZNcLlfnMW2WD1+XtAaYR9L/V4GTI+LzWhz/HeAy4MOIOLAO4rkMmBERj21sW2ZmjUGzZs24+uqr6d27N0uXLqVPnz4MHjyYyZMnl9c577zz2GabbQAYN24cAPPmzWPx4sUcdthhvPjiizRpUnfjws0yYQIrIqIXgKQ7gDOA35btlCRAEbG2muNPBX4UEU/URTARcXFdtFPZilVr6DT6ofpouqDO67Gake5X0XC/ikuh+1U65ggAOnToQIcOHQBo3bo13bp147333mP33XcHICK4++67mT59OgALFixg0KBBALRr145tt92WmTNn0rdv3zqLzVOyybqTXSR1kvSqpN8DLwFflzRc0jxJr0i6AkDSxcAA4I+SrpTUNP3+oqS5kn6Q1usgaYak2enx+6d1x6ev50n6f2nd8ZKOS7cHSXo53X+LpOZpeamkSyW9lO7brQA/KzOzTa60tJSXX36Zfv36lZc99dRTtG/fnq5duwLQs2dPpk6dyurVq3n77beZNWsWixYtqtM4VN1K1o2ZpGUR0UpSM+Be4GHgb8BbwH4R8Zyk7YHngD7AJ8CjwPURcZ+kHHB+RMyUdDrQLiJ+mSa3Z4DvAEOBrSLiV5KaAlsDuwBjImJwGse2EfFvSeOBB9OvN4BBEfG6pNuAlyLiWkmlwNURcYOkHwG9I+K0Kvp2OnA6QJs2bftcfO24uv8BFlj7FvDRikJHUffcr+LiftWPHjtsU+H1ihUrOPvssznxxBMZOHBgefk111zDDjvswPHHHw/AmjVr+OMf/8jLL79M+/btWbNmDUceeSQDBgwoP2bZsmW0atVqvec/8MADZ0XE3lXt21ynZFtImp1uPwXcDGwPvBMRz6Xl+wC5iPgYyqduBwL3VWrrYGDPshEisA3QFXgRuEXSFsB9ETFb0ltAZ0k3AA+RJOF8uwJvR8Tr6esJwI+Ba9PXf06/zyJJyOuIiLHAWIAdO3eJq+c1vrf4vB6rcb+Kh/tVXArdr9IRJeXbq1at4sgjj+SMM87g3HPPLS9fvXo1w4YNY9asWXTs2LG8vGxKFmC//fZj6NCh5VO4ALlcjpKSL9uvrcb3bmdTfg2zTHLZkuX5RRnbEvCTiHhknR3SQOAIYKKkKyPiNkk9gUNIEuHxwKhanPOL9PsaMrx3LbZoysL0ekBjksvlKvynaizcr+LiftWviODUU0+lW7duFZIlwGOPPcZuu+1WIVl+/vnnRAQtW7Zk2rRpNGvWrEKyrAuba8LM4nngOkltSKZkhwM3VFHvEeCHkqZHxCpJuwDvAW2A9yJinKSWQG9JfwVWRsS9kv4BjK/U1mtAJ0ldIuJN4HvAk/XSOzOzBuyZZ55h4sSJ9OjRo/yjI7/+9a85/PDDueuuuxg+fHiF+osXL+aQQw6hSZMm7LDDDkycOLHOY3LCrEZEfCDpf4AnSEZ+f42IqVVUvQnoBLyU3l37MXAMUAJcIGkVsAw4CdgBuFVS2c1W/1PpnP+RdAowJb2++iLwx7rum5lZQzdgwACqu8dm/Pjx65R16tSJhQsX1mtMm2XCjIh1rvpGRCnQvVLZncCdVdQtydteC/w0/co3If2qrHcV7Y3M234c2KuKOp3ytmeSJGQzM9tE/LESMzOzDJwwzczMMnDCNDMzy8AJ08zMLAMnTDMzswycMM3MzDJwwjQzM8vACdPMzCwDJ0wzM7MMnDDNzMwycMI0MzPLwAnTzGwDjRo1inbt2tG9+5ePoZ4zZw79+/enR48eHHXUUXz22WcArFy5klNOOYUePXrQs2dPcrlcgaK2DdVgE6YST0s6LK/seEkPb6Lzt5f0gqSXJe1Xad/TkhZKmpPW2XNTxJR3/tslHbMpz2lm6xo5ciQPP1zxV9Jpp53GmDFjmDdvHsceeyxXXnklAOPGjQNg3rx5TJs2jfPOO4+1a9du8phtwzXY1UoiIiSdQbLU1RNAU+BXwKEb066kZhGxOkPVwcC8iDi1mv3DImK2pO8DVwCHVVNvo9Qi3nWsWLWGTqMfquuQCu68HqsZ6X4VjcbYr9J0YfaBAwdSWlpaYd/ChQsZOHAgAIMHD+aQQw7hF7/4BQsWLGDQoEEAtGvXjm233ZaZM2fSt2/fTRq7bbgGO8IEiIhXgAeA/wZ+DtwWEf+QdHI6spst6fdl60tKGitppqT5ki4ua0fSu5L+V9IzwLH555C0s6QnJM2VNE1SR0l7A78Gjk7PseV6wnyWZJ3LsvYOk/SspJckTU4Xj0ZSv7R8jqTnJW0tqYWkCZLmpfUHpnVPk3SXpAeBv0lqkvZzgaQHSBanNrMGqHv37tx///0ATJkyhUWLFgHQs2dPpk6dyurVq3n77beZNWtW+T4rDg06YaYuBb5LMoL7jaTuJElvv4joRTJKPiGtOzoi9gZ6AoMl7Z7XzvKI+GZETKnU/u+BmyJiT2AKcG263uRlwB0R0SsiVq4nvkOB+wAktQNGA4MiojcwFzhb0lbAXcCPI6IncDDwBXAWsDIiegDfAybmJef+wPciYjBwHLAzyXqdPwQqTBGbWcNxyy23cOONN9KnTx+WLl3Kllsm/6VHjRpFx44d2XvvvTnnnHPYb7/9aNaswU7yWRUa/LsVEcslTQaWRcQXkg4C9gFmSgJoAZT9mTZc0qkk/doe2B1YkO6bXM0p+gFHptu3Ab/IGFrZ6FF8uSj0fuk5/57GtiXwNNAN+L+IeCnt06cAkgYAV6Zl8yW9D3RJ23o0Ij5JtwcCk9LFqt+VlKsuKEmnA6cDtGnTlot7bNBsboPWvkUyzdfYuF/FI5fLsWzZMnK5HB9++CHLly+vcBPPT3+arCe/aNEi2rVrV75vyJAhDBkyBIAzzzyTTz75pMHd/FPWr8ZoY/vW4BNmam36BUmCuiUi/je/gqSuwNlA34j4t6Tbga3yqiyv45iGkSTj3wA3AMensT0cEd+rFFtvIKpoQ+tpv3K8VR2/jogYC4wF2LFzl7h6XrG8xdmd12M17lfxaIz9Kh1RQi6Xo6SkhNLSUlq2bElJSQkAixcvpl27dqxdu5aRI0dywQUXUFJSwueff05E0LJlS6ZNm8Z2223HyJEjC9qPqpT1qzHa2L4V47/ix4B7JF0XEUskfRVoCXwFWAp8JqkDcAiQ5Y7a50iS3STgRGBG1kAiYqWknwJvSdoF+DtwnaTOEfFWOgLdHpgP7CSpd0S8JOkrJAlxBjACmCGpG9ABeJN1p1xnACdLuiOtcwBwS03xtdiiKQvTmxMak1wuR+mIkkKHUefcr+IzfPhwcrkcS5YsoWPHjlx66aUsW7aMG2+8EYChQ4dyyimnAEkiPeSQQ2jSpAk77LADEydOLGTotgGKLmFGxDxJlwKPpTf7rALOAGaSjPheAd4CnsnY5JnAzZL+B/gIOKWW8Xwu6RrgvIj4QTolPDnvWuRPI+INScOBP6TXM1cA3yIZmf5J0ry0HyelSbjyae4BDkz7tpBaJHUzqz+TJk2qsvzss89ep6xTp04sXLiwvkOyelQUCTMiLqn0+k7gziqqfq+KMiKi43rafoskGVUuv2k9xwyo9PqKvO1pwLQqjnmO5HppZSfVdO702uUPq4vHzMzqXzHcJWtmZlZwTphmZmYZOGGamZll4IRpZmaWgROmmZlZBk6YZmZmGThhmpmZZeCEaWZmloETppmZWQZOmGZmZhk4YZqZmWXghGlmZpaBE6aZNVqjRo2iXbt2dO/evbxs2LBh9OrVi169etGpUyd69eoFwB133FFe3qtXL5o0acLs2bMLFbo1QE6YBSLpWEkhabdCx2LWWI0cOZKHH664LO7kyZOZPXs2s2fP5tvf/jZDhw4FYMSIEeXlEydOrJBMzaBIlvdqpIYDTwMnAJfUxwlWrFpDp9EP1UfTBXVej9WMdL+KRiH6VZounD5w4EBKS0urrBMR3H333UyfPn2dfZMmTWL48OH1GaIVIY8wC0BSK+CbwKkkCRNJTST9XtJ8SQ9K+quk49J9fSQ9KWmWpEckdShg+GaNwlNPPUX79u3p2rXrOvsmT57shGnr8AizMI4BHo6I1yX9S1JvoDPQCegBtANeBW6RtAVwAzAkIj6WNAz4FTCqqoYlnQ6cDtCmTVsu7rG63juzqbVvkYxaGhv3q+7kcrny7Q8//JDly5dXKAO45ppr6Nu37zrlCxYsICJYsmTJOvvyLVu2bL37i1Vj7RdsfN+cMAtjOHBtun1X+noLYEpErAU+lPREun9XoDswTRJAU+CD6hqOiLHAWIAdO3eJq+c1vrf4vB6rcb+KRyH6VTqi5Mvt0lJatmxJScmXZatXr2bYsGHMmjWLjh07Vjh26tSpnHbaaRXqVyWXy9VYpxg11n7Bxvet8f3vbOAkfRX4FtBdUpAkwAD+Ut0hwPyI6L+JQjRr9B577DF22223dZLl2rVrmTJlCjNmzChQZNaQ1TphSvov4OsRMbce4tkcHAfcFhE/KCuQ9CSwBPi2pAlAW6AEuBNYCLSV1D8ink2naHeJiPk1najFFk1ZmN780JjkcrkKI4jGwv2qe8OHDyeXy7FkyRI6duzIpZdeyqmnnspdd91V5TXKGTNm0LFjRzp37lyAaK2hy5QwJeWAo9P6s4GPJT0ZEefWY2yN1XBgTKWye4FuwLvAK8DrwPPApxGxMr3553pJ25C8B9cCNSZMs83dpEmTqiwfP358leUlJSU899xz9RiRFbOsI8xtIuIzSacBt0bEzyV5hLkBIqKkirLrIbl7NiKWpdO2LwDz0v2zgYGbMk4zM6soa8Jsln6U4XjgonqMZ3P3oKRtgS2BX0TEh4UOyMzMElkT5mXAI8AzEfGipM7AG/UX1uapqtGnmZk1DJkSZkRMAabkvX4L+HZ9BWVmZtbQZHrSj6RdJD0u6ZX09Z6Sfla/oZmZmTUcWR+NNw74H2AVQPqRkhPqKygzM7OGJmvC3DoiXqhU1vie4WVmZlaNrAlziaRvkDyRhvRzgdU+ns3MzKyxyXqX7I9Jnk+6m6T3gLeBEfUWlZmZWQNTY8KU1ATYOyIOktQSaBIRS+s/NDMzs4ajxinZdPWMM9Pt5U6WZma2Ocp6DXOapPMlfV3SdmVf9RqZmZlZA5L1GmbZYsU/zisLkkWPzczMGr1MI8yI2LmKLydLM6sXo0aNol27dnTv3r1C+Q033MCuu+7KHnvswYUXXlhefvnll9OlSxd23XVXHnnkkU0drm0msi7vdVJV5RFxW92Gs3EkrSFZ4UPAGuDMiPh7LY4fDzwYEfdkrJ8Dzo+ImenrTunx3ddzzPbA9RFxXA1tL4uIVlWUHwO8HhELssRoVoxGjhzJmWeeyUknffmr54knnmDq1KnMnTuX5s2bs3jxYgAWLFjAXXfdxfz583n//fc56KCDeP3112natGmhwrdGKuuU7D5521sBg4CXgAaVMIEVEdELQNIhwOXAAYUNqaKIeJ9kEekNdQzwIFBjwlyxag2dRj+0EadqmM7rsZqR7lfRqE2/StMFzwcOHEhpaWmFfX/4wx8YPXo0zZs3B6Bdu3YATJ06lRNOOIHmzZuz884706VLF1544QX69+9fd50wI/uU7E/yvr4P7EWyBFVD9hXgEwAlrpT0iqR5kobllf9O0gJJDwHt0vJBkv5S1pCkwZL+XJuTS2qanvNFSXMl/SAt75T3TN6tJd2d7p8s6XlJe+e18StJcyQ9J6m9pP1IFvK+UtLs9GESZpuF119/naeeeop+/fpxwAEH8OKLLwLw3nvv8fWvf728XseOHXnvvfcKFaY1YllHmJV9DnSty0DqSAtJs0lGwR2Ab6XlQ4FeQE+gDfCipBlAf2BXoAfQnmTUdgswHbhRUtuI+Bg4Bbi1mnPeIWlFur0lsDbdPhX4NCL2kdQceEbSo6RPS0r9CPgkIvaU1B2YnbevJfBcRFwk6TfA9yPil5LuZz3TxpJOB04HaNOmLRf3aHxPMGzfIhm1NDbuF+RyufLtDz/8kOXLl5eXffrpp8ybN48xY8bw2muvcfTRR3PnnXfy7rvv8uqrr5bX++CDD5g/fz5t2rSp455UtGzZsgrxNhaNtV+w8X3Leg3zAb78Rd8E2J285b4akPwp2f7AbWkiGgBMiog1wEeSniSZZh6YV/6+pOkAERGSJgInSrqVJLFWeR0XGFH5GmZafjCwZ/oYQYBtSP7IeD3v2AHAdek5X5E0N2/fyry2ZgGDs/wAImIsyVOZ2LFzl7h63ob+TdRwnddjNe5X8ahNv0pHlHy5XVpKy5YtKSlJynbddVfOOussSkpKOPDAA7nqqqvo3r07/fr1Ayivd/nll3PwwQfX+5RsLpcrP2dj0lj7BRvft6z/O6/K214NvBMR727wWTeBiHhWUhugLclNQNVWrab8VuAB4D/AlIio7Z/+An4SERVu2UuTan6d6qyKiLLY1rABswEttmjKwvSaUGOSy+Uq/GJtLNyv9TvmmGOYPn06JSUlvP7666xcuZI2bdpw9NFH893vfpdzzz2X999/nzfeeIO+fftufOBmlWR9cMHhEfFk+vVMRLwr6Yp6jWwjSdoNaAr8E5gBDEuvK7YlGVm+kJafkJZ3AA4sOz69Oed94GfA+A0I4RHgh5K2SOPZJX20YL6ngePT/buTTA3XZCnQegPiMSsaw4cPp3///ixcuJCOHTty8803M2rUKN566y26d+/OCSecwIQJE5DEHnvswfHHH8/uu+/OoYceyo033ug7ZK1eZB21DAb+u1LZYVWUFVrZNUxIRm8nR8Sa9Aae/sAckhHlhRHxYVr+LZKPorwOPFmpvTuAthv4EY6bgE7AS5IEfExyh2u+3wMT0qnYl4G5wKc1tHsXME7SWcBxEfGPDYjNrEGbNGlSleW33357leUXXXQRF110UX2GZLb+hCnphyQ3pnSudH2tNfBMfQa2ISKiyj8r06nNC9KvyuVnrqfJASSLZ1d3vpJKr0uB7un2WuCn6Ve+T8vqkEz3nhgR/0nveH0ceCc9vvwzmOkNPvek28+QXEM2M7NNqKYR5p3A30g+zzg6r3xpRPyr3qJqACTNApYD59XjabYGnkinbQX8MCJW1uP5zMxsA603YUbEpyQjouEAktqRfGSjlaRWEfF/9R9iYUREn01wjqXA3jVWNDOzgst004+koyS9QbJw9JNAKcnI08zMbLOQ9S7ZXwL7kjzDdGeSR+M1uG1cuj0AABjbSURBVGuYZmZm9SVrwlwVEf8EmkhqEhFPkDw5x8zMbLOQ9WMl/5bUCniK5FFwi0keYGBmZrZZyDrCHELy/NhzgIeBfwBH1VdQZmZmDU2mEWZELJe0E9A1IiZI2prkKTpmZmabhax3yX6f5IPzf0qLdgDuq6+gzMzMGpqsU7I/Br4JfAYQEW+Qrh1pZma2OciaML/IfwKNpGZUv8qHmZlZo5M1YT4p6ackDzcfTLIW5gP1F5aZmVnDkvVjJaOBU0lW9fgB8FeS1TjMGqWFCxcybNiw8tdvvfUWl112Ge+99x4PPPAAW265Jd/4xje49dZb2XbbbQsYqZltKusdYUraEZKVNyJiXER8JyKOS7eLZkpW0kWS5kuaK2m2pH5p+U3pOpT1cc62kp6X9LKk/Svt20LSGElvSHpF0guSDkv3/VXStunXj+ojNqvZrrvuyuzZs5k9ezazZs1i66235thjj2Xw4MG88sorzJ07l1122YXLL7+80KGa2SZS0wjzPqA3gKR7I+Lb9R9S3ZLUHzgS6B0RX0hqA2wJEBGn1eOpBwGvRcTJVez7BdAB6J7G1B44II3p8DTuTiRLq/1+QwNYsWoNnUY/tKGHN1jn9VjNyHrsV+mYIyq8fvzxx/nGN77BTjvtxE477VRevu+++3LPPffUWxxm1rDUdA1Tedud6zOQetQBWBIRXwBExJKIeB9AUk7S3pKOTkeesyUtlPR2ur+PpCclzZL0iKQOlRuXtJOkx9PR6+OSdpTUC/gNcHjaZou8+lsD3wd+khfTRxFxd7q/NE3qY4BvpMdfKWmipCF57dwh6eh6+plZnrvuuovhw4evU37LLbdw2GGHFSAiMysErW9mVdJLEdG78nYxSR/p9zTJ2pOPAZMj4sl0Xw44PyJm5tW/m2RFlrHp9yER8bGkYcAhETGqUvsPAPekD3QYBRwdEcdIGgnsHRFnVqq/JzAhIvaqJt5SkiW/WgEPRkT3tPwA4P+lbW8DzCZ5kMTqSsefDpwO0KZN2z4XX1vt+tdFq30L+GhF/bXfY4dtyrdXrVrFcccdx6233sp2221XXn777bezcOFCLrvsMiRV1UytLVu2jFatWtVcsci4X8WlsfYLsvXtwAMPnBURVS67WNOUbE9Jn5GMNFuk26SvIyK+UtuAN7WIWCapD7A/cCAwWdLoiBhfua6kC4EVEXGjpO5Ad2Ba+guxKfBBFafoDwxNtyeSjCzrXEQ8KenGdE3SocC9lZNlWm8sSbJnx85d4up5We/rKh7n9VhNffardERJ+fbUqVPp168fQ4cOLS+bMGEC8+fP5/HHH2frrbeus/PmcjlKSkpqrFds3K/i0lj7BRvft5oWkG4Uj7+LiDVADshJmgecDIzPryNpEPAdYGBZETA/IvrX9nQ17H8T2FFS63QB6dqYCIwATgBG1VCXFls0ZWGl63GNQS6Xq5DU6tOkSZMqTMc+/PDDXHHFFTz55JN1mizNrOHL+jnMoiVpV0ld84p6Ae9UqrMTyc01x0dE2WTfQqBtetNQ2Z2te1Rxir+TJDBIktnT64snIj4Hbgaul7Rl2nYHSSdWqroUaF2pbDzJA/CJiPnrO49tvM8//5xp06ZVGF2eeeaZLF26lMGDB9OrVy/OOOOMAkZoZptS45uvW1cr4AZJ25IsSfYm6TW+PCOBrwJ/Sadf34+IwyUdR5LYtiH5WV0LVE5UZwG3SLoA+Bg4JUNMPyNZlHuBpP8Ay4GL8ytExD8lPSPpFeBvEXFBRHwk6VX8HN9NYuutt+af//xnhbI333yzQNGYWaE1+oQZEbOA/arZV5JuzgQurWL/bL6coq2u/VLgW1WUj6fStG/evpXAhelX5X2d8ra/m78vvcO2KzBpfTGZmVnda/RTso2FpIOA14AbIuLTQsdjZra5afQjzMYiIh4Ddix0HGZmmyuPMM3MzDJwwjQzM8vACdPMzCwDJ0wzM7MMnDDNzMwycMI0MzPLwAnTzMwsAydMMzOzDJwwzczMMnDCtAZtzZo17LXXXhx55JEAjBgxgpNOOonu3bszatQoVq1aVeAIzWxzUS8JU1KndJWN/LJLJJ1fw3F7S7q+PmLKO8ff82L8bk31Kx1bIulTSS9LWihphqQj6ydSA7juuuvo1q1b+esRI0YwYcIE5s2bx4oVK7jpppsKGJ2ZbU4a1LNkI2ImycohG0VSs4hYXc05ylYu6QR8F7izls0/FRFHpufpBdwnaUVEPL6h8aZtCVBErN2YdvKtWLWGTqMfqqvmNonSvAWv3333XR566CEuuugifvvb3wJw+OGHk8vlkETfvn159913CxWqmW1mCjIlKykn6QpJL0h6XdL+aXmJpAclNZFUmq5hWXbMm5LaS2or6V5JL6Zf30z3XyJprKRHgdsk7ZG2P1vS3LJFpCUtS5scA+yf7v9/kp5KE2DZ+Z6RtOf6+pEu/3UZcGZ6THWxtZU0TdJLkv4k6R1JbdJR7quSfg+8BHxd0sGSnk3rTpHUKm2jj6QnJc2S9IikDnXyZjRg55xzDr/5zW9o0mTdf6arVq1i4sSJHHrooQWIzMw2R4W8htksIvoC5wA/z9+RjrKmAscCSOoHlEbER8B1wDURsQ/wbSB/Tq4PMCRdR/IM4LqI6AXsDVQeiowmGS32iohr0nZGpufbBWgeEXMz9OMlYLd0u7rYfg5Mj4jewF+ouOrIrsBtEbEXyULSPwMOSuvOBM6VtAVwA3BcRPQBbgF+lSG2ovXggw/Srl07+vTpU+X+H/3oRwwcOJD9999/E0dmZpur+pqSjQzlf06/zyKZHq1sMnAxcCtwQvoa4CBg92QGE4CvSGqdbt8fESvS7WeBiyR1BP4cEW/UEPMU4H8lXQCMoprFn6ugvO3qYhtAmvwj4mFJn+Qd805EPJdu7wvsDjyTtrFl2o9dge7AtLS8KfBBlcFIpwOnA7Rp05aLe1Q5M91g5XI5ACZNmsSjjz7Kn//8Z1auXMnnn3/O4MGDueiiixg3bhzvvPMOl112WXn9xmDZsmWNqj9l3K/i0lj7BRvft/pKmP8E/qtS2XbA23mvv0i/r6kmjmeBLpLaAscAv0zLmwD98xIjAGkiWV72OiLulPQ8cATwiKTTImJ6dQFHxOeSpgFDgONJRqVZ7AW8WkNsWueoLy3P2xYwLSKGVzq+BzA/IvrXFExEjAXGAuzYuUtcPa9BXaauUemIEgBKSkrKy3K5HFdddRUPPvggN910E3PmzOHFF1+kRYsWhQmynuRyuQr9bizcr+LSWPsFG9+3evltGhHLJH0gaVBEPC5pO+BQkinLrG2EpL8AvwVejYh/prseJblmeCUkN96k1xIrkNQZeCsirk+39wTyE+ZSoHWlw24CHiCZqv1XTTGm1zj/FzithtieJknCV0g6mHX/mCjzHHCjpC4R8aakrYGOwEKgraT+EfFsOkW7S0TMX198LbZoysK8m2gagzPOOIP27dvTv3/yt8PQoUO5+OKLCxyVmW0O6nP4cRLJL/+r09eXRsQ/atnGZOBF0muLqbPSdueSxD+D5HplZcOAEyWtAj4kuTkn31xgtaQ5wPiIuCYiZkn6jGQauDr7S3oZ2BpYDJyVd4dsdbFdCkySNAx4kmQ6dSnQKr/hiPhY0si0bvO0+GcR8bqk44DrJW2Ttn0tsN6E2ViUlJSU/1W4evXqRv0XsJk1XPWWMCNiAXBgNftK8raXkF7DjIgckMvbN5OK1wjL6g+ros1LKr2+HLi8inqt0u+rgEH5+yRtTzKt+mg1ceeAbarat77YgE+BQyJitaT+wIER8QVQSnJtMr+N6cA+VbQ9GxhY3bnNzKx+FdcFrnok6SSSO0/PrcvPQqZ2BO6W1ARYCXy/jts3M7N65oSZiojbgNvqqe03SG4OMjOzIuVnyZqZmWXghGlmZpaBE6aZmVkGTphmZmYZOGGamZll4IRpZmaWgROmmZlZBk6YZmZmGThhmpmZZeCEaWZmloETplUwatQo2rVrR/fuXz4T/l//+heDBw+ma9euDB48mE8++WQ9LZiZNU5OmLUkqb2kOyW9JWmWpGclHVsH7ZZIerAuYtwYI0eO5OGHH65QNmbMGAYNGsQbb7zBoEGDGDNmTIGiMzMrHD98vRYkCbgPmBAR303LdgKOLkAszSJi9frqrFi1hk6jH6qxrdK8RaYHDhxIaWlphf1Tp04ll8sBcPLJJ1NSUsIVV1xR+6DNzIqYR5i18y1gZUT8sawgIt6JiBskNZV0paQXJc2V9AMoHznmJN0j6TVJd6SJF0mHpmVPA0PL2pTUUtItaVsvSxqSlo+UNEXSA1SzZmd9+Oijj+jQoQMAHTp0YPHixZvq1GZmDYZHmLWzB/BSNftOBT6NiH0kNQeekVSW1PZKj30feAb4pqSZwDiSJPwmMDmvrYuA6RExStK2wAuSHkv39Qf2jIh/VRWEpNOB0wHatGnLxT3WOwgFKB89lvnwww9Zvnx5efnq1asr1Kn8elNbtmxZQc9fX9yv4uJ+FZ+N7ZsT5kaQdCMwgGRR6HeAPSUdl+7eBuia7nshIt5Nj5kNdAKWAW+na2Ui6XbSRAccDBwt6fz09VYki1ADTKsuWQJExFhgLMCOnbvE1fNqfotLR5RUfF1aSsuWLSkpScp32GEHdt11Vzp06MAHH3zA9ttvX76vEHK5XEHPX1/cr+LifhWfje2bE2btzAe+XfYiIn4sqQ0wE/g/4CcR8Uj+AZJKgC/yitbw5c89qjmPgG9HxMJKbfUDlmcNtsUWTVmYd31yQx199NFMmDCB0aNHM2HCBIYMGbLRbZqZFRtfw6yd6cBWkn6YV7Z1+v0R4IeStgCQtIuklutp6zVgZ0nfSF8Pz9v3CPCTvGude9VJ9BkMHz6c/v37s3DhQjp27MjNN9/M6NGjmTZtGl27dmXatGmMHj16U4VjZtZgeIRZCxERko4BrpF0IfAxyYjvv4EpJFOtL6WJ7mPgmPW09Z/0euNDkpYATwNlH378BXAtMDdtqxQ4sl46VcmkSZOqLH/88cc3xenNzBosJ8xaiogPgBOq2f3T9CtfLv0qO/7MvO2Hgd2qOMcK4AdVlI8HxtcuYjMzqwuekjUzM8vACdPMzCwDJ0wzM7MMnDDNzMwycMI0MzPLwAnTzMwsAydMMzOzDJwwzczMMnDCNDMzy8AJ08zMLAMnTDMzswycMM3MzDLww9c3Y506daJ169Y0bdqUZs2aMXPmzEKHZGbWYG3WCVPSsoholfd6JLB3/ooiVRxzNLB7RIzJ0H5f4CqgPcli0U8DZ0XE59XUr/H8de2JJ56gTZs2m+p0ZmZFa7NOmBsiIu4H7q9cLqlZRKzOe92eZI3MEyLi2XRdy28DrYEqE2ZtVT5nZStWraHT6IcqlJWOOaIuTm1mttnxNcxqSDpK0vOSXpb0WJoAkTRS0u/S7fGSfivpCeCKSk38GJgQEc9Csvh0RNwTER9J2k7SfZLmSnpO0p5VnH8nSY+ndR6XtGOGc9a2jxx88MH06dOHsWPHbkxTZmaN3uY+wmwhaXbe6+34cvT4NLBvRISk04ALgfOqaGMX4KCIWFOpvDswoZrzXgq8HBHHSPoWcBvQq1Kd3wG3RcQESaOA64Fjajgnkk4HTgdo06YtF/eoOADN5XLl21deeSVt2rThk08+4fzzz2fFihX07NmzmpAbjmXLllXoR2PhfhUX96v4bGzfNveEuSIiyhNV2TXE9GVHYLKkDsCWwNvVtDGlqsRVgwEk07NExHRJX5W0TaU6/YGh6fZE4DdZzhkRY4GxADt27hJXz6v4FpeOKKkyoDlz5rBq1SpKSqre35DkcrmiiLO23K/i4n4Vn43t2+aeMNfnBuC3EXG/pBLgkmrqLa+mfD7QB5haxT5VURY1xJO/v7pzVtBii6YsrOaa5fLly1m7di2tW7dm+fLlPProo1x88cVZmjUz2yz5Gmb1tgHeS7dP3oDjfwecLKlfWYGkEyV9DZgBjEjLSoAlEfFZpeP/DpyQbo8gmSKuMx999BEDBgygZ8+e9O3blyOOOIJDDz20Lk9hZtaoeIRZvUuAKZLeA54Ddq7NwenNPScAV0lqB6wlSZR/Ttu+VdJckjtmq0rIZwG3SLoA+Bg4ZQP7UaXOnTszZ86cumzSzKxR26wTZv5nMNPX44Hx6fZUqphOrVRnZA3tPwvsX8Wuz4EhNbRdCnyrijrrPaeZmdUPT8mamZll4IRpZmaWgROmmZlZBk6YZmZmGThhmpmZZeCEaWZmloETppmZWQZOmGZmZhk4YZqZmWXghGlmZpaBE6aZmVkGTphmZmYZOGGamZll4IRpZmaWgROmmZlZBk6YZmZmGSgiCh2D1RNJS4GFhY6jHrQBlhQ6iHrgfhUX96v4ZOnbThHRtqodzeo+HmtAFkbE3oUOoq5Jmul+FQ/3q7g01n7BxvfNU7JmZmYZOGGamZll4ITZuI0tdAD1xP0qLu5XcWms/YKN7Jtv+jEzM8vAI0wzM7MMnDDNzMwycMJshCQdKmmhpDcljS50PBtDUqmkeZJmS5qZlm0naZqkN9Lv/1XoOLOQdIukxZJeySursi9KXJ++h3Ml9S5c5OtXTb8ukfRe+r7NlnR43r7/Sfu1UNIhhYm6ZpK+LukJSa9Kmi/p7LS8qN+z9fSrqN8zSVtJekHSnLRfl6blO0t6Pn2/JkvaMi1vnr5+M93fqcaTRIS/GtEX0BT4B9AZ2BKYA+xe6Lg2oj+lQJtKZb8BRqfbo4ErCh1nxr4MBHoDr9TUF+Bw4G+AgH2B5wsdfy37dQlwfhV1d0//TTYHdk7/rTYtdB+q6VcHoHe63Rp4PY2/qN+z9fSrqN+z9OfeKt3eAng+fR/uBk5Iy/8I/DDd/hHwx3T7BGByTefwCLPx6Qu8GRFvRcRK4C5gSIFjqmtDgAnp9gTgmALGkllEzAD+Vam4ur4MAW6LxHPAtpI6bJpIa6eaflVnCHBXRHwREW8Db5L8m21wIuKDiHgp3V4KvArsQJG/Z+vpV3WK4j1Lf+7L0pdbpF8BfAu4Jy2v/H6VvY/3AIMkaX3ncMJsfHYAFuW9fpf1/2do6AJ4VNIsSaenZe0j4gNI/vMD7QoW3carri+N4X08M52avCVv2rwo+5VO1+1FMmppNO9ZpX5Bkb9nkppKmg0sBqaRjIb/HRGr0yr5sZf3K93/KfDV9bXvhNn4VPUXUjF/duibEdEbOAz4saSBhQ5oEyn29/EPwDeAXsAHwNVpedH1S1Ir4F7gnIj4bH1VqyhrsH2rol9F/55FxJqI6AV0JBkFd6uqWvq91v1ywmx83gW+nve6I/B+gWLZaBHxfvp9MfAXkv8EH5VNdaXfFxcuwo1WXV+K+n2MiI/SX15rgXF8OYVXVP2StAVJUrkjIv6cFhf9e1ZVvxrLewYQEf8GciTXMLeVVPbc9PzYy/uV7t+GGi4tOGE2Pi8CXdM7w7YkuZh9f4Fj2iCSWkpqXbYNHAy8QtKfk9NqJwNTCxNhnaiuL/cDJ6V3Xu4LfFo2DVgMKl27O5bkfYOkXyekdyjuDHQFXtjU8WWRXs+6GXg1In6bt6uo37Pq+lXs75mktpK2TbdbAAeRXJ99AjgurVb5/Sp7H48Dpkd6B1C1Cn1nk7/q5W6xw0nufPsHcFGh49mIfnQmuTtvDjC/rC8k1xkeB95Iv29X6Fgz9mcSyVTXKpK/bk+tri8k00U3pu/hPGDvQsdfy35NTOOem/5i6pBX/6K0XwuBwwod/3r6NYBkim4uMDv9OrzY37P19Kuo3zNgT+DlNP5XgIvT8s4kCf5NYArQPC3fKn39Zrq/c03n8KPxzMzMMvCUrJmZWQZOmGZmZhk4YZqZmWXghGlmZpaBE6aZmVkGzWquYmb2JUlrSD5+UOaYiCgtUDhmm4w/VmJmtSJpWUS02oTnaxZfPgvUrGA8JWtmdUpSB0kz0jUVX5G0f1p+qKSX0vUKH0/LtpN0X/rA7+ck7ZmWXyJprKRHgdvSh2pfKenFtO4PCthF20x5StbMaqtFuiIEwNsRcWyl/d8FHomIX0lqCmwtqS3J80kHRsTbkrZL614KvBwRx0j6FnAbycO/AfoAAyJiRbpSzacRsY+k5sAzkh6NZLkps03CCdPMamtFJCtCVOdF4Jb0Ad/3RcRsSSXAjLIEFxFlD7keAHw7LZsu6auStkn33R8RK9Ltg4E9JZU9E3QbkmeaOmHaJuOEaWZ1KiJmpMuwHQFMlHQl8G+qXjppfUssLa9U7ycR8UidBmtWC76GaWZ1StJOwOKIGEeyKkZv4FnggHS1C/KmZGcAI9KyEmBJVL3m5CPAD9NRK5J2SVewMdtkPMI0s7pWAlwgaRWwDDgpIj5Or0P+WVITkjUkBwOXALdKmgt8zpfLLVV2E9AJeCldnupj4Jj67IRZZf5YiZmZWQaekjUzM8vACdPMzCwDJ0wzM7MMnDDNzMwycMI0MzPLwAnTzMwsAydMMzOzDP4/5V69exx0VjIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xg_reg = xgb.train(params=params, dtrain=data_dmatrix, num_boost_round=10)\n",
    "xgb.plot_importance(xg_reg)\n",
    "plt.rcParams['figure.figsize'] = [5, 5]\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold, RandomizedSearchCV, GridSearchCV\n",
    "kfold = KFold(n_splits =5, shuffle=False)\n",
    "rmse_scores = []\n",
    "folds = 3\n",
    "param_comb = 5\n",
    "params = {\n",
    "        'min_child_weight': [1, 5, 10],\n",
    "        'gamma': [0.5, 1, 1.5, 2, 5],\n",
    "        'subsample': [0.6, 0.8, 1.0, 0.75],\n",
    "        'colsample_bytree': [0.6, 0.8, 1.0, 0.85],\n",
    "        'max_depth': [3, 4, 5, 6, 7, 8],\n",
    "        'learning_rate': [0.1, 0.2, 0.5, 0.05],\n",
    "        'n_estimator': [80, 100, 70, 60, 200]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vansh\\Anaconda3\\lib\\site-packages\\xgboost\\core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:52:20] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Time elapsed: 138.99601340293884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "53962.645469043535"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import statistics\n",
    "begin = time()\n",
    "\n",
    "model = xgb.XGBRegressor(objective=\"reg:linear\", random_state=145, learning_rate = 0.1, max_depth = 7, subsample= 0.75, \n",
    "                         colsample_bytree = 0.9, n_estimators=80)\n",
    "# random_search = GridSearchCV(model, param_grid=params, n_jobs=-1)\n",
    "# for train_idx, test_idx in kfold.split(train):\n",
    "#     X_train, y_train, X_valid, y_valid = train[train_idx], train_income[train_idx], train[test_idx], train_income[test_idx]\n",
    "lmodel = model.fit(X_, y_train)\n",
    "y_pred = lmodel.predict(test_)\n",
    "# rmse_scores.append(math.sqrt(mean_squared_error(y_test, y_pred)))\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))\n",
    "# y_pred = cross_val_predict(lmodel, X_test, cv=10)\n",
    "# y_pred = lmodel.predict(X_test)\n",
    "math.sqrt(mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:59:29] WARNING: C:/Jenkins/workspace/xgboost-win64_release_0.90/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Time elapsed: 186.737957239151\n"
     ]
    }
   ],
   "source": [
    "begin = time()\n",
    "model = xgb.XGBRegressor(objective=\"reg:linear\", random_state=140, learning_rate = 0.1, max_depth = 7, subsample= 0.75, \n",
    "                         colsample_bytree = 0.9, n_estimators=80)\n",
    "lmodel = model.fit(new_train, train_income)\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))\n",
    "y_pred = lmodel.predict(new_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import neighbors\n",
    "begin = time()\n",
    "model = neighbors.KNeighborsRegressor(n_neighbors = 107)\n",
    "lmodel = model.fit(X_train, y_train)\n",
    "end = time()\n",
    "print(\"Time elapsed: {}\".format(end-begin))\n",
    "y_pred = lmodel.predict(X_test)\n",
    "math.sqrt(mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "params = {'n_neighbors':[21, 22,23,24, 25,26,27,28,29,30 ]}\n",
    "\n",
    "knn = neighbors.KNeighborsRegressor()\n",
    "\n",
    "model = GridSearchCV(knn, params, cv=5)\n",
    "model.fit(X_train,y_train)\n",
    "model.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.6920059 , 0.68275332, 0.62374457, 0.66352634, 0.55277182,\n",
       "       0.70031145, 0.64329849, 0.69199835, 0.65355293, 0.68163334,\n",
       "       0.68652761, 0.63072705, 0.56980449, 0.67705928, 0.68301221,\n",
       "       0.75316579, 0.7055189 , 0.61963735, 0.73879632, 0.64324346,\n",
       "       0.66227077, 0.71761778, 0.6273328 , 0.75427479, 0.6517717 ,\n",
       "       0.63206473, 0.60323423, 0.71242704, 0.64507502, 0.65246756,\n",
       "       0.72151913, 0.64174699, 0.61614238, 0.68347153, 0.67370904,\n",
       "       0.59182153, 0.66024207, 0.71152851, 0.76493018, 0.69579225,\n",
       "       0.74576291, 0.73522636, 0.70760334, 0.64216747, 0.72974338,\n",
       "       0.7285919 , 0.62451864, 0.51862191, 0.6363468 ])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "regressor = DecisionTreeRegressor(random_state=0)\n",
    "cross_val_score(regressor, train, train_income, cv=49)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Instance</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111994</th>\n",
       "      <td>35552.507812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111995</th>\n",
       "      <td>11573.395508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111996</th>\n",
       "      <td>28929.787109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111997</th>\n",
       "      <td>114239.734375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111998</th>\n",
       "      <td>29116.521484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Income\n",
       "Instance               \n",
       "111994     35552.507812\n",
       "111995     11573.395508\n",
       "111996     28929.787109\n",
       "111997    114239.734375\n",
       "111998     29116.521484"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submit[\"Income\"] = y_pred\n",
    "submit.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit.to_csv(r\"C:\\Users\\vansh\\TCD MAI\\Machine Learning\\IndividualAssignment\\MLbasic\\prediction.csv\", index = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}