partie_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PARTIE 1 : Prétraitement & nettoyage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore', category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>idg</th>\n",
       "      <th>condtn</th>\n",
       "      <th>wave</th>\n",
       "      <th>round</th>\n",
       "      <th>position</th>\n",
       "      <th>positin1</th>\n",
       "      <th>order</th>\n",
       "      <th>partner</th>\n",
       "      <th>pid</th>\n",
       "      <th>match</th>\n",
       "      <th>int_corr</th>\n",
       "      <th>samerace</th>\n",
       "      <th>age_o</th>\n",
       "      <th>race_o</th>\n",
       "      <th>pf_o_att</th>\n",
       "      <th>pf_o_sin</th>\n",
       "      <th>pf_o_int</th>\n",
       "      <th>pf_o_fun</th>\n",
       "      <th>pf_o_amb</th>\n",
       "      <th>pf_o_sha</th>\n",
       "      <th>dec_o</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>like_o</th>\n",
       "      <th>prob_o</th>\n",
       "      <th>met_o</th>\n",
       "      <th>age</th>\n",
       "      <th>field</th>\n",
       "      <th>field_cd</th>\n",
       "      <th>undergra</th>\n",
       "      <th>mn_sat</th>\n",
       "      <th>tuition</th>\n",
       "      <th>race</th>\n",
       "      <th>imprace</th>\n",
       "      <th>imprelig</th>\n",
       "      <th>from</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>income</th>\n",
       "      <th>goal</th>\n",
       "      <th>date</th>\n",
       "      <th>go_out</th>\n",
       "      <th>career</th>\n",
       "      <th>career_c</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>expnum</th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>attr4_1</th>\n",
       "      <th>sinc4_1</th>\n",
       "      <th>intel4_1</th>\n",
       "      <th>fun4_1</th>\n",
       "      <th>amb4_1</th>\n",
       "      <th>shar4_1</th>\n",
       "      <th>attr2_1</th>\n",
       "      <th>sinc2_1</th>\n",
       "      <th>intel2_1</th>\n",
       "      <th>fun2_1</th>\n",
       "      <th>amb2_1</th>\n",
       "      <th>shar2_1</th>\n",
       "      <th>attr3_1</th>\n",
       "      <th>sinc3_1</th>\n",
       "      <th>fun3_1</th>\n",
       "      <th>intel3_1</th>\n",
       "      <th>amb3_1</th>\n",
       "      <th>attr5_1</th>\n",
       "      <th>sinc5_1</th>\n",
       "      <th>intel5_1</th>\n",
       "      <th>fun5_1</th>\n",
       "      <th>amb5_1</th>\n",
       "      <th>dec</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "      <th>like</th>\n",
       "      <th>prob</th>\n",
       "      <th>met</th>\n",
       "      <th>match_es</th>\n",
       "      <th>attr1_s</th>\n",
       "      <th>sinc1_s</th>\n",
       "      <th>intel1_s</th>\n",
       "      <th>fun1_s</th>\n",
       "      <th>amb1_s</th>\n",
       "      <th>shar1_s</th>\n",
       "      <th>attr3_s</th>\n",
       "      <th>sinc3_s</th>\n",
       "      <th>intel3_s</th>\n",
       "      <th>fun3_s</th>\n",
       "      <th>amb3_s</th>\n",
       "      <th>satis_2</th>\n",
       "      <th>length</th>\n",
       "      <th>numdat_2</th>\n",
       "      <th>attr7_2</th>\n",
       "      <th>sinc7_2</th>\n",
       "      <th>intel7_2</th>\n",
       "      <th>fun7_2</th>\n",
       "      <th>amb7_2</th>\n",
       "      <th>shar7_2</th>\n",
       "      <th>attr1_2</th>\n",
       "      <th>sinc1_2</th>\n",
       "      <th>intel1_2</th>\n",
       "      <th>fun1_2</th>\n",
       "      <th>amb1_2</th>\n",
       "      <th>shar1_2</th>\n",
       "      <th>attr4_2</th>\n",
       "      <th>sinc4_2</th>\n",
       "      <th>intel4_2</th>\n",
       "      <th>fun4_2</th>\n",
       "      <th>amb4_2</th>\n",
       "      <th>shar4_2</th>\n",
       "      <th>attr2_2</th>\n",
       "      <th>sinc2_2</th>\n",
       "      <th>intel2_2</th>\n",
       "      <th>fun2_2</th>\n",
       "      <th>amb2_2</th>\n",
       "      <th>shar2_2</th>\n",
       "      <th>attr3_2</th>\n",
       "      <th>sinc3_2</th>\n",
       "      <th>intel3_2</th>\n",
       "      <th>fun3_2</th>\n",
       "      <th>amb3_2</th>\n",
       "      <th>attr5_2</th>\n",
       "      <th>sinc5_2</th>\n",
       "      <th>intel5_2</th>\n",
       "      <th>fun5_2</th>\n",
       "      <th>amb5_2</th>\n",
       "      <th>you_call</th>\n",
       "      <th>them_cal</th>\n",
       "      <th>date_3</th>\n",
       "      <th>numdat_3</th>\n",
       "      <th>num_in_3</th>\n",
       "      <th>attr1_3</th>\n",
       "      <th>sinc1_3</th>\n",
       "      <th>intel1_3</th>\n",
       "      <th>fun1_3</th>\n",
       "      <th>amb1_3</th>\n",
       "      <th>shar1_3</th>\n",
       "      <th>attr7_3</th>\n",
       "      <th>sinc7_3</th>\n",
       "      <th>intel7_3</th>\n",
       "      <th>fun7_3</th>\n",
       "      <th>amb7_3</th>\n",
       "      <th>shar7_3</th>\n",
       "      <th>attr4_3</th>\n",
       "      <th>sinc4_3</th>\n",
       "      <th>intel4_3</th>\n",
       "      <th>fun4_3</th>\n",
       "      <th>amb4_3</th>\n",
       "      <th>shar4_3</th>\n",
       "      <th>attr2_3</th>\n",
       "      <th>sinc2_3</th>\n",
       "      <th>intel2_3</th>\n",
       "      <th>fun2_3</th>\n",
       "      <th>amb2_3</th>\n",
       "      <th>shar2_3</th>\n",
       "      <th>attr3_3</th>\n",
       "      <th>sinc3_3</th>\n",
       "      <th>intel3_3</th>\n",
       "      <th>fun3_3</th>\n",
       "      <th>amb3_3</th>\n",
       "      <th>attr5_3</th>\n",
       "      <th>sinc5_3</th>\n",
       "      <th>intel5_3</th>\n",
       "      <th>fun5_3</th>\n",
       "      <th>amb5_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Law</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>60,521</td>\n",
       "      <td>69,487.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.44</td>\n",
       "      <td>16.67</td>\n",
       "      <td>13.89</td>\n",
       "      <td>22.22</td>\n",
       "      <td>11.11</td>\n",
       "      <td>16.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Law</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>60,521</td>\n",
       "      <td>69,487.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.44</td>\n",
       "      <td>16.67</td>\n",
       "      <td>13.89</td>\n",
       "      <td>22.22</td>\n",
       "      <td>11.11</td>\n",
       "      <td>16.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.16</td>\n",
       "      <td>1</td>\n",
       "      <td>22.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Law</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>60,521</td>\n",
       "      <td>69,487.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.44</td>\n",
       "      <td>16.67</td>\n",
       "      <td>13.89</td>\n",
       "      <td>22.22</td>\n",
       "      <td>11.11</td>\n",
       "      <td>16.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Law</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>60,521</td>\n",
       "      <td>69,487.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.44</td>\n",
       "      <td>16.67</td>\n",
       "      <td>13.89</td>\n",
       "      <td>22.22</td>\n",
       "      <td>11.11</td>\n",
       "      <td>16.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>Law</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>60,521</td>\n",
       "      <td>69,487.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.44</td>\n",
       "      <td>16.67</td>\n",
       "      <td>13.89</td>\n",
       "      <td>22.22</td>\n",
       "      <td>11.11</td>\n",
       "      <td>16.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   iid   id  gender  idg  condtn  wave  round  position  positin1  order  \\\n",
       "0    1  1.0       0    1       1     1     10         7       NaN      4   \n",
       "1    1  1.0       0    1       1     1     10         7       NaN      3   \n",
       "2    1  1.0       0    1       1     1     10         7       NaN     10   \n",
       "3    1  1.0       0    1       1     1     10         7       NaN      5   \n",
       "4    1  1.0       0    1       1     1     10         7       NaN      7   \n",
       "\n",
       "   partner   pid  match  int_corr  samerace  age_o  race_o  pf_o_att  \\\n",
       "0        1  11.0      0      0.14         0   27.0     2.0      35.0   \n",
       "1        2  12.0      0      0.54         0   22.0     2.0      60.0   \n",
       "2        3  13.0      1      0.16         1   22.0     4.0      19.0   \n",
       "3        4  14.0      1      0.61         0   23.0     2.0      30.0   \n",
       "4        5  15.0      1      0.21         0   24.0     3.0      30.0   \n",
       "\n",
       "   pf_o_sin  pf_o_int  pf_o_fun  pf_o_amb  pf_o_sha  dec_o  attr_o  sinc_o  \\\n",
       "0      20.0      20.0      20.0       0.0       5.0      0     6.0     8.0   \n",
       "1       0.0       0.0      40.0       0.0       0.0      0     7.0     8.0   \n",
       "2      18.0      19.0      18.0      14.0      12.0      1    10.0    10.0   \n",
       "3       5.0      15.0      40.0       5.0       5.0      1     7.0     8.0   \n",
       "4      10.0      20.0      10.0      10.0      20.0      1     8.0     7.0   \n",
       "\n",
       "   intel_o  fun_o  amb_o  shar_o  like_o  prob_o  met_o   age field  field_cd  \\\n",
       "0      8.0    8.0    8.0     6.0     7.0     4.0    2.0  21.0   Law       1.0   \n",
       "1     10.0    7.0    7.0     5.0     8.0     4.0    2.0  21.0   Law       1.0   \n",
       "2     10.0   10.0   10.0    10.0    10.0    10.0    1.0  21.0   Law       1.0   \n",
       "3      9.0    8.0    9.0     8.0     7.0     7.0    2.0  21.0   Law       1.0   \n",
       "4      9.0    6.0    9.0     7.0     8.0     6.0    2.0  21.0   Law       1.0   \n",
       "\n",
       "  undergra mn_sat tuition  race  imprace  imprelig     from zipcode  \\\n",
       "0      NaN    NaN     NaN   4.0      2.0       4.0  Chicago  60,521   \n",
       "1      NaN    NaN     NaN   4.0      2.0       4.0  Chicago  60,521   \n",
       "2      NaN    NaN     NaN   4.0      2.0       4.0  Chicago  60,521   \n",
       "3      NaN    NaN     NaN   4.0      2.0       4.0  Chicago  60,521   \n",
       "4      NaN    NaN     NaN   4.0      2.0       4.0  Chicago  60,521   \n",
       "\n",
       "      income  goal  date  go_out  career  career_c  sports  tvsports  \\\n",
       "0  69,487.00   2.0   7.0     1.0  lawyer       NaN     9.0       2.0   \n",
       "1  69,487.00   2.0   7.0     1.0  lawyer       NaN     9.0       2.0   \n",
       "2  69,487.00   2.0   7.0     1.0  lawyer       NaN     9.0       2.0   \n",
       "3  69,487.00   2.0   7.0     1.0  lawyer       NaN     9.0       2.0   \n",
       "4  69,487.00   2.0   7.0     1.0  lawyer       NaN     9.0       2.0   \n",
       "\n",
       "   exercise  dining  museums  art  hiking  gaming  clubbing  reading   tv  \\\n",
       "0       8.0     9.0      1.0  1.0     5.0     1.0       5.0      6.0  9.0   \n",
       "1       8.0     9.0      1.0  1.0     5.0     1.0       5.0      6.0  9.0   \n",
       "2       8.0     9.0      1.0  1.0     5.0     1.0       5.0      6.0  9.0   \n",
       "3       8.0     9.0      1.0  1.0     5.0     1.0       5.0      6.0  9.0   \n",
       "4       8.0     9.0      1.0  1.0     5.0     1.0       5.0      6.0  9.0   \n",
       "\n",
       "   theater  movies  concerts  music  shopping  yoga  exphappy  expnum  \\\n",
       "0      1.0    10.0      10.0    9.0       8.0   1.0       3.0     2.0   \n",
       "1      1.0    10.0      10.0    9.0       8.0   1.0       3.0     2.0   \n",
       "2      1.0    10.0      10.0    9.0       8.0   1.0       3.0     2.0   \n",
       "3      1.0    10.0      10.0    9.0       8.0   1.0       3.0     2.0   \n",
       "4      1.0    10.0      10.0    9.0       8.0   1.0       3.0     2.0   \n",
       "\n",
       "   attr1_1  sinc1_1  intel1_1  fun1_1  amb1_1  shar1_1  attr4_1  sinc4_1  \\\n",
       "0     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "1     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "2     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "3     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "4     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "\n",
       "   intel4_1  fun4_1  amb4_1  shar4_1  attr2_1  sinc2_1  intel2_1  fun2_1  \\\n",
       "0       NaN     NaN     NaN      NaN     35.0     20.0      15.0    20.0   \n",
       "1       NaN     NaN     NaN      NaN     35.0     20.0      15.0    20.0   \n",
       "2       NaN     NaN     NaN      NaN     35.0     20.0      15.0    20.0   \n",
       "3       NaN     NaN     NaN      NaN     35.0     20.0      15.0    20.0   \n",
       "4       NaN     NaN     NaN      NaN     35.0     20.0      15.0    20.0   \n",
       "\n",
       "   amb2_1  shar2_1  attr3_1  sinc3_1  fun3_1  intel3_1  amb3_1  attr5_1  \\\n",
       "0     5.0      5.0      6.0      8.0     8.0       8.0     7.0      NaN   \n",
       "1     5.0      5.0      6.0      8.0     8.0       8.0     7.0      NaN   \n",
       "2     5.0      5.0      6.0      8.0     8.0       8.0     7.0      NaN   \n",
       "3     5.0      5.0      6.0      8.0     8.0       8.0     7.0      NaN   \n",
       "4     5.0      5.0      6.0      8.0     8.0       8.0     7.0      NaN   \n",
       "\n",
       "   sinc5_1  intel5_1  fun5_1  amb5_1  dec  attr  sinc  intel  fun  amb  shar  \\\n",
       "0      NaN       NaN     NaN     NaN    1   6.0   9.0    7.0  7.0  6.0   5.0   \n",
       "1      NaN       NaN     NaN     NaN    1   7.0   8.0    7.0  8.0  5.0   6.0   \n",
       "2      NaN       NaN     NaN     NaN    1   5.0   8.0    9.0  8.0  5.0   7.0   \n",
       "3      NaN       NaN     NaN     NaN    1   7.0   6.0    8.0  7.0  6.0   8.0   \n",
       "4      NaN       NaN     NaN     NaN    1   5.0   6.0    7.0  7.0  6.0   6.0   \n",
       "\n",
       "   like  prob  met  match_es  attr1_s  sinc1_s  intel1_s  fun1_s  amb1_s  \\\n",
       "0   7.0   6.0  2.0       4.0      NaN      NaN       NaN     NaN     NaN   \n",
       "1   7.0   5.0  1.0       4.0      NaN      NaN       NaN     NaN     NaN   \n",
       "2   7.0   NaN  1.0       4.0      NaN      NaN       NaN     NaN     NaN   \n",
       "3   7.0   6.0  2.0       4.0      NaN      NaN       NaN     NaN     NaN   \n",
       "4   6.0   6.0  2.0       4.0      NaN      NaN       NaN     NaN     NaN   \n",
       "\n",
       "   shar1_s  attr3_s  sinc3_s  intel3_s  fun3_s  amb3_s  satis_2  length  \\\n",
       "0      NaN      NaN      NaN       NaN     NaN     NaN      6.0     2.0   \n",
       "1      NaN      NaN      NaN       NaN     NaN     NaN      6.0     2.0   \n",
       "2      NaN      NaN      NaN       NaN     NaN     NaN      6.0     2.0   \n",
       "3      NaN      NaN      NaN       NaN     NaN     NaN      6.0     2.0   \n",
       "4      NaN      NaN      NaN       NaN     NaN     NaN      6.0     2.0   \n",
       "\n",
       "   numdat_2  attr7_2  sinc7_2  intel7_2  fun7_2  amb7_2  shar7_2  attr1_2  \\\n",
       "0       1.0      NaN      NaN       NaN     NaN     NaN      NaN    19.44   \n",
       "1       1.0      NaN      NaN       NaN     NaN     NaN      NaN    19.44   \n",
       "2       1.0      NaN      NaN       NaN     NaN     NaN      NaN    19.44   \n",
       "3       1.0      NaN      NaN       NaN     NaN     NaN      NaN    19.44   \n",
       "4       1.0      NaN      NaN       NaN     NaN     NaN      NaN    19.44   \n",
       "\n",
       "   sinc1_2  intel1_2  fun1_2  amb1_2  shar1_2  attr4_2  sinc4_2  intel4_2  \\\n",
       "0    16.67     13.89   22.22   11.11    16.67      NaN      NaN       NaN   \n",
       "1    16.67     13.89   22.22   11.11    16.67      NaN      NaN       NaN   \n",
       "2    16.67     13.89   22.22   11.11    16.67      NaN      NaN       NaN   \n",
       "3    16.67     13.89   22.22   11.11    16.67      NaN      NaN       NaN   \n",
       "4    16.67     13.89   22.22   11.11    16.67      NaN      NaN       NaN   \n",
       "\n",
       "   fun4_2  amb4_2  shar4_2  attr2_2  sinc2_2  intel2_2  fun2_2  amb2_2  \\\n",
       "0     NaN     NaN      NaN      NaN      NaN       NaN     NaN     NaN   \n",
       "1     NaN     NaN      NaN      NaN      NaN       NaN     NaN     NaN   \n",
       "2     NaN     NaN      NaN      NaN      NaN       NaN     NaN     NaN   \n",
       "3     NaN     NaN      NaN      NaN      NaN       NaN     NaN     NaN   \n",
       "4     NaN     NaN      NaN      NaN      NaN       NaN     NaN     NaN   \n",
       "\n",
       "   shar2_2  attr3_2  sinc3_2  intel3_2  fun3_2  amb3_2  attr5_2  sinc5_2  \\\n",
       "0      NaN      6.0      7.0       8.0     7.0     6.0      NaN      NaN   \n",
       "1      NaN      6.0      7.0       8.0     7.0     6.0      NaN      NaN   \n",
       "2      NaN      6.0      7.0       8.0     7.0     6.0      NaN      NaN   \n",
       "3      NaN      6.0      7.0       8.0     7.0     6.0      NaN      NaN   \n",
       "4      NaN      6.0      7.0       8.0     7.0     6.0      NaN      NaN   \n",
       "\n",
       "   intel5_2  fun5_2  amb5_2  you_call  them_cal  date_3  numdat_3  num_in_3  \\\n",
       "0       NaN     NaN     NaN       1.0       1.0     0.0       NaN       NaN   \n",
       "1       NaN     NaN     NaN       1.0       1.0     0.0       NaN       NaN   \n",
       "2       NaN     NaN     NaN       1.0       1.0     0.0       NaN       NaN   \n",
       "3       NaN     NaN     NaN       1.0       1.0     0.0       NaN       NaN   \n",
       "4       NaN     NaN     NaN       1.0       1.0     0.0       NaN       NaN   \n",
       "\n",
       "   attr1_3  sinc1_3  intel1_3  fun1_3  amb1_3  shar1_3  attr7_3  sinc7_3  \\\n",
       "0     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "1     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "2     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "3     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "4     15.0     20.0      20.0    15.0    15.0     15.0      NaN      NaN   \n",
       "\n",
       "   intel7_3  fun7_3  amb7_3  shar7_3  attr4_3  sinc4_3  intel4_3  fun4_3  \\\n",
       "0       NaN     NaN     NaN      NaN      NaN      NaN       NaN     NaN   \n",
       "1       NaN     NaN     NaN      NaN      NaN      NaN       NaN     NaN   \n",
       "2       NaN     NaN     NaN      NaN      NaN      NaN       NaN     NaN   \n",
       "3       NaN     NaN     NaN      NaN      NaN      NaN       NaN     NaN   \n",
       "4       NaN     NaN     NaN      NaN      NaN      NaN       NaN     NaN   \n",
       "\n",
       "   amb4_3  shar4_3  attr2_3  sinc2_3  intel2_3  fun2_3  amb2_3  shar2_3  \\\n",
       "0     NaN      NaN      NaN      NaN       NaN     NaN     NaN      NaN   \n",
       "1     NaN      NaN      NaN      NaN       NaN     NaN     NaN      NaN   \n",
       "2     NaN      NaN      NaN      NaN       NaN     NaN     NaN      NaN   \n",
       "3     NaN      NaN      NaN      NaN       NaN     NaN     NaN      NaN   \n",
       "4     NaN      NaN      NaN      NaN       NaN     NaN     NaN      NaN   \n",
       "\n",
       "   attr3_3  sinc3_3  intel3_3  fun3_3  amb3_3  attr5_3  sinc5_3  intel5_3  \\\n",
       "0      5.0      7.0       7.0     7.0     7.0      NaN      NaN       NaN   \n",
       "1      5.0      7.0       7.0     7.0     7.0      NaN      NaN       NaN   \n",
       "2      5.0      7.0       7.0     7.0     7.0      NaN      NaN       NaN   \n",
       "3      5.0      7.0       7.0     7.0     7.0      NaN      NaN       NaN   \n",
       "4      5.0      7.0       7.0     7.0     7.0      NaN      NaN       NaN   \n",
       "\n",
       "   fun5_3  amb5_3  \n",
       "0     NaN     NaN  \n",
       "1     NaN     NaN  \n",
       "2     NaN     NaN  \n",
       "3     NaN     NaN  \n",
       "4     NaN     NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(8378, 195)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_columns', 200)\n",
    "data = pd.read_csv('./src/Speed+Dating+Data.csv', encoding='Latin-1')\n",
    "display(data.head())\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Champs inutiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['iid', 'id', 'gender', 'idg', 'condtn', 'wave', 'round',\n",
       "       'position', 'positin1', 'order', 'partner', 'pid', 'match',\n",
       "       'int_corr', 'samerace', 'age_o', 'race_o', 'pf_o_att', 'pf_o_sin',\n",
       "       'pf_o_int', 'pf_o_fun', 'pf_o_amb', 'pf_o_sha', 'dec_o', 'attr_o',\n",
       "       'sinc_o', 'intel_o', 'fun_o', 'amb_o', 'shar_o', 'like_o',\n",
       "       'prob_o', 'met_o', 'age', 'field', 'field_cd', 'undergra',\n",
       "       'mn_sat', 'tuition', 'race', 'imprace', 'imprelig', 'from',\n",
       "       'zipcode', 'income', 'goal', 'date', 'go_out', 'career',\n",
       "       'career_c', 'sports', 'tvsports', 'exercise', 'dining', 'museums',\n",
       "       'art', 'hiking', 'gaming', 'clubbing', 'reading', 'tv', 'theater',\n",
       "       'movies', 'concerts', 'music', 'shopping', 'yoga', 'exphappy',\n",
       "       'expnum', 'attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1',\n",
       "       'shar1_1', 'attr4_1', 'sinc4_1', 'intel4_1', 'fun4_1', 'amb4_1',\n",
       "       'shar4_1', 'attr2_1', 'sinc2_1', 'intel2_1', 'fun2_1', 'amb2_1',\n",
       "       'shar2_1', 'attr3_1', 'sinc3_1', 'fun3_1', 'intel3_1', 'amb3_1',\n",
       "       'attr5_1', 'sinc5_1', 'intel5_1', 'fun5_1', 'amb5_1', 'dec',\n",
       "       'attr', 'sinc', 'intel', 'fun', 'amb', 'shar', 'like', 'prob',\n",
       "       'met', 'match_es', 'attr1_s', 'sinc1_s', 'intel1_s', 'fun1_s',\n",
       "       'amb1_s', 'shar1_s', 'attr3_s', 'sinc3_s', 'intel3_s', 'fun3_s',\n",
       "       'amb3_s', 'satis_2', 'length', 'numdat_2', 'attr7_2', 'sinc7_2',\n",
       "       'intel7_2', 'fun7_2', 'amb7_2', 'shar7_2', 'attr1_2', 'sinc1_2',\n",
       "       'intel1_2', 'fun1_2', 'amb1_2', 'shar1_2', 'attr4_2', 'sinc4_2',\n",
       "       'intel4_2', 'fun4_2', 'amb4_2', 'shar4_2', 'attr2_2', 'sinc2_2',\n",
       "       'intel2_2', 'fun2_2', 'amb2_2', 'shar2_2', 'attr3_2', 'sinc3_2',\n",
       "       'intel3_2', 'fun3_2', 'amb3_2', 'attr5_2', 'sinc5_2', 'intel5_2',\n",
       "       'fun5_2', 'amb5_2', 'you_call', 'them_cal', 'date_3', 'numdat_3',\n",
       "       'num_in_3', 'attr1_3', 'sinc1_3', 'intel1_3', 'fun1_3', 'amb1_3',\n",
       "       'shar1_3', 'attr7_3', 'sinc7_3', 'intel7_3', 'fun7_3', 'amb7_3',\n",
       "       'shar7_3', 'attr4_3', 'sinc4_3', 'intel4_3', 'fun4_3', 'amb4_3',\n",
       "       'shar4_3', 'attr2_3', 'sinc2_3', 'intel2_3', 'fun2_3', 'amb2_3',\n",
       "       'shar2_3', 'attr3_3', 'sinc3_3', 'intel3_3', 'fun3_3', 'amb3_3',\n",
       "       'attr5_3', 'sinc5_3', 'intel5_3', 'fun5_3', 'amb5_3'], dtype=object)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_columns = data.columns\n",
    "print(len(all_columns))\n",
    "all_columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8378, 188)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Nous identifions et retirons les champs qui ne sont pas utiles pour notre étude \n",
    "# (informations redondantes et/ou non pertinentes)\n",
    "useless_columns = ['id','idg','partner','field','from','zipcode','career']\n",
    "data.drop(useless_columns, axis=1, inplace=True)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lignes erronées"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8368, 188)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Nous identifions 10 lignes erronées dans le jeu de données (valeur pid NaN)\n",
    "# -> Nous retirons ces lignes\n",
    "data = data[np.isnan(data.pid) != True]\n",
    "data = data.reset_index(drop=True)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Colonnes avec données manquantes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_name</th>\n",
       "      <th>missing_values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>num_in_3</td>\n",
       "      <td>92.041109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>numdat_3</td>\n",
       "      <td>82.122371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>expnum</td>\n",
       "      <td>78.608987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>amb7_2</td>\n",
       "      <td>76.637189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>sinc7_2</td>\n",
       "      <td>76.637189</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    column_name  missing_values\n",
       "153    num_in_3       92.041109\n",
       "152    numdat_3       82.122371\n",
       "61       expnum       78.608987\n",
       "119      amb7_2       76.637189\n",
       "116     sinc7_2       76.637189"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values = data.isna().sum()/data.shape[0]*100\n",
    "missing_values_df = missing_values.to_frame().reset_index()\n",
    "missing_values_df.rename(columns={'index': 'column_name', 0: 'missing_values'}, inplace=True)\n",
    "missing_values_df.sort_values(by='missing_values', ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "alignmentgroup": "True",
         "hovertemplate": "proportion of missing values (%)=%{x}<br>columns=%{y}<extra></extra>",
         "legendgroup": "",
         "marker": {
          "color": "#636efa",
          "pattern": {
           "shape": ""
          }
         },
         "name": "",
         "offsetgroup": "",
         "orientation": "h",
         "showlegend": false,
         "textposition": "auto",
         "type": "bar",
         "x": [
          21.940726577437857,
          41.27629063097514,
          62.55975143403442,
          57.1821223709369,
          48.936424474187376,
          78.60898661567877,
          22.454588910133843,
          22.454588910133843,
          22.454588910133843,
          22.454588910133843,
          22.454588910133843,
          22.71749521988528,
          41.37189292543021,
          41.37189292543021,
          41.37189292543021,
          41.37189292543021,
          41.37189292543021,
          51.05162523900574,
          51.05162523900574,
          51.05162523900574,
          51.05162523900574,
          51.05162523900574,
          51.05162523900574,
          52.198852772466545,
          52.198852772466545,
          52.198852772466545,
          52.198852772466545,
          52.198852772466545,
          76.2906309751434,
          76.63718929254301,
          76.2906309751434,
          76.2906309751434,
          76.63718929254301,
          76.41013384321224,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          30.987093690248567,
          47.69359464627151,
          47.69359464627151,
          47.69359464627151,
          47.69359464627151,
          47.69359464627151,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          82.12237093690248,
          92.04110898661568,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          64.63910133843213,
          75.90822179732314,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          52.56931166347992,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314,
          75.90822179732314
         ],
         "xaxis": "x",
         "y": [
          "positin1",
          "undergra",
          "mn_sat",
          "tuition",
          "income",
          "expnum",
          "attr4_1",
          "sinc4_1",
          "intel4_1",
          "fun4_1",
          "amb4_1",
          "shar4_1",
          "attr5_1",
          "sinc5_1",
          "intel5_1",
          "fun5_1",
          "amb5_1",
          "attr1_s",
          "sinc1_s",
          "intel1_s",
          "fun1_s",
          "amb1_s",
          "shar1_s",
          "attr3_s",
          "sinc3_s",
          "intel3_s",
          "fun3_s",
          "amb3_s",
          "attr7_2",
          "sinc7_2",
          "intel7_2",
          "fun7_2",
          "amb7_2",
          "shar7_2",
          "attr4_2",
          "sinc4_2",
          "intel4_2",
          "fun4_2",
          "amb4_2",
          "shar4_2",
          "attr2_2",
          "sinc2_2",
          "intel2_2",
          "fun2_2",
          "amb2_2",
          "shar2_2",
          "attr5_2",
          "sinc5_2",
          "intel5_2",
          "fun5_2",
          "amb5_2",
          "you_call",
          "them_cal",
          "date_3",
          "numdat_3",
          "num_in_3",
          "attr1_3",
          "sinc1_3",
          "intel1_3",
          "fun1_3",
          "amb1_3",
          "shar1_3",
          "attr7_3",
          "sinc7_3",
          "intel7_3",
          "fun7_3",
          "amb7_3",
          "shar7_3",
          "attr4_3",
          "sinc4_3",
          "intel4_3",
          "fun4_3",
          "amb4_3",
          "shar4_3",
          "attr2_3",
          "sinc2_3",
          "intel2_3",
          "fun2_3",
          "amb2_3",
          "shar2_3",
          "attr3_3",
          "sinc3_3",
          "intel3_3",
          "fun3_3",
          "amb3_3",
          "attr5_3",
          "sinc5_3",
          "intel5_3",
          "fun5_3",
          "amb5_3"
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "barmode": "relative",
        "font": {
         "size": 7
        },
        "height": 600,
        "legend": {
         "tracegroupgap": 0
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "font": {
          "size": 18
         },
         "text": "Columns where missing values exceed 20%",
         "x": 0.5
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "tickfont": {
          "size": 11
         },
         "title": {
          "font": {
           "size": 15
          },
          "text": "proportion of missing values (%)"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "font": {
           "size": 15
          },
          "text": "columns"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualisation avec Plotly Express\n",
    "mask = missing_values_df[missing_values_df['missing_values'] > 20]\n",
    "fig = px.bar(y=mask['column_name'], \n",
    "                x=mask['missing_values'],\n",
    "                title='Columns where missing values exceed 20%', \n",
    "                labels={'x':'proportion of missing values (%)', 'y': 'columns'},\n",
    "                orientation = 'h', height=600)\n",
    "#fig.add_vline(x=20)\n",
    "fig.update_layout(title_x=.5,\n",
    "                  title_font_size=18,\n",
    "                  xaxis_title_font_size=15,\n",
    "                  yaxis_title_font_size=15,\n",
    "                  font=dict(size=7))\n",
    "fig.update_xaxes(tickfont_size=11)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['positin1', 'undergra', 'mn_sat', 'tuition', 'income', 'expnum',\n",
       "       'attr4_1', 'sinc4_1', 'intel4_1', 'fun4_1', 'amb4_1', 'shar4_1',\n",
       "       'attr5_1', 'sinc5_1', 'intel5_1', 'fun5_1', 'amb5_1', 'attr1_s',\n",
       "       'sinc1_s', 'intel1_s', 'fun1_s', 'amb1_s', 'shar1_s', 'attr3_s',\n",
       "       'sinc3_s', 'intel3_s', 'fun3_s', 'amb3_s', 'attr7_2', 'sinc7_2',\n",
       "       'intel7_2', 'fun7_2', 'amb7_2', 'shar7_2', 'attr4_2', 'sinc4_2',\n",
       "       'intel4_2', 'fun4_2', 'amb4_2', 'shar4_2', 'attr2_2', 'sinc2_2',\n",
       "       'intel2_2', 'fun2_2', 'amb2_2', 'shar2_2', 'attr5_2', 'sinc5_2',\n",
       "       'intel5_2', 'fun5_2', 'amb5_2', 'you_call', 'them_cal', 'date_3',\n",
       "       'numdat_3', 'num_in_3', 'attr1_3', 'sinc1_3', 'intel1_3', 'fun1_3',\n",
       "       'amb1_3', 'shar1_3', 'attr7_3', 'sinc7_3', 'intel7_3', 'fun7_3',\n",
       "       'amb7_3', 'shar7_3', 'attr4_3', 'sinc4_3', 'intel4_3', 'fun4_3',\n",
       "       'amb4_3', 'shar4_3', 'attr2_3', 'sinc2_3', 'intel2_3', 'fun2_3',\n",
       "       'amb2_3', 'shar2_3', 'attr3_3', 'sinc3_3', 'intel3_3', 'fun3_3',\n",
       "       'amb3_3', 'attr5_3', 'sinc5_3', 'intel5_3', 'fun5_3', 'amb5_3'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Liste des champs avec plus de 20% de valeurs manquantes\n",
    "missing_values_columns = mask['column_name'].values\n",
    "print(len(missing_values_columns))\n",
    "missing_values_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8368, 98)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_remove = missing_values_columns.tolist()\n",
    "data_clean_mv = data.copy()\n",
    "data_clean_mv.drop(to_remove, axis=1, inplace=True)\n",
    "data_clean_mv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_name</th>\n",
       "      <th>missing_values</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>match_es</td>\n",
       "      <td>14.017686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>shar_o</td>\n",
       "      <td>12.739006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>shar</td>\n",
       "      <td>12.739006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>numdat_2</td>\n",
       "      <td>11.281071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>attr1_2</td>\n",
       "      <td>11.137667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>intel1_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>satis_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>length</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>sinc1_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>amb3_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>fun1_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>amb1_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>shar1_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>attr3_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>sinc3_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>intel3_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>fun3_2</td>\n",
       "      <td>10.922562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>amb_o</td>\n",
       "      <td>8.508604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>amb</td>\n",
       "      <td>8.508604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>met</td>\n",
       "      <td>4.481358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>met_o</td>\n",
       "      <td>4.481358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>fun</td>\n",
       "      <td>4.182600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>fun_o</td>\n",
       "      <td>4.182600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>prob_o</td>\n",
       "      <td>3.680688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>prob</td>\n",
       "      <td>3.680688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>intel_o</td>\n",
       "      <td>3.537285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>intel</td>\n",
       "      <td>3.537285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>sinc_o</td>\n",
       "      <td>3.310229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>sinc</td>\n",
       "      <td>3.310229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>like</td>\n",
       "      <td>2.868069</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_name  missing_values\n",
       "83    match_es       14.017686\n",
       "25      shar_o       12.739006\n",
       "79        shar       12.739006\n",
       "86    numdat_2       11.281071\n",
       "87     attr1_2       11.137667\n",
       "89    intel1_2       10.922562\n",
       "84     satis_2       10.922562\n",
       "85      length       10.922562\n",
       "88     sinc1_2       10.922562\n",
       "97      amb3_2       10.922562\n",
       "90      fun1_2       10.922562\n",
       "91      amb1_2       10.922562\n",
       "92     shar1_2       10.922562\n",
       "93     attr3_2       10.922562\n",
       "94     sinc3_2       10.922562\n",
       "95    intel3_2       10.922562\n",
       "96      fun3_2       10.922562\n",
       "24       amb_o        8.508604\n",
       "78         amb        8.508604\n",
       "82         met        4.481358\n",
       "28       met_o        4.481358\n",
       "77         fun        4.182600\n",
       "23       fun_o        4.182600\n",
       "27      prob_o        3.680688\n",
       "81        prob        3.680688\n",
       "22     intel_o        3.537285\n",
       "76       intel        3.537285\n",
       "21      sinc_o        3.310229\n",
       "75        sinc        3.310229\n",
       "80        like        2.868069"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values = data_clean_mv.isna().sum()/data_clean_mv.shape[0]*100\n",
    "missing_values_df = missing_values.to_frame().reset_index()\n",
    "missing_values_df.rename(columns={'index': 'column_name', 0: 'missing_values'}, inplace=True)\n",
    "missing_values_df.sort_values(by='missing_values', ascending=False).head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Valeurs erronées"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Erreurs relevées : \n",
    "- Nous remarquons des valeurs erronées pour les champs *attr_o*, *fun_o* (valeurs devant être comprises entre 1 et 10)  \n",
    "\n",
    "- De même que pour les champs *reading* et *gaming* (valeurs devant être comprises entre 1 et 10)  \n",
    "\n",
    "- Ainsi que pour les champs *met* et *met_o* (valeurs devant être égales à 1 ou 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>gender</th>\n",
       "      <th>condtn</th>\n",
       "      <th>wave</th>\n",
       "      <th>round</th>\n",
       "      <th>position</th>\n",
       "      <th>order</th>\n",
       "      <th>pid</th>\n",
       "      <th>match</th>\n",
       "      <th>int_corr</th>\n",
       "      <th>samerace</th>\n",
       "      <th>age_o</th>\n",
       "      <th>race_o</th>\n",
       "      <th>pf_o_att</th>\n",
       "      <th>pf_o_sin</th>\n",
       "      <th>pf_o_int</th>\n",
       "      <th>pf_o_fun</th>\n",
       "      <th>pf_o_amb</th>\n",
       "      <th>pf_o_sha</th>\n",
       "      <th>dec_o</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>like_o</th>\n",
       "      <th>prob_o</th>\n",
       "      <th>met_o</th>\n",
       "      <th>age</th>\n",
       "      <th>field_cd</th>\n",
       "      <th>race</th>\n",
       "      <th>imprace</th>\n",
       "      <th>imprelig</th>\n",
       "      <th>goal</th>\n",
       "      <th>date</th>\n",
       "      <th>go_out</th>\n",
       "      <th>career_c</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>attr2_1</th>\n",
       "      <th>sinc2_1</th>\n",
       "      <th>intel2_1</th>\n",
       "      <th>fun2_1</th>\n",
       "      <th>amb2_1</th>\n",
       "      <th>shar2_1</th>\n",
       "      <th>attr3_1</th>\n",
       "      <th>sinc3_1</th>\n",
       "      <th>fun3_1</th>\n",
       "      <th>intel3_1</th>\n",
       "      <th>amb3_1</th>\n",
       "      <th>dec</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "      <th>like</th>\n",
       "      <th>prob</th>\n",
       "      <th>met</th>\n",
       "      <th>match_es</th>\n",
       "      <th>satis_2</th>\n",
       "      <th>length</th>\n",
       "      <th>numdat_2</th>\n",
       "      <th>attr1_2</th>\n",
       "      <th>sinc1_2</th>\n",
       "      <th>intel1_2</th>\n",
       "      <th>fun1_2</th>\n",
       "      <th>amb1_2</th>\n",
       "      <th>shar1_2</th>\n",
       "      <th>attr3_2</th>\n",
       "      <th>sinc3_2</th>\n",
       "      <th>intel3_2</th>\n",
       "      <th>fun3_2</th>\n",
       "      <th>amb3_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.00000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8210.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8274.000000</td>\n",
       "      <td>8305.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8249.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8166.000000</td>\n",
       "      <td>8091.000000</td>\n",
       "      <td>8072.000000</td>\n",
       "      <td>8018.000000</td>\n",
       "      <td>7656.000000</td>\n",
       "      <td>7302.000000</td>\n",
       "      <td>8128.000000</td>\n",
       "      <td>8060.000000</td>\n",
       "      <td>7993.000000</td>\n",
       "      <td>8274.000000</td>\n",
       "      <td>8286.000000</td>\n",
       "      <td>8305.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8230.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8267.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8249.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8166.000000</td>\n",
       "      <td>8091.000000</td>\n",
       "      <td>8072.000000</td>\n",
       "      <td>8018.000000</td>\n",
       "      <td>7656.000000</td>\n",
       "      <td>7302.000000</td>\n",
       "      <td>8128.000000</td>\n",
       "      <td>8060.000000</td>\n",
       "      <td>7993.000000</td>\n",
       "      <td>7195.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7424.000000</td>\n",
       "      <td>7436.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>283.863767</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>1.829828</td>\n",
       "      <td>11.358509</td>\n",
       "      <td>16.880258</td>\n",
       "      <td>9.048757</td>\n",
       "      <td>8.931764</td>\n",
       "      <td>283.863767</td>\n",
       "      <td>0.164914</td>\n",
       "      <td>0.196300</td>\n",
       "      <td>0.396272</td>\n",
       "      <td>26.364999</td>\n",
       "      <td>2.756653</td>\n",
       "      <td>22.495347</td>\n",
       "      <td>17.396867</td>\n",
       "      <td>20.270759</td>\n",
       "      <td>17.459714</td>\n",
       "      <td>10.685375</td>\n",
       "      <td>11.845930</td>\n",
       "      <td>0.420053</td>\n",
       "      <td>6.190411</td>\n",
       "      <td>7.175256</td>\n",
       "      <td>7.369301</td>\n",
       "      <td>6.400599</td>\n",
       "      <td>6.778409</td>\n",
       "      <td>5.474870</td>\n",
       "      <td>6.134498</td>\n",
       "      <td>5.208251</td>\n",
       "      <td>1.960215</td>\n",
       "      <td>26.364999</td>\n",
       "      <td>7.665339</td>\n",
       "      <td>2.756653</td>\n",
       "      <td>3.785981</td>\n",
       "      <td>3.653879</td>\n",
       "      <td>2.121848</td>\n",
       "      <td>5.007375</td>\n",
       "      <td>2.158523</td>\n",
       "      <td>5.277886</td>\n",
       "      <td>6.424418</td>\n",
       "      <td>4.574255</td>\n",
       "      <td>6.244300</td>\n",
       "      <td>7.783448</td>\n",
       "      <td>6.987453</td>\n",
       "      <td>6.716371</td>\n",
       "      <td>5.737483</td>\n",
       "      <td>3.880565</td>\n",
       "      <td>5.745325</td>\n",
       "      <td>7.680058</td>\n",
       "      <td>5.304379</td>\n",
       "      <td>6.778140</td>\n",
       "      <td>7.920618</td>\n",
       "      <td>6.824587</td>\n",
       "      <td>7.850887</td>\n",
       "      <td>5.631077</td>\n",
       "      <td>4.338038</td>\n",
       "      <td>5.534051</td>\n",
       "      <td>22.495347</td>\n",
       "      <td>17.396867</td>\n",
       "      <td>20.270759</td>\n",
       "      <td>17.459714</td>\n",
       "      <td>10.685375</td>\n",
       "      <td>11.845930</td>\n",
       "      <td>30.361422</td>\n",
       "      <td>13.264852</td>\n",
       "      <td>14.417394</td>\n",
       "      <td>18.427955</td>\n",
       "      <td>11.744583</td>\n",
       "      <td>11.855626</td>\n",
       "      <td>7.084352</td>\n",
       "      <td>8.295776</td>\n",
       "      <td>7.704345</td>\n",
       "      <td>8.404454</td>\n",
       "      <td>7.577877</td>\n",
       "      <td>0.420053</td>\n",
       "      <td>6.190350</td>\n",
       "      <td>7.175256</td>\n",
       "      <td>7.369301</td>\n",
       "      <td>6.400474</td>\n",
       "      <td>6.778409</td>\n",
       "      <td>5.474870</td>\n",
       "      <td>6.134498</td>\n",
       "      <td>5.208251</td>\n",
       "      <td>0.947579</td>\n",
       "      <td>3.206991</td>\n",
       "      <td>5.711833</td>\n",
       "      <td>1.843708</td>\n",
       "      <td>2.337823</td>\n",
       "      <td>26.197823</td>\n",
       "      <td>15.872299</td>\n",
       "      <td>17.817152</td>\n",
       "      <td>17.656897</td>\n",
       "      <td>9.918161</td>\n",
       "      <td>12.762388</td>\n",
       "      <td>7.124497</td>\n",
       "      <td>7.931983</td>\n",
       "      <td>8.238664</td>\n",
       "      <td>7.601959</td>\n",
       "      <td>7.486048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>158.584899</td>\n",
       "      <td>0.50003</td>\n",
       "      <td>0.375806</td>\n",
       "      <td>5.995461</td>\n",
       "      <td>4.354579</td>\n",
       "      <td>5.515476</td>\n",
       "      <td>5.478099</td>\n",
       "      <td>158.584899</td>\n",
       "      <td>0.371125</td>\n",
       "      <td>0.303522</td>\n",
       "      <td>0.489151</td>\n",
       "      <td>3.563648</td>\n",
       "      <td>1.230689</td>\n",
       "      <td>12.569802</td>\n",
       "      <td>7.044003</td>\n",
       "      <td>6.782895</td>\n",
       "      <td>6.085526</td>\n",
       "      <td>6.126544</td>\n",
       "      <td>6.362746</td>\n",
       "      <td>0.493597</td>\n",
       "      <td>1.950305</td>\n",
       "      <td>1.740575</td>\n",
       "      <td>1.550501</td>\n",
       "      <td>1.954078</td>\n",
       "      <td>1.794080</td>\n",
       "      <td>2.156163</td>\n",
       "      <td>1.841258</td>\n",
       "      <td>2.129354</td>\n",
       "      <td>0.245925</td>\n",
       "      <td>3.563648</td>\n",
       "      <td>3.759695</td>\n",
       "      <td>1.230689</td>\n",
       "      <td>2.846506</td>\n",
       "      <td>2.805641</td>\n",
       "      <td>1.406983</td>\n",
       "      <td>1.444439</td>\n",
       "      <td>1.105428</td>\n",
       "      <td>3.309885</td>\n",
       "      <td>2.619419</td>\n",
       "      <td>2.801521</td>\n",
       "      <td>2.419612</td>\n",
       "      <td>1.755105</td>\n",
       "      <td>2.051600</td>\n",
       "      <td>2.262518</td>\n",
       "      <td>2.570595</td>\n",
       "      <td>2.621188</td>\n",
       "      <td>2.502831</td>\n",
       "      <td>2.004806</td>\n",
       "      <td>2.529580</td>\n",
       "      <td>2.233341</td>\n",
       "      <td>1.700264</td>\n",
       "      <td>2.156155</td>\n",
       "      <td>1.792227</td>\n",
       "      <td>2.609472</td>\n",
       "      <td>2.717712</td>\n",
       "      <td>1.733627</td>\n",
       "      <td>12.569802</td>\n",
       "      <td>7.044003</td>\n",
       "      <td>6.782895</td>\n",
       "      <td>6.085526</td>\n",
       "      <td>6.126544</td>\n",
       "      <td>6.362746</td>\n",
       "      <td>16.252894</td>\n",
       "      <td>6.964893</td>\n",
       "      <td>6.265235</td>\n",
       "      <td>6.577442</td>\n",
       "      <td>6.888958</td>\n",
       "      <td>6.169747</td>\n",
       "      <td>1.395527</td>\n",
       "      <td>1.407226</td>\n",
       "      <td>1.564041</td>\n",
       "      <td>1.076268</td>\n",
       "      <td>1.778718</td>\n",
       "      <td>0.493597</td>\n",
       "      <td>1.950178</td>\n",
       "      <td>1.740575</td>\n",
       "      <td>1.550501</td>\n",
       "      <td>1.953816</td>\n",
       "      <td>1.794080</td>\n",
       "      <td>2.156163</td>\n",
       "      <td>1.841258</td>\n",
       "      <td>2.129354</td>\n",
       "      <td>0.989878</td>\n",
       "      <td>2.444377</td>\n",
       "      <td>1.821610</td>\n",
       "      <td>0.975682</td>\n",
       "      <td>0.631340</td>\n",
       "      <td>14.371856</td>\n",
       "      <td>6.657147</td>\n",
       "      <td>6.535516</td>\n",
       "      <td>6.124525</td>\n",
       "      <td>5.675667</td>\n",
       "      <td>6.652739</td>\n",
       "      <td>1.371758</td>\n",
       "      <td>1.503184</td>\n",
       "      <td>1.180390</td>\n",
       "      <td>1.548276</td>\n",
       "      <td>1.744980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.830000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>154.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>154.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.520000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.520000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>281.000000</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>281.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.640000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.640000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>19.050000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>408.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.430000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.810000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.810000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.630000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.500000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>22.220000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               iid      gender       condtn         wave        round  \\\n",
       "count  8368.000000  8368.00000  8368.000000  8368.000000  8368.000000   \n",
       "mean    283.863767     0.50000     1.829828    11.358509    16.880258   \n",
       "std     158.584899     0.50003     0.375806     5.995461     4.354579   \n",
       "min       1.000000     0.00000     1.000000     1.000000     5.000000   \n",
       "25%     154.000000     0.00000     2.000000     7.000000    14.000000   \n",
       "50%     281.000000     0.50000     2.000000    11.000000    18.000000   \n",
       "75%     408.000000     1.00000     2.000000    15.000000    20.000000   \n",
       "max     552.000000     1.00000     2.000000    21.000000    22.000000   \n",
       "\n",
       "          position        order          pid        match     int_corr  \\\n",
       "count  8368.000000  8368.000000  8368.000000  8368.000000  8210.000000   \n",
       "mean      9.048757     8.931764   283.863767     0.164914     0.196300   \n",
       "std       5.515476     5.478099   158.584899     0.371125     0.303522   \n",
       "min       1.000000     1.000000     1.000000     0.000000    -0.830000   \n",
       "25%       4.000000     4.000000   154.000000     0.000000    -0.010000   \n",
       "50%       8.000000     8.000000   281.000000     0.000000     0.210000   \n",
       "75%      13.000000    13.000000   408.000000     0.000000     0.430000   \n",
       "max      22.000000    22.000000   552.000000     1.000000     0.910000   \n",
       "\n",
       "          samerace        age_o       race_o     pf_o_att     pf_o_sin  \\\n",
       "count  8368.000000  8274.000000  8305.000000  8289.000000  8289.000000   \n",
       "mean      0.396272    26.364999     2.756653    22.495347    17.396867   \n",
       "std       0.489151     3.563648     1.230689    12.569802     7.044003   \n",
       "min       0.000000    18.000000     1.000000     0.000000     0.000000   \n",
       "25%       0.000000    24.000000     2.000000    15.000000    15.000000   \n",
       "50%       0.000000    26.000000     2.000000    20.000000    18.370000   \n",
       "75%       1.000000    28.000000     4.000000    25.000000    20.000000   \n",
       "max       1.000000    55.000000     6.000000   100.000000    60.000000   \n",
       "\n",
       "          pf_o_int     pf_o_fun     pf_o_amb     pf_o_sha        dec_o  \\\n",
       "count  8289.000000  8280.000000  8271.000000  8249.000000  8368.000000   \n",
       "mean     20.270759    17.459714    10.685375    11.845930     0.420053   \n",
       "std       6.782895     6.085526     6.126544     6.362746     0.493597   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%      17.390000    15.000000     5.000000     9.520000     0.000000   \n",
       "50%      20.000000    18.000000    10.000000    10.640000     0.000000   \n",
       "75%      23.810000    20.000000    15.000000    16.000000     1.000000   \n",
       "max      50.000000    50.000000    53.000000    30.000000     1.000000   \n",
       "\n",
       "            attr_o       sinc_o      intel_o        fun_o        amb_o  \\\n",
       "count  8166.000000  8091.000000  8072.000000  8018.000000  7656.000000   \n",
       "mean      6.190411     7.175256     7.369301     6.400599     6.778409   \n",
       "std       1.950305     1.740575     1.550501     1.954078     1.794080   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     6.000000     6.000000     5.000000     6.000000   \n",
       "50%       6.000000     7.000000     7.000000     7.000000     7.000000   \n",
       "75%       8.000000     8.000000     8.000000     8.000000     8.000000   \n",
       "max      10.500000    10.000000    10.000000    11.000000    10.000000   \n",
       "\n",
       "            shar_o       like_o       prob_o        met_o          age  \\\n",
       "count  7302.000000  8128.000000  8060.000000  7993.000000  8274.000000   \n",
       "mean      5.474870     6.134498     5.208251     1.960215    26.364999   \n",
       "std       2.156163     1.841258     2.129354     0.245925     3.563648   \n",
       "min       0.000000     0.000000     0.000000     1.000000    18.000000   \n",
       "25%       4.000000     5.000000     4.000000     2.000000    24.000000   \n",
       "50%       6.000000     6.000000     5.000000     2.000000    26.000000   \n",
       "75%       7.000000     7.000000     7.000000     2.000000    28.000000   \n",
       "max      10.000000    10.000000    10.000000     8.000000    55.000000   \n",
       "\n",
       "          field_cd         race      imprace     imprelig         goal  \\\n",
       "count  8286.000000  8305.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      7.665339     2.756653     3.785981     3.653879     2.121848   \n",
       "std       3.759695     1.230689     2.846506     2.805641     1.406983   \n",
       "min       1.000000     1.000000     0.000000     1.000000     1.000000   \n",
       "25%       5.000000     2.000000     1.000000     1.000000     1.000000   \n",
       "50%       8.000000     2.000000     3.000000     3.000000     2.000000   \n",
       "75%      10.000000     4.000000     6.000000     6.000000     2.000000   \n",
       "max      18.000000     6.000000    10.000000    10.000000     6.000000   \n",
       "\n",
       "              date       go_out     career_c       sports     tvsports  \\\n",
       "count  8271.000000  8289.000000  8230.000000  8289.000000  8289.000000   \n",
       "mean      5.007375     2.158523     5.277886     6.424418     4.574255   \n",
       "std       1.444439     1.105428     3.309885     2.619419     2.801521   \n",
       "min       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "25%       4.000000     1.000000     2.000000     4.000000     2.000000   \n",
       "50%       5.000000     2.000000     6.000000     7.000000     4.000000   \n",
       "75%       6.000000     3.000000     7.000000     9.000000     7.000000   \n",
       "max       7.000000     7.000000    17.000000    10.000000    10.000000   \n",
       "\n",
       "          exercise       dining      museums          art       hiking  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      6.244300     7.783448     6.987453     6.716371     5.737483   \n",
       "std       2.419612     1.755105     2.051600     2.262518     2.570595   \n",
       "min       1.000000     1.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     7.000000     6.000000     5.000000     4.000000   \n",
       "50%       6.000000     8.000000     7.000000     7.000000     6.000000   \n",
       "75%       8.000000     9.000000     9.000000     8.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "            gaming     clubbing      reading           tv      theater  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      3.880565     5.745325     7.680058     5.304379     6.778140   \n",
       "std       2.621188     2.502831     2.004806     2.529580     2.233341   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       2.000000     4.000000     7.000000     3.000000     5.000000   \n",
       "50%       3.000000     6.000000     8.000000     6.000000     7.000000   \n",
       "75%       6.000000     8.000000     9.000000     7.000000     9.000000   \n",
       "max      14.000000    10.000000    13.000000    10.000000    10.000000   \n",
       "\n",
       "            movies     concerts        music     shopping         yoga  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      7.920618     6.824587     7.850887     5.631077     4.338038   \n",
       "std       1.700264     2.156155     1.792227     2.609472     2.717712   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       7.000000     5.000000     7.000000     4.000000     2.000000   \n",
       "50%       8.000000     7.000000     8.000000     6.000000     4.000000   \n",
       "75%       9.000000     8.000000     9.000000     8.000000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "          exphappy      attr1_1      sinc1_1     intel1_1       fun1_1  \\\n",
       "count  8267.000000  8289.000000  8289.000000  8289.000000  8280.000000   \n",
       "mean      5.534051    22.495347    17.396867    20.270759    17.459714   \n",
       "std       1.733627    12.569802     7.044003     6.782895     6.085526   \n",
       "min       1.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000    15.000000    15.000000    17.390000    15.000000   \n",
       "50%       6.000000    20.000000    18.370000    20.000000    18.000000   \n",
       "75%       7.000000    25.000000    20.000000    23.810000    20.000000   \n",
       "max      10.000000   100.000000    60.000000    50.000000    50.000000   \n",
       "\n",
       "            amb1_1      shar1_1      attr2_1      sinc2_1     intel2_1  \\\n",
       "count  8271.000000  8249.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean     10.685375    11.845930    30.361422    13.264852    14.417394   \n",
       "std       6.126544     6.362746    16.252894     6.964893     6.265235   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     9.520000    20.000000    10.000000    10.000000   \n",
       "50%      10.000000    10.640000    25.000000    15.000000    15.000000   \n",
       "75%      15.000000    16.000000    40.000000    18.750000    20.000000   \n",
       "max      53.000000    30.000000   100.000000    50.000000    40.000000   \n",
       "\n",
       "            fun2_1       amb2_1      shar2_1      attr3_1      sinc3_1  \\\n",
       "count  8289.000000  8280.000000  8280.000000  8263.000000  8263.000000   \n",
       "mean     18.427955    11.744583    11.855626     7.084352     8.295776   \n",
       "std       6.577442     6.888958     6.169747     1.395527     1.407226   \n",
       "min       0.000000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.000000     6.000000    10.000000     6.000000     8.000000   \n",
       "50%      20.000000    10.000000    10.000000     7.000000     8.000000   \n",
       "75%      20.000000    15.000000    15.630000     8.000000     9.000000   \n",
       "max      50.000000    50.000000    30.000000    10.000000    10.000000   \n",
       "\n",
       "            fun3_1     intel3_1       amb3_1          dec         attr  \\\n",
       "count  8263.000000  8263.000000  8263.000000  8368.000000  8166.000000   \n",
       "mean      7.704345     8.404454     7.577877     0.420053     6.190350   \n",
       "std       1.564041     1.076268     1.778718     0.493597     1.950178   \n",
       "min       2.000000     3.000000     2.000000     0.000000     0.000000   \n",
       "25%       7.000000     8.000000     7.000000     0.000000     5.000000   \n",
       "50%       8.000000     8.000000     8.000000     0.000000     6.000000   \n",
       "75%       9.000000     9.000000     9.000000     1.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000     1.000000    10.000000   \n",
       "\n",
       "              sinc        intel          fun          amb         shar  \\\n",
       "count  8091.000000  8072.000000  8018.000000  7656.000000  7302.000000   \n",
       "mean      7.175256     7.369301     6.400474     6.778409     5.474870   \n",
       "std       1.740575     1.550501     1.953816     1.794080     2.156163   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       6.000000     6.000000     5.000000     6.000000     4.000000   \n",
       "50%       7.000000     7.000000     7.000000     7.000000     6.000000   \n",
       "75%       8.000000     8.000000     8.000000     8.000000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "              like         prob          met     match_es      satis_2  \\\n",
       "count  8128.000000  8060.000000  7993.000000  7195.000000  7454.000000   \n",
       "mean      6.134498     5.208251     0.947579     3.206991     5.711833   \n",
       "std       1.841258     2.129354     0.989878     2.444377     1.821610   \n",
       "min       0.000000     0.000000     0.000000     0.000000     1.000000   \n",
       "25%       5.000000     4.000000     0.000000     2.000000     5.000000   \n",
       "50%       6.000000     5.000000     0.000000     3.000000     6.000000   \n",
       "75%       7.000000     7.000000     2.000000     4.000000     7.000000   \n",
       "max      10.000000    10.000000     8.000000    18.000000    10.000000   \n",
       "\n",
       "            length     numdat_2      attr1_2      sinc1_2     intel1_2  \\\n",
       "count  7454.000000  7424.000000  7436.000000  7454.000000  7454.000000   \n",
       "mean      1.843708     2.337823    26.197823    15.872299    17.817152   \n",
       "std       0.975682     0.631340    14.371856     6.657147     6.535516   \n",
       "min       1.000000     1.000000     5.000000     0.000000     0.000000   \n",
       "25%       1.000000     2.000000    16.670000    10.000000    15.000000   \n",
       "50%       1.000000     2.000000    20.000000    16.670000    19.050000   \n",
       "75%       3.000000     3.000000    30.000000    20.000000    20.000000   \n",
       "max       3.000000     3.000000    85.000000    50.000000    40.000000   \n",
       "\n",
       "            fun1_2       amb1_2      shar1_2      attr3_2      sinc3_2  \\\n",
       "count  7454.000000  7454.000000  7454.000000  7454.000000  7454.000000   \n",
       "mean     17.656897     9.918161    12.762388     7.124497     7.931983   \n",
       "std       6.124525     5.675667     6.652739     1.371758     1.503184   \n",
       "min       0.000000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.000000     5.000000    10.000000     6.250000     7.000000   \n",
       "50%      18.370000    10.000000    13.000000     7.000000     8.000000   \n",
       "75%      20.000000    15.000000    16.670000     8.000000     9.000000   \n",
       "max      50.000000    22.220000    35.000000    10.000000    10.000000   \n",
       "\n",
       "          intel3_2       fun3_2       amb3_2  \n",
       "count  7454.000000  7454.000000  7454.000000  \n",
       "mean      8.238664     7.601959     7.486048  \n",
       "std       1.180390     1.548276     1.744980  \n",
       "min       4.000000     1.000000     2.000000  \n",
       "25%       8.000000     7.000000     7.000000  \n",
       "50%       8.000000     8.000000     8.000000  \n",
       "75%       9.000000     9.000000     9.000000  \n",
       "max      10.000000    10.000000    10.000000  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean_mv.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "attr_o\n",
       "6.0     1655\n",
       "7.0     1642\n",
       "5.0     1260\n",
       "8.0     1230\n",
       "4.0      748\n",
       "9.0      540\n",
       "3.0      390\n",
       "10.0     324\n",
       "2.0      244\n",
       "1.0      108\n",
       "0.0        8\n",
       "6.5        7\n",
       "7.5        3\n",
       "9.5        3\n",
       "8.5        1\n",
       "9.9        1\n",
       "10.5       1\n",
       "3.5        1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Vérification des valeurs attr_o \n",
    "# -> Nous avons une valeur attr_o qui vaut 10.5\n",
    "data_clean_mv['attr_o'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subject's score card: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>pid</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6206</th>\n",
       "      <td>404</td>\n",
       "      <td>383.0</td>\n",
       "      <td>10.5</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      iid    pid  attr_o  sinc_o  intel_o  fun_o  amb_o  shar_o  attr  sinc  \\\n",
       "6206  404  383.0    10.5     9.0      7.0    7.0    NaN     NaN   8.0   7.0   \n",
       "\n",
       "      intel  fun  amb  shar  \n",
       "6206    6.0  5.0  6.0   NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Partner's score card: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>pid</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5817</th>\n",
       "      <td>383</td>\n",
       "      <td>404.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      iid    pid  attr_o  sinc_o  intel_o  fun_o  amb_o  shar_o  attr  sinc  \\\n",
       "5817  383  404.0     8.0     7.0      6.0    5.0    6.0     NaN  10.0   9.0   \n",
       "\n",
       "      intel  fun  amb  shar  \n",
       "5817    7.0  7.0  NaN   NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Ici il s'agit d'une erreur de saisie\n",
    "# -> Nous pouvons retrouver la valeur correcte en regardant la fiche du partenaire (où attr=10)\n",
    "print(\"Subject's score card: \")\n",
    "display(data_clean_mv[data_clean_mv['attr_o'] == 10.5].iloc[:, np.r_[0:1,7:8,20:26,74:80]])\n",
    "print(\"Partner's score card: \")\n",
    "display(data_clean_mv[(data_clean_mv['pid']==404.0) & (data_clean_mv['iid']==383)].iloc[:, np.r_[0:1,7:8,20:26,74:80]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application de la correction\n",
    "new_value = 10\n",
    "data_clean_mv.loc[6216, 'attr_o'] = new_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fun_o\n",
       "7.0     1657\n",
       "6.0     1529\n",
       "8.0     1453\n",
       "5.0     1157\n",
       "4.0      605\n",
       "9.0      600\n",
       "10.0     386\n",
       "3.0      281\n",
       "2.0      220\n",
       "1.0      107\n",
       "0.0       14\n",
       "5.5        2\n",
       "6.5        2\n",
       "7.5        2\n",
       "9.5        1\n",
       "8.5        1\n",
       "11.0       1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Vérification des valeurs fun_o\n",
    "# -> Nous avons une valeur fun_o qui vaut 11\n",
    "data_clean_mv['fun_o'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subject's score card: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>pid</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6598</th>\n",
       "      <td>436</td>\n",
       "      <td>440.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      iid    pid  attr_o  sinc_o  intel_o  fun_o  amb_o  shar_o  attr  sinc  \\\n",
       "6598  436  440.0    10.0    10.0     10.0   11.0   10.0    10.0  10.0  10.0   \n",
       "\n",
       "      intel   fun   amb  shar  \n",
       "6598   10.0  10.0  10.0  10.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Partner's score card: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>pid</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6660</th>\n",
       "      <td>440</td>\n",
       "      <td>436.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      iid    pid  attr_o  sinc_o  intel_o  fun_o  amb_o  shar_o  attr  sinc  \\\n",
       "6660  440  436.0    10.0    10.0     10.0   10.0   10.0    10.0  10.0  10.0   \n",
       "\n",
       "      intel   fun   amb  shar  \n",
       "6660   10.0  10.0  10.0  10.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Ici il s'agit d'une erreur de saisie\n",
    "# -> Nous pouvons retrouver la valeur correcte en regardant la fiche du partenaire (où fun=10)\n",
    "print(\"Subject's score card: \")\n",
    "display(data_clean_mv[data_clean_mv['fun_o'] == 11].iloc[:, np.r_[0:1,7:8,20:26,74:80]])\n",
    "print(\"Partner's score card: \")\n",
    "display(data_clean_mv[(data_clean_mv['pid']==436.0) & (data_clean_mv['iid']==440)].iloc[:, np.r_[0:1,7:8,20:26,74:80]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application de la correction\n",
    "new_value = 10\n",
    "data_clean_mv.loc[6608, 'fun_o'] = new_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "met\n",
      "0.0    4047\n",
      "2.0    3588\n",
      "1.0     350\n",
      "7.0       3\n",
      "5.0       2\n",
      "3.0       1\n",
      "8.0       1\n",
      "6.0       1\n",
      "Name: count, dtype: int64\n",
      "met_o\n",
      "2.0    7635\n",
      "1.0     350\n",
      "7.0       3\n",
      "5.0       2\n",
      "3.0       1\n",
      "6.0       1\n",
      "8.0       1\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Vérification des valeurs met et met_o\n",
    "# -> Nous trouvons des valeurs supérieures à 2\n",
    "print(data_clean_mv['met'].value_counts())\n",
    "print(data_clean_mv['met_o'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Errors met values:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>met_o</th>\n",
       "      <th>met</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>826</th>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1072</th>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2414</th>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2415</th>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2481</th>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2856</th>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6657</th>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7517</th>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      met_o  met\n",
       "826     2.0  3.0\n",
       "1072    2.0  5.0\n",
       "2414    1.0  7.0\n",
       "2415    2.0  8.0\n",
       "2481    2.0  6.0\n",
       "2856    1.0  7.0\n",
       "6657    2.0  7.0\n",
       "7517    2.0  5.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Errors met_o values:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>met_o</th>\n",
       "      <th>met</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515</th>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2536</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2559</th>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2569</th>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3370</th>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6556</th>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8043</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      met_o  met\n",
       "989     3.0  2.0\n",
       "1515    5.0  2.0\n",
       "2536    6.0  2.0\n",
       "2559    7.0  1.0\n",
       "2569    8.0  2.0\n",
       "3370    7.0  1.0\n",
       "6556    7.0  2.0\n",
       "8043    5.0  0.0"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Nous supposons qu'il s'agit d'erreurs de saisie\n",
    "# -> Nous corrigeons avec la valeur met_o ou met correspondante\n",
    "\n",
    "print('Errors met values:')\n",
    "met_errors = data_clean_mv[data_clean_mv['met'] > 2]\n",
    "display(met_errors.iloc[:, np.r_[28:29,82:83]])\n",
    "\n",
    "print('Errors met_o values:')\n",
    "met_o_errors = data_clean_mv[data_clean_mv['met_o'] > 2]\n",
    "met_o_errors.iloc[:, np.r_[28:29,82:83]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application de la correction pour met\n",
    "new_value_1 = met_errors[met_errors['met_o'] == 1.0].index\n",
    "new_value_2 = met_errors[met_errors['met_o'] == 2.0].index\n",
    "\n",
    "for i in new_value_1:\n",
    "    data_clean_mv.loc[i, 'met'] = 1.0\n",
    "for i in new_value_2:\n",
    "    data_clean_mv.loc[i, 'met'] = 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application de la correction pour met_o\n",
    "new_value_1 = met_o_errors[met_o_errors['met'] == 1.0].index\n",
    "new_value_2 = met_o_errors[(met_o_errors['met'] == 2.0) | (met_o_errors['met'] == 0)].index\n",
    "\n",
    "for i in new_value_1:\n",
    "    data_clean_mv.loc[i, 'met_o'] = 1.0\n",
    "for i in new_value_2:\n",
    "    data_clean_mv.loc[i, 'met_o'] = 2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gaming\n",
      "1.0     1982\n",
      "2.0     1173\n",
      "3.0     1078\n",
      "5.0     1021\n",
      "6.0      759\n",
      "7.0      733\n",
      "4.0      710\n",
      "8.0      429\n",
      "9.0      220\n",
      "14.0      78\n",
      "0.0       59\n",
      "10.0      47\n",
      "Name: count, dtype: int64\n",
      "reading\n",
      "9.0     1998\n",
      "8.0     1616\n",
      "10.0    1377\n",
      "7.0     1273\n",
      "6.0      766\n",
      "5.0      571\n",
      "3.0      246\n",
      "4.0      222\n",
      "2.0      160\n",
      "13.0      51\n",
      "1.0        9\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Vérification des valeurs reading et gaming\n",
    "# -> Nous trouvons les valeurs erronées 13 et 14\n",
    "print(data_clean_mv['gaming'].value_counts())\n",
    "print(data_clean_mv['reading'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     gender  sports  tvsports  exercise  dining  museums   art  hiking  \\\n",
       "iid                                                                      \n",
       "142       0     5.0       6.0       6.0     6.0      5.0   5.0     6.0   \n",
       "240       0    10.0      10.0       9.0    10.0     10.0   8.0     9.0   \n",
       "285       1     5.0       7.0       6.0    10.0      9.0  10.0     9.0   \n",
       "441       1     6.0       1.0       6.0     7.0      7.0   8.0     7.0   \n",
       "544       1     3.0       4.0       3.0    10.0      7.0   8.0     5.0   \n",
       "\n",
       "     gaming  clubbing  reading    tv  theater  movies  concerts  music  \\\n",
       "iid                                                                      \n",
       "142    14.0       7.0      6.0   6.0      6.0     6.0       6.0    8.0   \n",
       "240    14.0       9.0     10.0  10.0     10.0    10.0      10.0   10.0   \n",
       "285    14.0       5.0      9.0   7.0      3.0    10.0      10.0   10.0   \n",
       "441    14.0       9.0      9.0   8.0      9.0    10.0      10.0   10.0   \n",
       "544    14.0       9.0      2.0  10.0      7.0     8.0       8.0    9.0   \n",
       "\n",
       "     shopping  yoga  \n",
       "iid                  \n",
       "142       6.0   6.0  \n",
       "240      10.0   3.0  \n",
       "285      10.0  10.0  \n",
       "441       7.0   2.0  \n",
       "544      10.0   3.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     gender  sports  tvsports  exercise  dining  museums  art  hiking  gaming  \\\n",
       "iid                                                                             \n",
       "216       1     2.0       1.0       2.0     7.0      7.0  5.0     3.0     5.0   \n",
       "244       1     5.0       2.0      10.0     8.0      6.0  4.0     9.0     1.0   \n",
       "523       0     8.0       1.0       5.0     9.0      4.0  3.0     6.0     7.0   \n",
       "\n",
       "     clubbing  reading   tv  theater  movies  concerts  music  shopping  yoga  \n",
       "iid                                                                            \n",
       "216       1.0     13.0  6.0      6.0     7.0       6.0    5.0       2.0   1.0  \n",
       "244       3.0     13.0  9.0      8.0     9.0       3.0    4.0       3.0   1.0  \n",
       "523       8.0     13.0  5.0      2.0     7.0       8.0    9.0       3.0   1.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(data_clean_mv[data_clean_mv['gaming'] == 14].iloc[:, np.r_[:, 0:2, 38:55]].groupby('iid').min())\n",
    "display(data_clean_mv[data_clean_mv['reading'] == 13].iloc[:, np.r_[:, 0:2, 38:55]].groupby('iid').min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ici nous assumons qu'il s'agit d'erreurs de saisie\n",
    "# Étant donné que nous sommes incertains sur la valeur réelle, nous choisissons de les remplacer par la médiane\n",
    "gaming_rows_to_correct = data_clean_mv[data_clean_mv['gaming'] == 14].index\n",
    "reading_rows_to_correct = data_clean_mv[data_clean_mv['reading'] == 13].index\n",
    "\n",
    "gaming_median = data_clean_mv.groupby('iid').max()['gaming'].median()\n",
    "reading_median = data_clean_mv.groupby('iid').max()['reading'].median()\n",
    "\n",
    "for i in gaming_rows_to_correct:\n",
    "    data_clean_mv.loc[i, 'gaming'] = gaming_median\n",
    "    \n",
    "for i in reading_rows_to_correct:\n",
    "    data_clean_mv.loc[i, 'reading'] = reading_median"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Valeurs aberrantes (outliers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>gender</th>\n",
       "      <th>condtn</th>\n",
       "      <th>wave</th>\n",
       "      <th>round</th>\n",
       "      <th>position</th>\n",
       "      <th>order</th>\n",
       "      <th>pid</th>\n",
       "      <th>match</th>\n",
       "      <th>int_corr</th>\n",
       "      <th>samerace</th>\n",
       "      <th>age_o</th>\n",
       "      <th>race_o</th>\n",
       "      <th>pf_o_att</th>\n",
       "      <th>pf_o_sin</th>\n",
       "      <th>pf_o_int</th>\n",
       "      <th>pf_o_fun</th>\n",
       "      <th>pf_o_amb</th>\n",
       "      <th>pf_o_sha</th>\n",
       "      <th>dec_o</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>like_o</th>\n",
       "      <th>prob_o</th>\n",
       "      <th>met_o</th>\n",
       "      <th>age</th>\n",
       "      <th>field_cd</th>\n",
       "      <th>race</th>\n",
       "      <th>imprace</th>\n",
       "      <th>imprelig</th>\n",
       "      <th>goal</th>\n",
       "      <th>date</th>\n",
       "      <th>go_out</th>\n",
       "      <th>career_c</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>attr2_1</th>\n",
       "      <th>sinc2_1</th>\n",
       "      <th>intel2_1</th>\n",
       "      <th>fun2_1</th>\n",
       "      <th>amb2_1</th>\n",
       "      <th>shar2_1</th>\n",
       "      <th>attr3_1</th>\n",
       "      <th>sinc3_1</th>\n",
       "      <th>fun3_1</th>\n",
       "      <th>intel3_1</th>\n",
       "      <th>amb3_1</th>\n",
       "      <th>dec</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "      <th>like</th>\n",
       "      <th>prob</th>\n",
       "      <th>met</th>\n",
       "      <th>match_es</th>\n",
       "      <th>satis_2</th>\n",
       "      <th>length</th>\n",
       "      <th>numdat_2</th>\n",
       "      <th>attr1_2</th>\n",
       "      <th>sinc1_2</th>\n",
       "      <th>intel1_2</th>\n",
       "      <th>fun1_2</th>\n",
       "      <th>amb1_2</th>\n",
       "      <th>shar1_2</th>\n",
       "      <th>attr3_2</th>\n",
       "      <th>sinc3_2</th>\n",
       "      <th>intel3_2</th>\n",
       "      <th>fun3_2</th>\n",
       "      <th>amb3_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.00000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8210.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8274.000000</td>\n",
       "      <td>8305.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8249.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8166.000000</td>\n",
       "      <td>8091.000000</td>\n",
       "      <td>8072.000000</td>\n",
       "      <td>8018.000000</td>\n",
       "      <td>7656.000000</td>\n",
       "      <td>7302.000000</td>\n",
       "      <td>8128.000000</td>\n",
       "      <td>8060.000000</td>\n",
       "      <td>7993.000000</td>\n",
       "      <td>8274.000000</td>\n",
       "      <td>8286.000000</td>\n",
       "      <td>8305.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8230.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8267.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8271.000000</td>\n",
       "      <td>8249.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8289.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8280.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8263.000000</td>\n",
       "      <td>8368.000000</td>\n",
       "      <td>8166.000000</td>\n",
       "      <td>8091.000000</td>\n",
       "      <td>8072.000000</td>\n",
       "      <td>8018.000000</td>\n",
       "      <td>7656.000000</td>\n",
       "      <td>7302.000000</td>\n",
       "      <td>8128.000000</td>\n",
       "      <td>8060.000000</td>\n",
       "      <td>7993.000000</td>\n",
       "      <td>7195.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7424.000000</td>\n",
       "      <td>7436.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "      <td>7454.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>283.863767</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>1.829828</td>\n",
       "      <td>11.358509</td>\n",
       "      <td>16.880258</td>\n",
       "      <td>9.048757</td>\n",
       "      <td>8.931764</td>\n",
       "      <td>283.863767</td>\n",
       "      <td>0.164914</td>\n",
       "      <td>0.196300</td>\n",
       "      <td>0.396272</td>\n",
       "      <td>26.364999</td>\n",
       "      <td>2.756653</td>\n",
       "      <td>22.495347</td>\n",
       "      <td>17.396867</td>\n",
       "      <td>20.270759</td>\n",
       "      <td>17.459714</td>\n",
       "      <td>10.685375</td>\n",
       "      <td>11.845930</td>\n",
       "      <td>0.420053</td>\n",
       "      <td>6.190779</td>\n",
       "      <td>7.175256</td>\n",
       "      <td>7.369301</td>\n",
       "      <td>6.400973</td>\n",
       "      <td>6.778409</td>\n",
       "      <td>5.474870</td>\n",
       "      <td>6.134498</td>\n",
       "      <td>5.208251</td>\n",
       "      <td>1.955961</td>\n",
       "      <td>26.364999</td>\n",
       "      <td>7.665339</td>\n",
       "      <td>2.756653</td>\n",
       "      <td>3.785981</td>\n",
       "      <td>3.653879</td>\n",
       "      <td>2.121848</td>\n",
       "      <td>5.007375</td>\n",
       "      <td>2.158523</td>\n",
       "      <td>5.277886</td>\n",
       "      <td>6.424418</td>\n",
       "      <td>4.574255</td>\n",
       "      <td>6.244300</td>\n",
       "      <td>7.783448</td>\n",
       "      <td>6.987453</td>\n",
       "      <td>6.716371</td>\n",
       "      <td>5.737483</td>\n",
       "      <td>3.777054</td>\n",
       "      <td>5.745325</td>\n",
       "      <td>7.649294</td>\n",
       "      <td>5.304379</td>\n",
       "      <td>6.778140</td>\n",
       "      <td>7.920618</td>\n",
       "      <td>6.824587</td>\n",
       "      <td>7.850887</td>\n",
       "      <td>5.631077</td>\n",
       "      <td>4.338038</td>\n",
       "      <td>5.534051</td>\n",
       "      <td>22.495347</td>\n",
       "      <td>17.396867</td>\n",
       "      <td>20.270759</td>\n",
       "      <td>17.459714</td>\n",
       "      <td>10.685375</td>\n",
       "      <td>11.845930</td>\n",
       "      <td>30.361422</td>\n",
       "      <td>13.264852</td>\n",
       "      <td>14.417394</td>\n",
       "      <td>18.427955</td>\n",
       "      <td>11.744583</td>\n",
       "      <td>11.855626</td>\n",
       "      <td>7.084352</td>\n",
       "      <td>8.295776</td>\n",
       "      <td>7.704345</td>\n",
       "      <td>8.404454</td>\n",
       "      <td>7.577877</td>\n",
       "      <td>0.420053</td>\n",
       "      <td>6.190350</td>\n",
       "      <td>7.175256</td>\n",
       "      <td>7.369301</td>\n",
       "      <td>6.400474</td>\n",
       "      <td>6.778409</td>\n",
       "      <td>5.474870</td>\n",
       "      <td>6.134498</td>\n",
       "      <td>5.208251</td>\n",
       "      <td>0.943325</td>\n",
       "      <td>3.206991</td>\n",
       "      <td>5.711833</td>\n",
       "      <td>1.843708</td>\n",
       "      <td>2.337823</td>\n",
       "      <td>26.197823</td>\n",
       "      <td>15.872299</td>\n",
       "      <td>17.817152</td>\n",
       "      <td>17.656897</td>\n",
       "      <td>9.918161</td>\n",
       "      <td>12.762388</td>\n",
       "      <td>7.124497</td>\n",
       "      <td>7.931983</td>\n",
       "      <td>8.238664</td>\n",
       "      <td>7.601959</td>\n",
       "      <td>7.486048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>158.584899</td>\n",
       "      <td>0.50003</td>\n",
       "      <td>0.375806</td>\n",
       "      <td>5.995461</td>\n",
       "      <td>4.354579</td>\n",
       "      <td>5.515476</td>\n",
       "      <td>5.478099</td>\n",
       "      <td>158.584899</td>\n",
       "      <td>0.371125</td>\n",
       "      <td>0.303522</td>\n",
       "      <td>0.489151</td>\n",
       "      <td>3.563648</td>\n",
       "      <td>1.230689</td>\n",
       "      <td>12.569802</td>\n",
       "      <td>7.044003</td>\n",
       "      <td>6.782895</td>\n",
       "      <td>6.085526</td>\n",
       "      <td>6.126544</td>\n",
       "      <td>6.362746</td>\n",
       "      <td>0.493597</td>\n",
       "      <td>1.950740</td>\n",
       "      <td>1.740575</td>\n",
       "      <td>1.550501</td>\n",
       "      <td>1.954480</td>\n",
       "      <td>1.794080</td>\n",
       "      <td>2.156163</td>\n",
       "      <td>1.841258</td>\n",
       "      <td>2.129354</td>\n",
       "      <td>0.205194</td>\n",
       "      <td>3.563648</td>\n",
       "      <td>3.759695</td>\n",
       "      <td>1.230689</td>\n",
       "      <td>2.846506</td>\n",
       "      <td>2.805641</td>\n",
       "      <td>1.406983</td>\n",
       "      <td>1.444439</td>\n",
       "      <td>1.105428</td>\n",
       "      <td>3.309885</td>\n",
       "      <td>2.619419</td>\n",
       "      <td>2.801521</td>\n",
       "      <td>2.419612</td>\n",
       "      <td>1.755105</td>\n",
       "      <td>2.051600</td>\n",
       "      <td>2.262518</td>\n",
       "      <td>2.570595</td>\n",
       "      <td>2.429707</td>\n",
       "      <td>2.502831</td>\n",
       "      <td>1.960810</td>\n",
       "      <td>2.529580</td>\n",
       "      <td>2.233341</td>\n",
       "      <td>1.700264</td>\n",
       "      <td>2.156155</td>\n",
       "      <td>1.792227</td>\n",
       "      <td>2.609472</td>\n",
       "      <td>2.717712</td>\n",
       "      <td>1.733627</td>\n",
       "      <td>12.569802</td>\n",
       "      <td>7.044003</td>\n",
       "      <td>6.782895</td>\n",
       "      <td>6.085526</td>\n",
       "      <td>6.126544</td>\n",
       "      <td>6.362746</td>\n",
       "      <td>16.252894</td>\n",
       "      <td>6.964893</td>\n",
       "      <td>6.265235</td>\n",
       "      <td>6.577442</td>\n",
       "      <td>6.888958</td>\n",
       "      <td>6.169747</td>\n",
       "      <td>1.395527</td>\n",
       "      <td>1.407226</td>\n",
       "      <td>1.564041</td>\n",
       "      <td>1.076268</td>\n",
       "      <td>1.778718</td>\n",
       "      <td>0.493597</td>\n",
       "      <td>1.950178</td>\n",
       "      <td>1.740575</td>\n",
       "      <td>1.550501</td>\n",
       "      <td>1.953816</td>\n",
       "      <td>1.794080</td>\n",
       "      <td>2.156163</td>\n",
       "      <td>1.841258</td>\n",
       "      <td>2.129354</td>\n",
       "      <td>0.976150</td>\n",
       "      <td>2.444377</td>\n",
       "      <td>1.821610</td>\n",
       "      <td>0.975682</td>\n",
       "      <td>0.631340</td>\n",
       "      <td>14.371856</td>\n",
       "      <td>6.657147</td>\n",
       "      <td>6.535516</td>\n",
       "      <td>6.124525</td>\n",
       "      <td>5.675667</td>\n",
       "      <td>6.652739</td>\n",
       "      <td>1.371758</td>\n",
       "      <td>1.503184</td>\n",
       "      <td>1.180390</td>\n",
       "      <td>1.548276</td>\n",
       "      <td>1.744980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.830000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>154.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>154.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.010000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.520000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.520000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>281.000000</td>\n",
       "      <td>0.50000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>281.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.640000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.640000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>19.050000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>408.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.430000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.810000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.810000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.630000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.500000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>22.220000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               iid      gender       condtn         wave        round  \\\n",
       "count  8368.000000  8368.00000  8368.000000  8368.000000  8368.000000   \n",
       "mean    283.863767     0.50000     1.829828    11.358509    16.880258   \n",
       "std     158.584899     0.50003     0.375806     5.995461     4.354579   \n",
       "min       1.000000     0.00000     1.000000     1.000000     5.000000   \n",
       "25%     154.000000     0.00000     2.000000     7.000000    14.000000   \n",
       "50%     281.000000     0.50000     2.000000    11.000000    18.000000   \n",
       "75%     408.000000     1.00000     2.000000    15.000000    20.000000   \n",
       "max     552.000000     1.00000     2.000000    21.000000    22.000000   \n",
       "\n",
       "          position        order          pid        match     int_corr  \\\n",
       "count  8368.000000  8368.000000  8368.000000  8368.000000  8210.000000   \n",
       "mean      9.048757     8.931764   283.863767     0.164914     0.196300   \n",
       "std       5.515476     5.478099   158.584899     0.371125     0.303522   \n",
       "min       1.000000     1.000000     1.000000     0.000000    -0.830000   \n",
       "25%       4.000000     4.000000   154.000000     0.000000    -0.010000   \n",
       "50%       8.000000     8.000000   281.000000     0.000000     0.210000   \n",
       "75%      13.000000    13.000000   408.000000     0.000000     0.430000   \n",
       "max      22.000000    22.000000   552.000000     1.000000     0.910000   \n",
       "\n",
       "          samerace        age_o       race_o     pf_o_att     pf_o_sin  \\\n",
       "count  8368.000000  8274.000000  8305.000000  8289.000000  8289.000000   \n",
       "mean      0.396272    26.364999     2.756653    22.495347    17.396867   \n",
       "std       0.489151     3.563648     1.230689    12.569802     7.044003   \n",
       "min       0.000000    18.000000     1.000000     0.000000     0.000000   \n",
       "25%       0.000000    24.000000     2.000000    15.000000    15.000000   \n",
       "50%       0.000000    26.000000     2.000000    20.000000    18.370000   \n",
       "75%       1.000000    28.000000     4.000000    25.000000    20.000000   \n",
       "max       1.000000    55.000000     6.000000   100.000000    60.000000   \n",
       "\n",
       "          pf_o_int     pf_o_fun     pf_o_amb     pf_o_sha        dec_o  \\\n",
       "count  8289.000000  8280.000000  8271.000000  8249.000000  8368.000000   \n",
       "mean     20.270759    17.459714    10.685375    11.845930     0.420053   \n",
       "std       6.782895     6.085526     6.126544     6.362746     0.493597   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%      17.390000    15.000000     5.000000     9.520000     0.000000   \n",
       "50%      20.000000    18.000000    10.000000    10.640000     0.000000   \n",
       "75%      23.810000    20.000000    15.000000    16.000000     1.000000   \n",
       "max      50.000000    50.000000    53.000000    30.000000     1.000000   \n",
       "\n",
       "            attr_o       sinc_o      intel_o        fun_o        amb_o  \\\n",
       "count  8166.000000  8091.000000  8072.000000  8018.000000  7656.000000   \n",
       "mean      6.190779     7.175256     7.369301     6.400973     6.778409   \n",
       "std       1.950740     1.740575     1.550501     1.954480     1.794080   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     6.000000     6.000000     5.000000     6.000000   \n",
       "50%       6.000000     7.000000     7.000000     7.000000     7.000000   \n",
       "75%       8.000000     8.000000     8.000000     8.000000     8.000000   \n",
       "max      10.500000    10.000000    10.000000    11.000000    10.000000   \n",
       "\n",
       "            shar_o       like_o       prob_o        met_o          age  \\\n",
       "count  7302.000000  8128.000000  8060.000000  7993.000000  8274.000000   \n",
       "mean      5.474870     6.134498     5.208251     1.955961    26.364999   \n",
       "std       2.156163     1.841258     2.129354     0.205194     3.563648   \n",
       "min       0.000000     0.000000     0.000000     1.000000    18.000000   \n",
       "25%       4.000000     5.000000     4.000000     2.000000    24.000000   \n",
       "50%       6.000000     6.000000     5.000000     2.000000    26.000000   \n",
       "75%       7.000000     7.000000     7.000000     2.000000    28.000000   \n",
       "max      10.000000    10.000000    10.000000     2.000000    55.000000   \n",
       "\n",
       "          field_cd         race      imprace     imprelig         goal  \\\n",
       "count  8286.000000  8305.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      7.665339     2.756653     3.785981     3.653879     2.121848   \n",
       "std       3.759695     1.230689     2.846506     2.805641     1.406983   \n",
       "min       1.000000     1.000000     0.000000     1.000000     1.000000   \n",
       "25%       5.000000     2.000000     1.000000     1.000000     1.000000   \n",
       "50%       8.000000     2.000000     3.000000     3.000000     2.000000   \n",
       "75%      10.000000     4.000000     6.000000     6.000000     2.000000   \n",
       "max      18.000000     6.000000    10.000000    10.000000     6.000000   \n",
       "\n",
       "              date       go_out     career_c       sports     tvsports  \\\n",
       "count  8271.000000  8289.000000  8230.000000  8289.000000  8289.000000   \n",
       "mean      5.007375     2.158523     5.277886     6.424418     4.574255   \n",
       "std       1.444439     1.105428     3.309885     2.619419     2.801521   \n",
       "min       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "25%       4.000000     1.000000     2.000000     4.000000     2.000000   \n",
       "50%       5.000000     2.000000     6.000000     7.000000     4.000000   \n",
       "75%       6.000000     3.000000     7.000000     9.000000     7.000000   \n",
       "max       7.000000     7.000000    17.000000    10.000000    10.000000   \n",
       "\n",
       "          exercise       dining      museums          art       hiking  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      6.244300     7.783448     6.987453     6.716371     5.737483   \n",
       "std       2.419612     1.755105     2.051600     2.262518     2.570595   \n",
       "min       1.000000     1.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     7.000000     6.000000     5.000000     4.000000   \n",
       "50%       6.000000     8.000000     7.000000     7.000000     6.000000   \n",
       "75%       8.000000     9.000000     9.000000     8.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "            gaming     clubbing      reading           tv      theater  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      3.777054     5.745325     7.649294     5.304379     6.778140   \n",
       "std       2.429707     2.502831     1.960810     2.529580     2.233341   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       2.000000     4.000000     7.000000     3.000000     5.000000   \n",
       "50%       3.000000     6.000000     8.000000     6.000000     7.000000   \n",
       "75%       6.000000     8.000000     9.000000     7.000000     9.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "            movies     concerts        music     shopping         yoga  \\\n",
       "count  8289.000000  8289.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean      7.920618     6.824587     7.850887     5.631077     4.338038   \n",
       "std       1.700264     2.156155     1.792227     2.609472     2.717712   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       7.000000     5.000000     7.000000     4.000000     2.000000   \n",
       "50%       8.000000     7.000000     8.000000     6.000000     4.000000   \n",
       "75%       9.000000     8.000000     9.000000     8.000000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "          exphappy      attr1_1      sinc1_1     intel1_1       fun1_1  \\\n",
       "count  8267.000000  8289.000000  8289.000000  8289.000000  8280.000000   \n",
       "mean      5.534051    22.495347    17.396867    20.270759    17.459714   \n",
       "std       1.733627    12.569802     7.044003     6.782895     6.085526   \n",
       "min       1.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000    15.000000    15.000000    17.390000    15.000000   \n",
       "50%       6.000000    20.000000    18.370000    20.000000    18.000000   \n",
       "75%       7.000000    25.000000    20.000000    23.810000    20.000000   \n",
       "max      10.000000   100.000000    60.000000    50.000000    50.000000   \n",
       "\n",
       "            amb1_1      shar1_1      attr2_1      sinc2_1     intel2_1  \\\n",
       "count  8271.000000  8249.000000  8289.000000  8289.000000  8289.000000   \n",
       "mean     10.685375    11.845930    30.361422    13.264852    14.417394   \n",
       "std       6.126544     6.362746    16.252894     6.964893     6.265235   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     9.520000    20.000000    10.000000    10.000000   \n",
       "50%      10.000000    10.640000    25.000000    15.000000    15.000000   \n",
       "75%      15.000000    16.000000    40.000000    18.750000    20.000000   \n",
       "max      53.000000    30.000000   100.000000    50.000000    40.000000   \n",
       "\n",
       "            fun2_1       amb2_1      shar2_1      attr3_1      sinc3_1  \\\n",
       "count  8289.000000  8280.000000  8280.000000  8263.000000  8263.000000   \n",
       "mean     18.427955    11.744583    11.855626     7.084352     8.295776   \n",
       "std       6.577442     6.888958     6.169747     1.395527     1.407226   \n",
       "min       0.000000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.000000     6.000000    10.000000     6.000000     8.000000   \n",
       "50%      20.000000    10.000000    10.000000     7.000000     8.000000   \n",
       "75%      20.000000    15.000000    15.630000     8.000000     9.000000   \n",
       "max      50.000000    50.000000    30.000000    10.000000    10.000000   \n",
       "\n",
       "            fun3_1     intel3_1       amb3_1          dec         attr  \\\n",
       "count  8263.000000  8263.000000  8263.000000  8368.000000  8166.000000   \n",
       "mean      7.704345     8.404454     7.577877     0.420053     6.190350   \n",
       "std       1.564041     1.076268     1.778718     0.493597     1.950178   \n",
       "min       2.000000     3.000000     2.000000     0.000000     0.000000   \n",
       "25%       7.000000     8.000000     7.000000     0.000000     5.000000   \n",
       "50%       8.000000     8.000000     8.000000     0.000000     6.000000   \n",
       "75%       9.000000     9.000000     9.000000     1.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000     1.000000    10.000000   \n",
       "\n",
       "              sinc        intel          fun          amb         shar  \\\n",
       "count  8091.000000  8072.000000  8018.000000  7656.000000  7302.000000   \n",
       "mean      7.175256     7.369301     6.400474     6.778409     5.474870   \n",
       "std       1.740575     1.550501     1.953816     1.794080     2.156163   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       6.000000     6.000000     5.000000     6.000000     4.000000   \n",
       "50%       7.000000     7.000000     7.000000     7.000000     6.000000   \n",
       "75%       8.000000     8.000000     8.000000     8.000000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "              like         prob          met     match_es      satis_2  \\\n",
       "count  8128.000000  8060.000000  7993.000000  7195.000000  7454.000000   \n",
       "mean      6.134498     5.208251     0.943325     3.206991     5.711833   \n",
       "std       1.841258     2.129354     0.976150     2.444377     1.821610   \n",
       "min       0.000000     0.000000     0.000000     0.000000     1.000000   \n",
       "25%       5.000000     4.000000     0.000000     2.000000     5.000000   \n",
       "50%       6.000000     5.000000     0.000000     3.000000     6.000000   \n",
       "75%       7.000000     7.000000     2.000000     4.000000     7.000000   \n",
       "max      10.000000    10.000000     2.000000    18.000000    10.000000   \n",
       "\n",
       "            length     numdat_2      attr1_2      sinc1_2     intel1_2  \\\n",
       "count  7454.000000  7424.000000  7436.000000  7454.000000  7454.000000   \n",
       "mean      1.843708     2.337823    26.197823    15.872299    17.817152   \n",
       "std       0.975682     0.631340    14.371856     6.657147     6.535516   \n",
       "min       1.000000     1.000000     5.000000     0.000000     0.000000   \n",
       "25%       1.000000     2.000000    16.670000    10.000000    15.000000   \n",
       "50%       1.000000     2.000000    20.000000    16.670000    19.050000   \n",
       "75%       3.000000     3.000000    30.000000    20.000000    20.000000   \n",
       "max       3.000000     3.000000    85.000000    50.000000    40.000000   \n",
       "\n",
       "            fun1_2       amb1_2      shar1_2      attr3_2      sinc3_2  \\\n",
       "count  7454.000000  7454.000000  7454.000000  7454.000000  7454.000000   \n",
       "mean     17.656897     9.918161    12.762388     7.124497     7.931983   \n",
       "std       6.124525     5.675667     6.652739     1.371758     1.503184   \n",
       "min       0.000000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.000000     5.000000    10.000000     6.250000     7.000000   \n",
       "50%      18.370000    10.000000    13.000000     7.000000     8.000000   \n",
       "75%      20.000000    15.000000    16.670000     8.000000     9.000000   \n",
       "max      50.000000    22.220000    35.000000    10.000000    10.000000   \n",
       "\n",
       "          intel3_2       fun3_2       amb3_2  \n",
       "count  7454.000000  7454.000000  7454.000000  \n",
       "mean      8.238664     7.601959     7.486048  \n",
       "std       1.180390     1.548276     1.744980  \n",
       "min       4.000000     1.000000     2.000000  \n",
       "25%       8.000000     7.000000     7.000000  \n",
       "50%       8.000000     8.000000     8.000000  \n",
       "75%       9.000000     9.000000     9.000000  \n",
       "max      10.000000    10.000000    10.000000  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean_mv.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAACMCAYAAABYgFHKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAPw0lEQVR4nO3deWxUZdvH8V8X2sLbjc0uUAoVgUApq5IGqQYICVGiaIQAr69BAwFKwmIMEJBaLQGXfzQB8gCJxkcSIioajIKEpQjWpC2WAiGV8jQUUlqChGlZWkjnfv/qPBS6Dh3OPaffTzKhnXPP6XVxneHHzJzOhBhjjAAAgHVCnS4AAAC0jJAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEuF+3tDr9erqqoqxcTEKCQkpCtrAgDA1YwxqqurU3JyskJDW3+87HdIV1VVKSUlxd+bAwDQ7V2+fFkDBw5sdbvfIR0TE+P7AbGxsf7uBgCAbqe2tlYpKSm+LG2N3yHd9BR3bGwsIQ0AgB/ae7mYE8cAALAUIQ0AgKUIaQAALOX3a9J4cmpqauTxeJwuo1Pi4uKUkJDgdBkAENQIacvV1NTof9/8P92/1+B0KZ3SIyJS3/z7a4IaAB4DIW05j8ej+/cadDftBXmj4rpsv6F3b6pnxXHdHZIlb8/4LtuvJIXWe6T/5Mvj8RDSAPAYCOkg4Y2Kk/d/+nX9fnvGB2S/AIDHx4ljAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQBALCUVSFdX1+vv//+W/X19U6XAgQN7jeAe1kV0pWVlVq8eLEqKyudLgUIGtxvAPeyKqQBAMB/EdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAAS4U7XQCAJ6OxsVGlpaW6ceOG+vTpo4yMDIWFhT32Wn/WB4otdcA9nD6mCGmgGzh+/Li2bdum6upq33WJiYlatmyZsrKy/F7rz/pAsaUOuIcNxxRPdwMud/z4ceXk5CgtLU1bt27VL7/8oq1btyotLU05OTk6fvy4X2v9WW9Dj0BH2HJMEdKAizU2Nmrbtm3KzMxUXl6eRo0apV69emnUqFHKy8tTZmamtm/frsbGxk6t7ey+bekR6AibjqkOP93d0NCghoYG3/e1tbUBKUiSLl26FLB9B5tg/rsI5tqDSVt/z6Wlpaqurtb777+v0NDm/ycPDQ3VggULlJ2drdLSUknq8Npx48Z1at/jxo3rom4fr8dA1gH3sOmY6nBIb968Wbm5uYGsxWfTpk1P5OcgsJij827cuCFJGjJkSIvbm65vWteZtf7sOxBsqQPuYdMx1eGQXrdunVavXu37vra2VikpKQEpav369UpNTQ3IvoPNpUuXgjbsmOOT0dYx0qdPH0lSRUWFRo0a9cj2ioqKZus6s9affQeCLXXAPWw6pjoc0pGRkYqMjAxkLT6pqakaNmzYE/lZCBzm6LyMjAwlJiZq9+7dysvLa/bUndfr1e7du5WUlKSMjAxJ6tTazu7blh6B9th0THHiGOBiYWFhWrZsmQoKCrRhwwadO3dOd+7c0blz57RhwwYVFBRo6dKlCgsL69Tazu7blh6BjrDpmOL3pAGXy8rKUm5urrZt26bs7Gzf9UlJScrNzW32+56dWevP+kCxpQ64hy3HFCENdANZWVmaPHlyh945qTNr/VkfKLbUAfew4ZgipIFuIiwsrMO/LtKZtf6sDxRb6oB7OH1M8Zo0AACWIqQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwlFUhPWjQIO3YsUODBg1yuhQgaHC/AdzLqvfujoqK4vOHgU7ifgO4l1WPpAEAwH8R0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFJWvXc3Whda7+na/d292ezPLt13F9cKAN0VIW25uLg49YiIlP6TH5D996w4HpD99oiIVFxcXED2DQDdBSFtuYSEBH3z76/l8QTXo9O4uDglJCQ4XQYABDVCOggkJCQQeADQDXHiGAAAliKkAQCwFCENAIClCGkAACzl94ljxhhJUm1tbZcVAwBAd9CUnU1Z2hq/Q7qurk6SlJKS4u8uAADo1urq6tp8T4kQ016Mt8Lr9aqqqkoxMTEKCQnxu8Anpba2VikpKbp8+bJiY2OdLicg6NEd6NEdukOPUvfoMxA9GmNUV1en5ORkhYa2/sqz34+kQ0NDNXDgQH9v7pjY2FjXHkhN6NEd6NEdukOPUvfos6t77Mi7MnLiGAAAliKkAQCwVLcJ6cjISOXk5CgyMtLpUgKGHt2BHt2hO/QodY8+nezR7xPHAABAYHWbR9IAAAQbQhoAAEsR0gAAWIqQBgDAUq4K6c2bN+vZZ59VTEyMnnrqKb366qsqKytrtqa+vl7Z2dnq27evoqOj9frrr6umpsahijuvIz2++OKLCgkJaXZZsmSJQxV33vbt25WRkeF744DMzEz9+uuvvu3BPsMm7fUZ7HN82JYtWxQSEqKVK1f6rnPLLJu01KMb5vjBBx880sOIESN8290wx/Z6dGqOrgrp/Px8ZWdn688//9ShQ4d0//59zZgxQ7dv3/atWbVqlfbv36+9e/cqPz9fVVVVeu211xysunM60qMkLVq0SFevXvVdPvnkE4cq7ryBAwdqy5YtKi4uVlFRkaZOnapXXnlF586dkxT8M2zSXp9ScM/xQYWFhfrXv/6ljIyMZte7ZZZS6z1K7pjjqFGjmvVw4sQJ3za3zLGtHiWH5mhc7Nq1a0aSyc/PN8YYc/PmTdOjRw+zd+9e35rz588bSaagoMCpMh/Lwz0aY8wLL7xgVqxY4VxRAdC7d2+za9cuV87wQU19GuOeOdbV1ZlnnnnGHDp0qFlPbpplaz0a44455uTkmDFjxrS4zS1zbKtHY5ybo6seST/M4/FIkvr06SNJKi4u1v379zV9+nTfmhEjRmjQoEEqKChwpMbH9XCPTXbv3q1+/fopPT1d69at0507d5wo77E1NjZqz549un37tjIzM105Q+nRPpu4YY7Z2dl66aWXms1Mctf9sbUem7hhjhcuXFBycrLS0tK0YMECVVZWSnLXHFvrsYkTc/T7AzZs5/V6tXLlSk2ePFnp6emSpOrqakVERCg+Pr7Z2oSEBFVXVztQ5eNpqUdJmj9/vlJTU5WcnKzS0lKtWbNGZWVl+uGHHxystnPOnDmjzMxM1dfXKzo6Wvv27dPIkSNVUlLiqhm21qfkjjnu2bNHp06dUmFh4SPb3HJ/bKtHyR1znDRpkr766isNHz5cV69eVW5urqZMmaKzZ8+6Zo5t9RgTE+PYHF0b0tnZ2Tp79uwjrym4SWs9Ll682Pf16NGjlZSUpGnTpunixYt6+umnn3SZfhk+fLhKSkrk8Xj03Xff6a233lJ+fr7TZXW51vocOXJk0M/x8uXLWrFihQ4dOqSoqCinywmIjvQY7HOUpJkzZ/q+zsjI0KRJk5Samqpvv/1WPXv2dLCyrtNWj++8845jc3Tl093Lly/Xzz//rKNHjzb7OM3ExETdu3dPN2/ebLa+pqZGiYmJT7jKx9Najy2ZNGmSJKm8vPxJlNYlIiIiNHToUE2YMEGbN2/WmDFj9Pnnn7tqhlLrfbYk2OZYXFysa9euafz48QoPD1d4eLjy8/P1xRdfKDw8XAkJCUE/y/Z6bGxsfOQ2wTbHlsTHx2vYsGEqLy933X2yyYM9tuRJzdFVIW2M0fLly7Vv3z4dOXJEQ4YMabZ9woQJ6tGjhw4fPuy7rqysTJWVlc1eB7RZez22pKSkRJKUlJQU4OoCx+v1qqGhwRUzbEtTny0JtjlOmzZNZ86cUUlJie8yceJELViwwPd1sM+yvR7DwsIeuU2wzbElt27d0sWLF5WUlOTa++SDPbbkic3xiZ+qFkBLly41cXFx5tixY+bq1au+y507d3xrlixZYgYNGmSOHDliioqKTGZmpsnMzHSw6s5pr8fy8nLz4YcfmqKiIlNRUWF++uknk5aWZrKyshyuvOPWrl1r8vPzTUVFhSktLTVr1641ISEh5rfffjPGBP8Mm7TVpxvm2JKHz5B1yywf9GCPbpnju+++a44dO2YqKirMyZMnzfTp002/fv3MtWvXjDHumGNbPTo5R1eFtKQWL19++aVvzd27d82yZctM7969Ta9evczs2bPN1atXnSu6k9rrsbKy0mRlZZk+ffqYyMhIM3ToUPPee+8Zj8fjbOGd8Pbbb5vU1FQTERFh+vfvb6ZNm+YLaGOCf4ZN2urTDXNsycMh7ZZZPujBHt0yx7lz55qkpCQTERFhBgwYYObOnWvKy8t9290wx7Z6dHKOfFQlAACWctVr0gAAuAkhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaSCIHDhwQM8//7zi4+PVt29fvfzyy7p48aJv+x9//KGxY8cqKipKEydO1I8//qiQkBDfJ/ZI0tmzZzVz5kxFR0crISFBb775pq5fv+5ANwDaQ0gDQeT27dtavXq1ioqKdPjwYYWGhmr27Nnyer2qra3VrFmzNHr0aJ06dUofffSR1qxZ0+z2N2/e1NSpUzVu3DgVFRXpwIEDqqmp0Zw5cxzqCEBb+IANIIhdv35d/fv315kzZ3TixAlt2LBBV65cUVRUlCRp165dWrRokf766y+NHTtWeXl5+v3333Xw4EHfPq5cuaKUlBSVlZVp2LBhTrUCoAU8kgaCyIULFzRv3jylpaUpNjZWgwcPliRVVlaqrKxMGRkZvoCWpOeee67Z7U+fPq2jR48qOjradxkxYoQkNXvaHIAdwp0uAEDHzZo1S6mpqdq5c6eSk5Pl9XqVnp6ue/fudej2t27d0qxZs/Txxx8/si0pKamrywXwmAhpIEj8888/Kisr086dOzVlyhRJ0okTJ3zbhw8frm+++UYNDQ2KjIyUJBUWFjbbx/jx4/X9999r8ODBCg/n7g/Yjqe7gSDRu3dv9e3bVzt27FB5ebmOHDmi1atX+7bPnz9fXq9Xixcv1vnz53Xw4EF99tlnkqSQkBBJUnZ2tm7cuKF58+apsLBQFy9e1MGDB7Vw4UI1NjY60heA1hHSQJAIDQ3Vnj17VFxcrPT0dK1atUqffvqpb3tsbKz279+vkpISjR07VuvXr9fGjRslyfc6dXJysk6ePKnGxkbNmDFDo0eP1sqVKxUfH6/QUP45AGzD2d2Ai+3evVsLFy6Ux+NRz549nS4HQCfxohTgIl9//bXS0tI0YMAAnT59WmvWrNGcOXMIaCBIEdKAi1RXV2vjxo2qrq5WUlKS3njjDW3atMnpsgD4iae7AQCwFGeKAABgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYKn/BxZuhsyw4Bd3AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x100 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Nous remarquons des valeurs aberrantes parmi les âges des participants\n",
    "# Avec une moyenne de 26 ans, les valeurs au-dessus de 36 ans sont considérés comme des outliers\n",
    "import seaborn as sns\n",
    "fig, ax = plt.subplots(figsize=(6, 1))\n",
    "sns.boxplot(x=data_clean_mv.groupby('iid').max()['age'], ax=ax, whis=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.359116022099446"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean_mv.groupby('iid').max()['age'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "age\n",
       "18.0     1\n",
       "19.0     2\n",
       "20.0     5\n",
       "21.0    22\n",
       "22.0    44\n",
       "23.0    56\n",
       "24.0    56\n",
       "25.0    55\n",
       "26.0    55\n",
       "27.0    68\n",
       "28.0    47\n",
       "29.0    40\n",
       "30.0    36\n",
       "31.0     7\n",
       "32.0    13\n",
       "33.0    12\n",
       "34.0    11\n",
       "35.0     4\n",
       "36.0     4\n",
       "37.0     1\n",
       "38.0     1\n",
       "39.0     1\n",
       "42.0     1\n",
       "55.0     1\n",
       "Name: iid, dtype: int64"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cela concerne 5 participants\n",
    "data_clean_mv.groupby('age')['iid'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8232, 98)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dans le cas de l'étude présente nous considérons ces données comme aberrantes\n",
    "# --> Nous les supprimons\n",
    "data_clean_outliers = data_clean_mv.copy()\n",
    "\n",
    "outliers = data_clean_outliers[data_clean_outliers['age'] > 36].index.tolist()\n",
    "outliers_o = data_clean_outliers[data_clean_outliers['age_o'] > 36].index.tolist()\n",
    "\n",
    "data_clean_outliers.drop(outliers, axis=0, inplace=True)\n",
    "data_clean_outliers.drop(outliers_o, axis=0, inplace=True)\n",
    "\n",
    "data_clean_outliers.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAACMCAYAAACzt+rJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAARPUlEQVR4nO3df2zV1f3H8dftaC/12/YWKZZW29pGKSoIyhTrFBYkrBqRiRGjfhV1g+Bkm+ImIBNEZ3DBLFn2h5nK/DHJzNwA8SfG8kNlZLFI+WHMte06CoHWAettsbQg9/3945veceHe217K5Z57+3wkDen9nM+57zenp6/cez+912NmJgAA4KSMZBcAAACiI6gBAHAYQQ0AgMMIagAAHEZQAwDgMIIaAACHEdQAADiMoAYAwGGD+nNyMBjUvn37lJubK4/Hc6ZqAgAg7ZmZOjo6VFxcrIyM6I+b+xXU+/btU0lJSX+mAABgQNuzZ48uuOCCqMf7FdS5ubmhO8nLy+vPVAAADCjt7e0qKSkJZWk0/Qrqnqe78/LyCGoAAE5Dby8dczEZAAAOI6gBAHAYQQ0AgMP69Ro14IrW1lYFAoFklzHg+Hw+FRYWJrsMIK0R1Eh5ra2t+t977tWxo93JLmXAyczy6vU/vUZYAwlEUCPlBQIBHTvarSMVExUc7Et2OXHJONKm7KaPdaR8goLZ+ckuJy4ZXQHpn5sUCAQIaiCBCGqkjeBgn4L/U5DsMk5LMDs/ZWsHkFhcTAYAgMMIagAAHEZQAwDgMIIaAACHEdQAADiMoAYAwGEENQAADiOoAQBwGEENAIDDCGoAABxGUAMA4DCCGgAAhzkX1F1dXfrqq6/U1dWV7FIAAJCU3GxyLqibm5s1e/ZsNTc3J7sUAAAkJTebnAtqAADwXwQ1AAAOI6gBAHAYQQ0AgMMIagAAHEZQAwDgMIIaAACHEdQAADiMoAYAwGEENQAADiOoAQBwGEENAIDDCGoAABw2KJ7B3d3d6u7uDn3f3t5+xgvqsXv37oTNjfTCz0py8f+PgSCZP+dxBfWyZcu0dOnSRNUS5plnnjkr9wOgf9irQGLFFdQLFy7UvHnzQt+3t7erpKTkjBclSYsWLVJZWVlC5kZ62b17N2GRROxVDATJ/D0TV1B7vV55vd5E1RKmrKxMI0aMOCv3BeD0sVeBxOJiMgAAHEZQAwDgMIIaAACHEdQAADiMoAYAwGEENQAADiOoAQBwGEENAIDDCGoAABxGUAMA4DCCGgAAhxHUAAA4jKAGAMBhBDUAAA5zLqhLS0v1wgsvqLS0NNmlAAAgKbnZFNfnUZ8NgwcP5rNtAQBOSWY2OfeIGgAA/BdBDQCAwwhqAAAcRlADAOAwghoAAIcR1AAAOIygBgDAYQQ1AAAOI6gBAHAYQQ0AgMMIagAAHObce30DpyujK5DsEuKWcaQt7N9Ukor/30AqIqiR8nw+nzKzvNI/NyW7lNOW3fRxsks4LZlZXvl8vmSXAaQ1ghopr7CwUK//6TUFAjzCO9t8Pp8KCwuTXQaQ1ghqpIXCwkICA0Ba4mIyAAAcRlADAOAwghoAAIcR1AAAOKxfF5OZmSSpvb39jBQDAMBA0ZOdPVkaTb+CuqOjQ5JUUlLSn2kAABiwOjo6Yr4fgcd6i/IYgsGg9u3bp9zcXHk8nohj2tvbVVJSoj179igvL+907yol0Gt6Gii9DpQ+JXpNR6nYp5mpo6NDxcXFysiI/kp0vx5RZ2Rk6IILLujT2Ly8vJT5z+svek1PA6XXgdKnRK/pKNX67Ms7+3ExGQAADiOoAQBwWMKD2uv1asmSJfJ6vYm+q6Sj1/Q0UHodKH1K9JqO0rnPfl1MBgAAEounvgEAcBhBDQCAwwhqAAAcRlADAOCwuIP6448/1tSpU1VcXCyPx6M1a9aEHfd4PBG/li9fHnXOJ5988pTxI0eOjLuZM2nZsmW66qqrlJubq/POO08//OEP5ff7w8Z0dXXpoYce0tChQ5WTk6PbbrtNra2tMec1My1evFhFRUXKzs7W5MmTVV9fn8hWYuqtz0OHDumnP/2pKisrlZ2drdLSUv3sZz9TIBCIOe999913yppWV1cnup2Y+rKm3//+90+pe86cOTHndW1Npd57/de//hV1r7755ptR53VxXZ9//nldfvnloTe6qKqq0vvvvx86ng77VIrdZzrtU6n3NU2XfdpnFqf33nvPFi1aZKtWrTJJtnr16rDj+/fvD/v64x//aB6PxxobG6POuWTJErvsssvCzvv3v/8db2ln1A9+8AN7+eWXbdeuXVZXV2c33XSTlZaW2uHDh0Nj5syZYyUlJVZTU2O1tbV2zTXX2LXXXhtz3meffdZ8Pp+tWbPGtm/fbrfccouVl5fbkSNHEt1SRL31uXPnTps+fbqtXbvWGhoarKamxi6++GK77bbbYs47c+ZMq66uDlvTQ4cOnY2WourLmk6cONFmzZoVVncgEIg5r2tratZ7r99+++0pe3Xp0qWWk5NjHR0dUed1cV3Xrl1r7777rn311Vfm9/vt8ccft8zMTNu1a5eZpcc+NYvdZzrtU7Pe1zRd9mlfxR3UYSdHCOqTTZs2zSZNmhRzzJIlS2zMmDH9KSXhvv76a5NkmzZtMjOztrY2y8zMtDfffDM05ssvvzRJtmXLlohzBINBGz58uC1fvjx0W1tbm3m9Xvvzn/+c2Ab66OQ+I/nLX/5iWVlZduzYsahjZs6cadOmTUtAhWdOpF4nTpxoP//5z/s8RyqsqVnf1nXs2LH2wAMPxJwnFdbVzGzIkCH20ksvpe0+7dHTZyTpsk97nNhruu7TaBL6GnVra6veffdd/ehHP+p1bH19vYqLi1VRUaG7775bzc3NiSwtbj1PIZ177rmSpK1bt+rYsWOaPHlyaMzIkSNVWlqqLVu2RJyjqalJLS0tYef4fD6NHz8+6jln28l9RhuTl5enQYNiv1X8xo0bdd5556myslIPPvigDh48eEZr7a9ova5cuVIFBQUaNWqUFi5cqM7OzqhzpMKaSr2v69atW1VXV9enveryuh4/flxvvPGGvvnmG1VVVaXtPj25z0jSZZ9G6zUd92k0/fpQjt68+uqrys3N1fTp02OOGz9+vF555RVVVlZq//79Wrp0qa6//nrt2rVLubm5iSyxT4LBoB5++GF973vf06hRoyRJLS0tysrKUn5+ftjYwsJCtbS0RJyn5/bCwsI+n3M2RerzZAcOHNDTTz+t2bNnx5yrurpa06dPV3l5uRobG/X444/rxhtv1JYtW/Sd73wnEeXHJVqvd911l8rKylRcXKwdO3Zo/vz58vv9WrVqVcR5XF9TqW/rumLFCl1yySW69tprY87l6rru3LlTVVVV6urqUk5OjlavXq1LL71UdXV1abVPo/V5snTYp7F6Tcd9GlN/Ho6rl6e+Kysrbe7cuXHP+5///Mfy8vKiPqVzts2ZM8fKyspsz549odtWrlxpWVlZp4y96qqr7LHHHos4z+bNm02S7du3L+z222+/3WbMmHFmiz4Nkfo8USAQsKuvvtqqq6vt6NGjcc3d2Nhokuyjjz46E6X2W2+99qipqTFJ1tDQEPG462tq1nuvnZ2d5vP57Lnnnot7blfWtbu72+rr6622ttYWLFhgBQUF9sUXX6TdPo3W54nSZZ/2pdce6bBPY0nYU9+ffPKJ/H6/fvzjH8d9bn5+vkaMGKGGhoYEVBafuXPn6p133tGGDRvCPtJz+PDhOnr0qNra2sLGt7a2avjw4RHn6rn95CtOY51ztkTrs0dHR4eqq6uVm5ur1atXKzMzM675KyoqVFBQ4PSaRjJ+/HhJilq3y2sq9a3Xv/71r+rs7NS9994b9/yurGtWVpYuuugijRs3TsuWLdOYMWP0u9/9Lu32abQ+e6TTPu2t1xOl+j7tTcKCesWKFRo3bpzGjBkT97mHDx9WY2OjioqKElBZ35iZ5s6dq9WrV2v9+vUqLy8POz5u3DhlZmaqpqYmdJvf71dzc3PU14zKy8s1fPjwsHPa29v1j3/8I+o5idZbn9L/1zhlyhRlZWVp7dq1Gjx4cNz3s3fvXh08eNDpNY2krq5OkqLW7eKaSvH1umLFCt1yyy0aNmxY3PfjwrpGEgwG1d3dnTb7NJqePqX02afRnNjryVJ1n/ZZvA/BOzo6bNu2bbZt2zaTZL/97W9t27Zttnv37tCYQCBg55xzjj3//PMR55g0aZL9/ve/D33/6KOP2saNG62pqck2b95skydPtoKCAvv666/jLe+MefDBB83n89nGjRvD/gSgs7MzNGbOnDlWWlpq69evt9raWquqqrKqqqqweSorK23VqlWh75999lnLz8+3t956y3bs2GHTpk1L6p8I9NZnIBCw8ePH2+jRo62hoSFszLfffhua58Q+Ozo67Be/+IVt2bLFmpqa7KOPPrIrr7zSLr74Yuvq6kpKn2a999rQ0GBPPfWU1dbWWlNTk7311ltWUVFhEyZMCJvH9TU169vPr5lZfX29eTwee//99yPOkwrrumDBAtu0aZM1NTXZjh07bMGCBebxeOzDDz80s/TYp2ax+0ynfWoWu9d02qd9FXdQb9iwwSSd8jVz5szQmD/84Q+WnZ1tbW1tEecoKyuzJUuWhL6/4447rKioyLKysuz888+3O+64I+prDWdLpB4l2csvvxwac+TIEfvJT35iQ4YMsXPOOcduvfVW279//ynznHhOMBi0J554wgoLC83r9doNN9xgfr//LHV1qt76jLbekqypqSlsnp5zOjs7bcqUKTZs2DDLzMy0srIymzVrlrW0tJz9Bk/QW6/Nzc02YcIEO/fcc83r9dpFF11kv/zlL0/5+0zX19Ssbz+/ZmYLFy60kpISO378eNR5XF/XBx54wMrKyiwrK8uGDRtmN9xwQyikzdJjn5rF7jOd9qlZ7F7TaZ/2FR9zCQCAw3ivbwAAHEZQAwDgMIIaAACHEdQAADiMoAYAwGEENQAADiOoAQBwGEENAIDDCGoAABxGUAMA4DCCGkgxH3zwga677jrl5+dr6NChuvnmm9XY2Bg6/ve//11jx47V4MGD9d3vfldr1qyRx+MJfcKQJO3atUs33nijcnJyVFhYqHvuuUcHDhxIQjcAekNQAynmm2++0bx581RbW6uamhplZGTo1ltvVTAYVHt7u6ZOnarRo0fr888/19NPP6358+eHnd/W1qZJkybpiiuuUG1trT744AO1trZqxowZSeoIQCx8KAeQ4g4cOKBhw4Zp586d+vTTT/WrX/1Ke/fuDX0e8UsvvaRZs2Zp27ZtGjt2rH7961/rk08+0bp160Jz7N27VyUlJfL7/RoxYkSyWgEQAY+ogRRTX1+vO++8UxUVFcrLy9OFF14oSWpubpbf79fll18eCmlJuvrqq8PO3759uzZs2KCcnJzQ18iRIyUp7Cl0AG4YlOwCAMRn6tSpKisr04svvqji4mIFg0GNGjVKR48e7dP5hw8f1tSpU/Wb3/zmlGNFRUVnulwA/URQAynk4MGD8vv9evHFF3X99ddLkj799NPQ8crKSr3++uvq7u6W1+uVJH322Wdhc1x55ZX629/+pgsvvFCDBvErAHAdT30DKWTIkCEaOnSoXnjhBTU0NGj9+vWaN29e6Phdd92lYDCo2bNn68svv9S6dev03HPPSZI8Ho8k6aGHHtKhQ4d055136rPPPlNjY6PWrVun+++/X8ePH09KXwCiI6iBFJKRkaE33nhDW7du1ahRo/TII49o+fLloeN5eXl6++23VVdXp7Fjx2rRokVavHixJIVety4uLtbmzZt1/PhxTZkyRaNHj9bDDz+s/Px8ZWTwKwFwDVd9A2lu5cqVuv/++xUIBJSdnZ3scgDEiReogDTz2muvqaKiQueff762b9+u+fPna8aMGYQ0kKIIaiDNtLS0aPHixWppaVFRUZFuv/12PfPMM8kuC8Bp4qlvAAAcxpUjAAA4jKAGAMBhBDUAAA4jqAEAcBhBDQCAwwhqAAAcRlADAOAwghoAAIcR1AAAOOz/ABgJBYHBGoqzAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x100 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualisation avec Seaborn\n",
    "fig, ax = plt.subplots(figsize=(6, 1))\n",
    "sns.boxplot(x=data_clean_outliers.groupby('iid').max()['age'], ax=ax, whis=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalisation des notes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deux échelles de notation ont été utilisées pour évaluer les préférences personnelles (attr, sinc, etc.) : \n",
    "- Allocation de 100 points\n",
    "- Note de 1 à 10.   \n",
    "\n",
    "Afin de pouvoir être comparées, nous devons mettre les préférences sur la même échelle.\n",
    "\n",
    "Tout d'abord, recherchons quelles échelles ont été utilisées pour chaque préférence.\n",
    "\n",
    "Pour ce faire, nous calculons les totaux de chaque groupe de préférences : \n",
    "- Les totaux max de 100 points suggèrent l'utilisation d'une allocation de 100 points\n",
    "- Les totaux max ne dépassant pas 50 points suggèrent l'utilisation d'une l'échelle de 1 à 10 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calcul des totaux des préférences\n",
    "data_clean_scores = data_clean_outliers.copy()\n",
    "\n",
    "data_clean_scores['sum_pf_o'] = data_clean_scores[['pf_o_att', 'pf_o_sin', 'pf_o_int', 'pf_o_fun', 'pf_o_amb', 'pf_o_sha']].sum(axis=1)\n",
    "data_clean_scores['sum1_1'] = data_clean_scores[['attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1', 'shar1_1']].sum(axis=1)\n",
    "data_clean_scores['sum2_1'] = data_clean_scores[['attr2_1', 'sinc2_1', 'intel2_1', 'fun2_1', 'amb2_1', 'shar2_1']].sum(axis=1)\n",
    "data_clean_scores['sum1_2'] = data_clean_scores[['attr1_2', 'sinc1_2', 'intel1_2', 'fun1_2', 'amb1_2', 'shar1_2']].sum(axis=1)\n",
    "data_clean_scores['sum3_1'] = data_clean_scores[['attr3_1', 'sinc3_1', 'intel3_1', 'fun3_1', 'amb3_1']].sum(axis=1)\n",
    "data_clean_scores['sum3_2'] = data_clean_scores[['attr3_2', 'sinc3_2', 'intel3_2', 'fun3_2', 'amb3_2']].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sum_pf_o</th>\n",
       "      <th>sum1_1</th>\n",
       "      <th>sum2_1</th>\n",
       "      <th>sum1_2</th>\n",
       "      <th>sum3_1</th>\n",
       "      <th>sum3_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>99.107513</td>\n",
       "      <td>99.107513</td>\n",
       "      <td>99.101376</td>\n",
       "      <td>89.407640</td>\n",
       "      <td>38.600097</td>\n",
       "      <td>34.275753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.955209</td>\n",
       "      <td>9.955209</td>\n",
       "      <td>9.862545</td>\n",
       "      <td>31.125713</td>\n",
       "      <td>6.392096</td>\n",
       "      <td>12.797556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>34.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>38.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>148.000000</td>\n",
       "      <td>148.000000</td>\n",
       "      <td>112.000000</td>\n",
       "      <td>120.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          sum_pf_o       sum1_1       sum2_1       sum1_2       sum3_1  \\\n",
       "count  8232.000000  8232.000000  8232.000000  8232.000000  8232.000000   \n",
       "mean     99.107513    99.107513    99.101376    89.407640    38.600097   \n",
       "std       9.955209     9.955209     9.862545    31.125713     6.392096   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%     100.000000   100.000000   100.000000   100.000000    36.000000   \n",
       "50%     100.000000   100.000000   100.000000   100.000000    39.000000   \n",
       "75%     100.000000   100.000000   100.000000   100.000000    42.000000   \n",
       "max     148.000000   148.000000   112.000000   120.000000    50.000000   \n",
       "\n",
       "            sum3_2  \n",
       "count  8232.000000  \n",
       "mean     34.275753  \n",
       "std      12.797556  \n",
       "min       0.000000  \n",
       "25%      34.000000  \n",
       "50%      38.000000  \n",
       "75%      41.000000  \n",
       "max      50.000000  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean_scores.describe(include='all').iloc[:, np.r_[98:104]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Résultats : \n",
    "\n",
    "- *attr1_1*, *attr_2_1*, *attr1_2* sont notées par allocation de 100 points :  \n",
    "il s'agit des préférences du participant pour les différentes qualités, cette échelle de notation est pertinente et nous la conservons\n",
    "\n",
    "- *attr3_1*, *attr3_2* sont notées sur une échelle de 1 à 10 :  \n",
    "il s'agit de l'évaluation du participant sur sa propre personne, cette échelle de notation est pertinente et nous la conservons\n",
    "\n",
    "Toutefois nous remarquons des erreurs puisque certains totaux dépassent largement les 100 points.  \n",
    "\n",
    "Nous corrigeons ces erreurs :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sum1_1\n",
       "148.00      10\n",
       "120.00      25\n",
       "110.00      22\n",
       "101.00      10\n",
       "100.02      90\n",
       "100.02      54\n",
       "100.01     237\n",
       "100.01     126\n",
       "100.00     107\n",
       "100.00    7031\n",
       "100.00      35\n",
       "99.99      197\n",
       "99.99       81\n",
       "99.99       16\n",
       "99.98       26\n",
       "95.00       22\n",
       "90.00       65\n",
       "0.00        78\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Vérification des totaux dans le groupe de préférence 1 (attr1_1, sinc1_1, etc.)\n",
    "data_clean_scores['sum1_1'].value_counts().sort_index(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>sum1_1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>20.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>148.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>21.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>101.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     attr1_1  sinc1_1  intel1_1  fun1_1  amb1_1  shar1_1  sum1_1\n",
       "iid                                                             \n",
       "66      20.0     25.0      15.0    15.0    53.0     20.0   148.0\n",
       "68      21.0     17.0      22.0    20.0     8.0     13.0   101.0"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ici il s'agit d'erreurs de saisie (53 à la place de 5, 21 à la place de 20)\n",
    "# --> Nous corrigeons ces erreurs de saisie\n",
    "data_clean_scores[(data_clean_scores['sum1_1'] == 148) | (data_clean_scores['sum1_1'] == 101)].iloc[:, np.r_[0:1,56:62,99:100]].groupby('iid').max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>sum1_1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>25.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>20.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>30.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427</th>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>20.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>20.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>110.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>95.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     attr1_1  sinc1_1  intel1_1  fun1_1  amb1_1  shar1_1  sum1_1\n",
       "iid                                                             \n",
       "44      25.0     15.0      25.0    25.0    15.0     15.0   120.0\n",
       "67      20.0     15.0      20.0    20.0     5.0     10.0    90.0\n",
       "105     30.0     15.0      20.0    20.0     0.0      5.0    90.0\n",
       "427     20.0     20.0      20.0    20.0    20.0     20.0   120.0\n",
       "489     20.0     10.0      20.0    20.0    20.0      0.0    90.0\n",
       "517     15.0     20.0      20.0    20.0     5.0     10.0    90.0\n",
       "518     20.0     25.0      25.0    30.0     5.0      5.0   110.0\n",
       "526     10.0     10.0      30.0    20.0    10.0     15.0    95.0"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ici il s'agit d'une erreur dans le total des points distribués\n",
    "# --> Nous remettons les notes sur une échelle de 100 points\n",
    "data_clean_scores[(data_clean_scores['sum1_1'] == 120) | (data_clean_scores['sum1_1'] == 110) | (data_clean_scores['sum1_1'] == 95) | (data_clean_scores['sum1_1'] == 90)].iloc[:, np.r_[0:1,56:62,99:100]].groupby('iid').max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous effectuons les mêmes vérifications pour les autres groupes de préférences et nous appliquons toutes les corrections nécessaires :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application des corrections pour tous les groupes de préférences\n",
    "\n",
    "# ------- Correction des erreurs de saisie -------\n",
    "\n",
    "pref_scores_to_correct_1 = data_clean_scores[data_clean_scores['sum1_1'] == 148].index.tolist()\n",
    "for i in pref_scores_to_correct_1:\n",
    "    data_clean_scores.loc[i, 'amb1_1'] = 5.0\n",
    "\n",
    "pref_scores_to_correct_2 = data_clean_scores[data_clean_scores['sum1_1'] == 101].index.tolist()\n",
    "for i in pref_scores_to_correct_2:\n",
    "    data_clean_scores.loc[i, 'attr1_1'] = 20.0\n",
    "\n",
    "pref_scores_to_correct_3 = data_clean_scores[data_clean_scores['sum2_1'] == 101].index.tolist()\n",
    "for i in pref_scores_to_correct_3:\n",
    "    data_clean_scores.loc[i, 'sinc2_1'] = 20.0\n",
    "\n",
    "pref_scores_to_correct_4 = data_clean_scores[data_clean_scores['sum_pf_o'] == 148].index.tolist()\n",
    "for i in pref_scores_to_correct_4:\n",
    "    data_clean_scores.loc[i, 'pf_o_amb'] = 5.0\n",
    "\n",
    "pref_scores_to_correct_5 = data_clean_scores[data_clean_scores['sum_pf_o'] == 101].index.tolist()\n",
    "for i in pref_scores_to_correct_5:\n",
    "    data_clean_scores.loc[i, 'pf_o_att'] = 20.0\n",
    "\n",
    "# ------- Remise à l'échelle sur 100 points --------\n",
    "    \n",
    "def rescale_pref_scores(data, pref_columns, sum_column, divisor):\n",
    "    ''' This function rescales the scores on 100 points '''\n",
    "    for pref in pref_columns:\n",
    "        pref_scores_to_correct = data[data[sum_column] == divisor].index.tolist()\n",
    "        for i in pref_scores_to_correct:\n",
    "            data.loc[i, pref] = data.loc[i, pref] * 100 / divisor\n",
    "\n",
    "pref1_1 = [\"attr1_1\", \"sinc1_1\", \"intel1_1\", \"fun1_1\", \"amb1_1\", \"shar1_1\"]\n",
    "pref2_1 = [\"attr2_1\", \"sinc2_1\", \"intel2_1\", \"fun2_1\", \"amb2_1\", \"shar2_1\"]\n",
    "pref1_2 = [\"attr1_2\", \"sinc1_2\", \"intel1_2\", \"fun1_2\", \"amb1_2\", \"shar1_2\"]\n",
    "pref_o = [\"pf_o_att\", \"pf_o_sin\", \"pf_o_int\", \"pf_o_fun\", \"pf_o_amb\", \"pf_o_sha\"]\n",
    "\n",
    "rescale_pref_scores(data_clean_scores, pref1_1, 'sum1_1', 120)\n",
    "rescale_pref_scores(data_clean_scores, pref1_1, 'sum1_1', 110)\n",
    "rescale_pref_scores(data_clean_scores, pref1_1, 'sum1_1', 95)\n",
    "rescale_pref_scores(data_clean_scores, pref1_1, 'sum1_1', 90)\n",
    "\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 112)\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 110)\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 105)\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 95)\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 90)\n",
    "rescale_pref_scores(data_clean_scores, pref2_1, 'sum2_1', 80)\n",
    "\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 120)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 115)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 110)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 105)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 103)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 102)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 99)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 95)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 90)\n",
    "rescale_pref_scores(data_clean_scores, pref1_2, 'sum1_2', 80)\n",
    "\n",
    "rescale_pref_scores(data_clean_scores, pref_o, 'sum_pf_o', 120)\n",
    "rescale_pref_scores(data_clean_scores, pref_o, 'sum_pf_o', 110)\n",
    "rescale_pref_scores(data_clean_scores, pref_o, 'sum_pf_o', 95)\n",
    "rescale_pref_scores(data_clean_scores, pref_o, 'sum_pf_o', 90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_drop = ['sum_pf_o','sum1_1','sum2_1','sum3_1','sum1_2','sum3_2']\n",
    "data_clean_scores.drop(to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nous calculons les nouveaux totaux après correction\n",
    "data_clean_scores['sum_pf_o'] = data_clean_scores[['pf_o_att', 'pf_o_sin', 'pf_o_int', 'pf_o_fun', 'pf_o_amb', 'pf_o_sha']].sum(axis=1)\n",
    "data_clean_scores['sum1_1'] = data_clean_scores[['attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1', 'shar1_1']].sum(axis=1)\n",
    "data_clean_scores['sum2_1'] = data_clean_scores[['attr2_1', 'sinc2_1', 'intel2_1', 'fun2_1', 'amb2_1', 'shar2_1']].sum(axis=1)\n",
    "data_clean_scores['sum1_2'] = data_clean_scores[['attr1_2', 'sinc1_2', 'intel1_2', 'fun1_2', 'amb1_2', 'shar1_2']].sum(axis=1)\n",
    "data_clean_scores['sum3_1'] = data_clean_scores[['attr3_1', 'sinc3_1', 'intel3_1', 'fun3_1', 'amb3_1']].sum(axis=1)\n",
    "data_clean_scores['sum3_2'] = data_clean_scores[['attr3_2', 'sinc3_2', 'intel3_2', 'fun3_2', 'amb3_2']].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sum_pf_o</th>\n",
       "      <th>sum1_1</th>\n",
       "      <th>sum2_1</th>\n",
       "      <th>sum1_2</th>\n",
       "      <th>sum3_1</th>\n",
       "      <th>sum3_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "      <td>8232.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>99.052849</td>\n",
       "      <td>99.052849</td>\n",
       "      <td>99.053150</td>\n",
       "      <td>89.261867</td>\n",
       "      <td>38.600097</td>\n",
       "      <td>34.275753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.688473</td>\n",
       "      <td>9.688473</td>\n",
       "      <td>9.688503</td>\n",
       "      <td>30.962350</td>\n",
       "      <td>6.392096</td>\n",
       "      <td>12.797556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>34.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>38.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>100.020000</td>\n",
       "      <td>100.020000</td>\n",
       "      <td>100.030000</td>\n",
       "      <td>100.020000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          sum_pf_o       sum1_1       sum2_1       sum1_2       sum3_1  \\\n",
       "count  8232.000000  8232.000000  8232.000000  8232.000000  8232.000000   \n",
       "mean     99.052849    99.052849    99.053150    89.261867    38.600097   \n",
       "std       9.688473     9.688473     9.688503    30.962350     6.392096   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%     100.000000   100.000000   100.000000   100.000000    36.000000   \n",
       "50%     100.000000   100.000000   100.000000   100.000000    39.000000   \n",
       "75%     100.000000   100.000000   100.000000   100.000000    42.000000   \n",
       "max     100.020000   100.020000   100.030000   100.020000    50.000000   \n",
       "\n",
       "            sum3_2  \n",
       "count  8232.000000  \n",
       "mean     34.275753  \n",
       "std      12.797556  \n",
       "min       0.000000  \n",
       "25%      34.000000  \n",
       "50%      38.000000  \n",
       "75%      41.000000  \n",
       "max      50.000000  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean_scores.describe(include='all').iloc[:, np.r_[98:104]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8232, 98)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_drop = ['sum_pf_o','sum1_1','sum2_1','sum3_1','sum1_2','sum3_2']\n",
    "data_clean_final = data_clean_scores.copy()\n",
    "data_clean_final.drop(to_drop, axis=1, inplace=True)\n",
    "data_clean_final.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correction des échelles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous remarquons que les échelles de notation pour les champs *date* et *go_out* sont inversées :  la plus petite valeur pour le plus actif, la plus grande valeur pour le moins actif. \n",
    "\n",
    "Les corriger facilitera la recherche de corrélations dans la partie 3 de ce projet.  \n",
    "\n",
    "Dans la même optique nous pouvons améliorer l'échelle de notation du champ *goal*, en classant les objectifs du moins déterminé au plus déterminé."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_correct = ['date', 'go_out']\n",
    "for col in to_correct:\n",
    "    data_clean_final[col] = data_clean_final[col].apply( \n",
    "                                lambda x : 7.0 if x == 1.0\n",
    "                                else 6.0 if x == 2.0\n",
    "                                else 5.0 if x == 3.0\n",
    "                                else 3.0 if x == 5.0\n",
    "                                else 2.0 if x == 6.0\n",
    "                                else 1.0 if x == 7.0\n",
    "                                else x\n",
    "                                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clean_final['goal'] = data_clean_final['goal'].apply( \n",
    "                            lambda x : 1.0 if x == 6.0\n",
    "                            else 2.0 if x == 5.0\n",
    "                            else 6.0 if x == 4.0\n",
    "                            else 5.0 if x == 3.0\n",
    "                            else 4.0 if x == 2.0\n",
    "                            else 3.0 if x == 1.0\n",
    "                            else x\n",
    "                            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Observations avec données manquantes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suppression de lignes\n",
    "\n",
    "- Une ligne est une observation. Chaque ligne représente une fiche remplie par un participant à l'issue du speed dating\n",
    "\n",
    "- Bien que nous ayons retiré les colonnes ayant une part élevée de données manquantes, certaines lignes comportent encore beaucoup de champs vides. Cette proportion atteint 79% dans le pire des cas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>843</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>842</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>838</th>\n",
       "      <td>78.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324</th>\n",
       "      <td>66.326531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1857</th>\n",
       "      <td>62.244898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>841</th>\n",
       "      <td>62.244898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>844</th>\n",
       "      <td>61.224490</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0\n",
       "845   79.591837\n",
       "847   79.591837\n",
       "843   79.591837\n",
       "842   79.591837\n",
       "839   79.591837\n",
       "838   78.571429\n",
       "324   66.326531\n",
       "1857  62.244898\n",
       "841   62.244898\n",
       "844   61.224490"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pourcentage de valeurs manquantes par lignes\n",
    "missing_values_in_rows = (data_clean_final.isna().sum(axis=1)/data_clean_final.shape[1]*100).to_frame()\n",
    "missing_values_in_rows.sort_values(by=0,ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous dénombrons 140 lignes avec plus de 20% de données manquantes, soit 1.70% des données. \n",
    "\n",
    "Il s'agit d'une part relativement faible du jeu de données total.  \n",
    "\n",
    "--> Nous supprimons ces lignes afin de ne garder uniquement des lignes avec moins de 20% de champs vides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>843</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>842</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>79.591837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6387</th>\n",
       "      <td>20.408163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2906</th>\n",
       "      <td>20.408163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6388</th>\n",
       "      <td>20.408163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2915</th>\n",
       "      <td>20.408163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6393</th>\n",
       "      <td>20.408163</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0\n",
       "845   79.591837\n",
       "843   79.591837\n",
       "842   79.591837\n",
       "839   79.591837\n",
       "847   79.591837\n",
       "...         ...\n",
       "6387  20.408163\n",
       "2906  20.408163\n",
       "6388  20.408163\n",
       "2915  20.408163\n",
       "6393  20.408163\n",
       "\n",
       "[140 rows x 1 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows_to_remove = missing_values_in_rows[missing_values_in_rows[0] > 20]\n",
    "rows_to_remove.sort_values(by=0,ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous devons supprimer les lignes complémentaires afin de ne pas avoir de décalage pour l'analyse en partie 2 de cette étude."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nous recherchons les lignes complémentaires à supprimer\n",
    "data_clean_final = data_clean_final.reset_index()\n",
    "to_remove = rows_to_remove.index.values.tolist()\n",
    "to_remove_df = data_clean_final.iloc[to_remove,:]\n",
    "iid_list = to_remove_df['iid'].values.tolist()\n",
    "pid_list = to_remove_df['pid'].values.tolist()\n",
    "\n",
    "index_list = []\n",
    "for iid, pid in zip(iid_list, pid_list):\n",
    "    index_list.append(data_clean_final[(data_clean_final['iid']==pid) & (data_clean_final['pid']==iid)].index[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7964, 98)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_to_remove = rows_to_remove.index.values.tolist()\n",
    "index_to_remove += index_list\n",
    "data_clean_final = data_clean_final.drop(index_to_remove, axis=0)\n",
    "data_clean_final.reset_index(drop=True)\n",
    "data_clean_final.drop(['index'], axis=1, inplace=True)\n",
    "data_clean_final.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputation des données manquantes\n",
    "\n",
    "- Les scores de préférence sont considérés comme des données essentielles dans l'étude présente  \n",
    "-> nous les conservons comme tels sans imputation\n",
    "\n",
    "- Nous appliquons une imputation sur les autres données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "match_es    13.483965\n",
       "shar        12.159864\n",
       "shar_o      12.159864\n",
       "numdat_2    10.544218\n",
       "attr1_2     10.386297\n",
       "intel1_2    10.179786\n",
       "satis_2     10.179786\n",
       "length      10.179786\n",
       "sinc1_2     10.179786\n",
       "amb3_2      10.179786\n",
       "fun1_2      10.179786\n",
       "amb1_2      10.179786\n",
       "shar1_2     10.179786\n",
       "attr3_2     10.179786\n",
       "fun3_2      10.179786\n",
       "sinc3_2     10.179786\n",
       "intel3_2    10.179786\n",
       "amb_o        8.017493\n",
       "amb          8.017493\n",
       "met          4.142371\n",
       "dtype: float64"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Nous identifions les colonnes avec le plus de valeurs manquantes\n",
    "(data_clean_final.isna().sum()/data_clean_scores.shape[0]*100).sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['int_corr', 'age_o', 'race_o', 'pf_o_att', 'pf_o_sin', 'pf_o_int',\n",
       "       'pf_o_fun', 'pf_o_amb', 'pf_o_sha', 'attr_o', 'sinc_o', 'intel_o',\n",
       "       'fun_o', 'amb_o', 'shar_o', 'like_o', 'prob_o', 'met_o', 'age',\n",
       "       'field_cd', 'race', 'imprace', 'imprelig', 'goal', 'date', 'go_out',\n",
       "       'career_c', 'sports', 'tvsports', 'exercise', 'dining', 'museums',\n",
       "       'art', 'hiking', 'gaming', 'clubbing', 'reading', 'tv', 'theater',\n",
       "       'movies', 'concerts', 'music', 'shopping', 'yoga', 'exphappy',\n",
       "       'attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1', 'shar1_1',\n",
       "       'attr2_1', 'sinc2_1', 'intel2_1', 'fun2_1', 'amb2_1', 'shar2_1',\n",
       "       'attr3_1', 'sinc3_1', 'fun3_1', 'intel3_1', 'amb3_1', 'attr', 'sinc',\n",
       "       'intel', 'fun', 'amb', 'shar', 'like', 'prob', 'met', 'match_es',\n",
       "       'satis_2', 'length', 'numdat_2', 'attr1_2', 'sinc1_2', 'intel1_2',\n",
       "       'fun1_2', 'amb1_2', 'shar1_2', 'attr3_2', 'sinc3_2', 'intel3_2',\n",
       "       'fun3_2', 'amb3_2'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Affichage de la liste totale des colonnes concernées\n",
    "missing_values_df = (data_clean_final.isna().sum()/data_clean_final.shape[0]*100).to_frame()\n",
    "missing_values_df[missing_values_df[0] > 0].index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nous définissons la méthode d'imputation des données manquantes :\n",
    "\n",
    "- Le mode pour les données catégorielles\n",
    "\n",
    "- La moyenne pour les données quantitatives continues\n",
    "\n",
    "- La médiane pour les données quantitatives discrètes\n",
    "\n",
    "- Pour certaines données, nous assumons qu'il s'agit d'une réponse négative ou nulle dès lors qu'elles ne sont pas renseignées :  \n",
    "like, prob, match_es, satis_2, met"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ajout du champ match_es_r pour calculer le nombre de match expéré en valeur relative\n",
    "data_clean_final['match_es_r'] = np.minimum(data_clean_final['match_es']/data['round']*100, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choix des méthodes d'imputation pour chaque champ\n",
    "\n",
    "by_mode = ['race', 'race_o', 'field_cd', 'date', 'career_c', 'length', 'numdat_2']\n",
    "by_mean = ['int_corr', 'match_es_r']\n",
    "by_mean_round = ['age_o', 'age']\n",
    "by_median = ['imprace', 'imprelig', 'exphappy', 'satis_2',\n",
    "              'sports', 'tvsports', 'exercise', 'dining','museums', 'art', 'hiking', 'gaming', \n",
    "              'clubbing', 'reading', 'tv', 'theater', 'movies', 'concerts', 'music', 'shopping', 'yoga']\n",
    "by_zero = ['like_o', 'prob_o', 'like', 'prob']\n",
    "by_two = ['met_o', 'met']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application des imputations\n",
    "data_clean = data_clean_final.copy()\n",
    "\n",
    "for col in by_mode:\n",
    "    mode_val = data_clean.groupby('iid').max()[col].mode()[0]\n",
    "    data_clean[col] = data_clean[col].fillna(mode_val)\n",
    "\n",
    "for col in by_mean:\n",
    "    mean_val = data_clean[col].mean()\n",
    "    data_clean[col] = data_clean[col].fillna(mean_val)\n",
    "\n",
    "for col in by_mean_round:\n",
    "    mean_val = data_clean.groupby('iid').max()[col].mean()\n",
    "    data_clean[col] = data_clean[col].fillna(round(mean_val))\n",
    "\n",
    "for col in by_median:\n",
    "    median_val = data_clean.groupby('iid').max()[col].median()\n",
    "    data_clean[col] = data_clean[col].fillna(median_val)\n",
    "\n",
    "for col in by_zero:\n",
    "    data_clean[col] = data_clean[col].fillna(0)\n",
    "\n",
    "for col in by_two:\n",
    "    data_clean[col] = data_clean[col].fillna(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1) Columns without missing values after imputation: 50 \n",
      "\n",
      "Index(['iid', 'gender', 'condtn', 'wave', 'round', 'position', 'order', 'pid',\n",
      "       'match', 'int_corr', 'samerace', 'age_o', 'race_o', 'dec_o', 'like_o',\n",
      "       'prob_o', 'met_o', 'age', 'field_cd', 'race', 'imprace', 'imprelig',\n",
      "       'date', 'career_c', 'sports', 'tvsports', 'exercise', 'dining',\n",
      "       'museums', 'art', 'hiking', 'gaming', 'clubbing', 'reading', 'tv',\n",
      "       'theater', 'movies', 'concerts', 'music', 'shopping', 'yoga',\n",
      "       'exphappy', 'dec', 'like', 'prob', 'met', 'satis_2', 'length',\n",
      "       'numdat_2', 'match_es_r'],\n",
      "      dtype='object')\n",
      "\n",
      "2) Columns that still have missing values (preferences scores): 49 \n",
      "\n",
      "Index(['pf_o_att', 'pf_o_sin', 'pf_o_int', 'pf_o_fun', 'pf_o_amb', 'pf_o_sha',\n",
      "       'attr_o', 'sinc_o', 'intel_o', 'fun_o', 'amb_o', 'shar_o', 'goal',\n",
      "       'go_out', 'attr1_1', 'sinc1_1', 'intel1_1', 'fun1_1', 'amb1_1',\n",
      "       'shar1_1', 'attr2_1', 'sinc2_1', 'intel2_1', 'fun2_1', 'amb2_1',\n",
      "       'shar2_1', 'attr3_1', 'sinc3_1', 'fun3_1', 'intel3_1', 'amb3_1', 'attr',\n",
      "       'sinc', 'intel', 'fun', 'amb', 'shar', 'match_es', 'attr1_2', 'sinc1_2',\n",
      "       'intel1_2', 'fun1_2', 'amb1_2', 'shar1_2', 'attr3_2', 'sinc3_2',\n",
      "       'intel3_2', 'fun3_2', 'amb3_2'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Vérification après imputation\n",
    "missing_values_df = (data_clean.isna().sum()/data_clean_scores.shape[0]*100).to_frame()\n",
    "columns_without_mv = missing_values_df[missing_values_df[0] == 0].index\n",
    "columns_with_mv = missing_values_df[missing_values_df[0] > 0].index\n",
    "\n",
    "print(f'1) Columns without missing values after imputation: {len(columns_without_mv)} \\n')\n",
    "print(columns_without_mv)\n",
    "print(f'\\n2) Columns that still have missing values (preferences scores): {len(columns_with_mv)} \\n')\n",
    "print(columns_with_mv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iid</th>\n",
       "      <th>gender</th>\n",
       "      <th>condtn</th>\n",
       "      <th>wave</th>\n",
       "      <th>round</th>\n",
       "      <th>position</th>\n",
       "      <th>order</th>\n",
       "      <th>pid</th>\n",
       "      <th>match</th>\n",
       "      <th>int_corr</th>\n",
       "      <th>samerace</th>\n",
       "      <th>age_o</th>\n",
       "      <th>race_o</th>\n",
       "      <th>pf_o_att</th>\n",
       "      <th>pf_o_sin</th>\n",
       "      <th>pf_o_int</th>\n",
       "      <th>pf_o_fun</th>\n",
       "      <th>pf_o_amb</th>\n",
       "      <th>pf_o_sha</th>\n",
       "      <th>dec_o</th>\n",
       "      <th>attr_o</th>\n",
       "      <th>sinc_o</th>\n",
       "      <th>intel_o</th>\n",
       "      <th>fun_o</th>\n",
       "      <th>amb_o</th>\n",
       "      <th>shar_o</th>\n",
       "      <th>like_o</th>\n",
       "      <th>prob_o</th>\n",
       "      <th>met_o</th>\n",
       "      <th>age</th>\n",
       "      <th>field_cd</th>\n",
       "      <th>race</th>\n",
       "      <th>imprace</th>\n",
       "      <th>imprelig</th>\n",
       "      <th>goal</th>\n",
       "      <th>date</th>\n",
       "      <th>go_out</th>\n",
       "      <th>career_c</th>\n",
       "      <th>sports</th>\n",
       "      <th>tvsports</th>\n",
       "      <th>exercise</th>\n",
       "      <th>dining</th>\n",
       "      <th>museums</th>\n",
       "      <th>art</th>\n",
       "      <th>hiking</th>\n",
       "      <th>gaming</th>\n",
       "      <th>clubbing</th>\n",
       "      <th>reading</th>\n",
       "      <th>tv</th>\n",
       "      <th>theater</th>\n",
       "      <th>movies</th>\n",
       "      <th>concerts</th>\n",
       "      <th>music</th>\n",
       "      <th>shopping</th>\n",
       "      <th>yoga</th>\n",
       "      <th>exphappy</th>\n",
       "      <th>attr1_1</th>\n",
       "      <th>sinc1_1</th>\n",
       "      <th>intel1_1</th>\n",
       "      <th>fun1_1</th>\n",
       "      <th>amb1_1</th>\n",
       "      <th>shar1_1</th>\n",
       "      <th>attr2_1</th>\n",
       "      <th>sinc2_1</th>\n",
       "      <th>intel2_1</th>\n",
       "      <th>fun2_1</th>\n",
       "      <th>amb2_1</th>\n",
       "      <th>shar2_1</th>\n",
       "      <th>attr3_1</th>\n",
       "      <th>sinc3_1</th>\n",
       "      <th>fun3_1</th>\n",
       "      <th>intel3_1</th>\n",
       "      <th>amb3_1</th>\n",
       "      <th>dec</th>\n",
       "      <th>attr</th>\n",
       "      <th>sinc</th>\n",
       "      <th>intel</th>\n",
       "      <th>fun</th>\n",
       "      <th>amb</th>\n",
       "      <th>shar</th>\n",
       "      <th>like</th>\n",
       "      <th>prob</th>\n",
       "      <th>met</th>\n",
       "      <th>match_es</th>\n",
       "      <th>satis_2</th>\n",
       "      <th>length</th>\n",
       "      <th>numdat_2</th>\n",
       "      <th>attr1_2</th>\n",
       "      <th>sinc1_2</th>\n",
       "      <th>intel1_2</th>\n",
       "      <th>fun1_2</th>\n",
       "      <th>amb1_2</th>\n",
       "      <th>shar1_2</th>\n",
       "      <th>attr3_2</th>\n",
       "      <th>sinc3_2</th>\n",
       "      <th>intel3_2</th>\n",
       "      <th>fun3_2</th>\n",
       "      <th>amb3_2</th>\n",
       "      <th>match_es_r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7915.000000</td>\n",
       "      <td>7906.000000</td>\n",
       "      <td>7884.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7785.000000</td>\n",
       "      <td>7713.000000</td>\n",
       "      <td>7700.000000</td>\n",
       "      <td>7659.000000</td>\n",
       "      <td>7304.00000</td>\n",
       "      <td>6963.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7915.000000</td>\n",
       "      <td>7906.000000</td>\n",
       "      <td>7884.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7924.000000</td>\n",
       "      <td>7915.000000</td>\n",
       "      <td>7915.000000</td>\n",
       "      <td>7899.000000</td>\n",
       "      <td>7899.000000</td>\n",
       "      <td>7899.000000</td>\n",
       "      <td>7899.000000</td>\n",
       "      <td>7899.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7785.000000</td>\n",
       "      <td>7713.000000</td>\n",
       "      <td>7700.000000</td>\n",
       "      <td>7659.000000</td>\n",
       "      <td>7304.00000</td>\n",
       "      <td>6963.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>6854.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "      <td>7109.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.00000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7126.000000</td>\n",
       "      <td>7964.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>284.513310</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.831743</td>\n",
       "      <td>11.387996</td>\n",
       "      <td>16.895530</td>\n",
       "      <td>9.073832</td>\n",
       "      <td>8.943621</td>\n",
       "      <td>284.513310</td>\n",
       "      <td>0.165244</td>\n",
       "      <td>0.194619</td>\n",
       "      <td>0.395028</td>\n",
       "      <td>26.270844</td>\n",
       "      <td>2.780889</td>\n",
       "      <td>22.443131</td>\n",
       "      <td>17.419560</td>\n",
       "      <td>20.269719</td>\n",
       "      <td>17.424695</td>\n",
       "      <td>10.663709</td>\n",
       "      <td>11.883570</td>\n",
       "      <td>0.418508</td>\n",
       "      <td>6.175389</td>\n",
       "      <td>7.159017</td>\n",
       "      <td>7.360649</td>\n",
       "      <td>6.399465</td>\n",
       "      <td>6.77437</td>\n",
       "      <td>5.464096</td>\n",
       "      <td>5.961728</td>\n",
       "      <td>5.028692</td>\n",
       "      <td>1.956429</td>\n",
       "      <td>26.217353</td>\n",
       "      <td>7.642265</td>\n",
       "      <td>2.760799</td>\n",
       "      <td>3.779633</td>\n",
       "      <td>3.661602</td>\n",
       "      <td>3.451666</td>\n",
       "      <td>2.997489</td>\n",
       "      <td>5.857017</td>\n",
       "      <td>5.228403</td>\n",
       "      <td>6.445756</td>\n",
       "      <td>4.578980</td>\n",
       "      <td>6.251507</td>\n",
       "      <td>7.807007</td>\n",
       "      <td>6.982170</td>\n",
       "      <td>6.706429</td>\n",
       "      <td>5.731793</td>\n",
       "      <td>3.777624</td>\n",
       "      <td>5.751758</td>\n",
       "      <td>7.651683</td>\n",
       "      <td>5.296082</td>\n",
       "      <td>6.764817</td>\n",
       "      <td>7.918257</td>\n",
       "      <td>6.821447</td>\n",
       "      <td>7.841411</td>\n",
       "      <td>5.637117</td>\n",
       "      <td>4.329357</td>\n",
       "      <td>5.521597</td>\n",
       "      <td>22.443131</td>\n",
       "      <td>17.419560</td>\n",
       "      <td>20.269719</td>\n",
       "      <td>17.424695</td>\n",
       "      <td>10.663709</td>\n",
       "      <td>11.883570</td>\n",
       "      <td>30.333417</td>\n",
       "      <td>13.267618</td>\n",
       "      <td>14.443982</td>\n",
       "      <td>18.385351</td>\n",
       "      <td>11.751721</td>\n",
       "      <td>11.845391</td>\n",
       "      <td>7.086213</td>\n",
       "      <td>8.298645</td>\n",
       "      <td>7.709710</td>\n",
       "      <td>8.413597</td>\n",
       "      <td>7.575516</td>\n",
       "      <td>0.418508</td>\n",
       "      <td>6.174939</td>\n",
       "      <td>7.159017</td>\n",
       "      <td>7.360649</td>\n",
       "      <td>6.398942</td>\n",
       "      <td>6.77437</td>\n",
       "      <td>5.464096</td>\n",
       "      <td>5.961728</td>\n",
       "      <td>5.028692</td>\n",
       "      <td>0.975515</td>\n",
       "      <td>3.222717</td>\n",
       "      <td>5.729282</td>\n",
       "      <td>1.757534</td>\n",
       "      <td>2.410347</td>\n",
       "      <td>26.177172</td>\n",
       "      <td>15.857460</td>\n",
       "      <td>17.790569</td>\n",
       "      <td>17.56748</td>\n",
       "      <td>9.906301</td>\n",
       "      <td>12.763983</td>\n",
       "      <td>7.124333</td>\n",
       "      <td>7.937693</td>\n",
       "      <td>8.243334</td>\n",
       "      <td>7.593882</td>\n",
       "      <td>7.490177</td>\n",
       "      <td>20.905422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>158.796091</td>\n",
       "      <td>0.500031</td>\n",
       "      <td>0.374118</td>\n",
       "      <td>6.007228</td>\n",
       "      <td>4.338223</td>\n",
       "      <td>5.525921</td>\n",
       "      <td>5.474034</td>\n",
       "      <td>158.796091</td>\n",
       "      <td>0.371424</td>\n",
       "      <td>0.303636</td>\n",
       "      <td>0.488887</td>\n",
       "      <td>3.315523</td>\n",
       "      <td>1.254236</td>\n",
       "      <td>12.529495</td>\n",
       "      <td>7.071042</td>\n",
       "      <td>6.817184</td>\n",
       "      <td>6.082324</td>\n",
       "      <td>5.928411</td>\n",
       "      <td>6.343081</td>\n",
       "      <td>0.493345</td>\n",
       "      <td>1.954206</td>\n",
       "      <td>1.744981</td>\n",
       "      <td>1.549377</td>\n",
       "      <td>1.955485</td>\n",
       "      <td>1.79415</td>\n",
       "      <td>2.156260</td>\n",
       "      <td>2.066148</td>\n",
       "      <td>2.295339</td>\n",
       "      <td>0.204152</td>\n",
       "      <td>3.270753</td>\n",
       "      <td>3.751910</td>\n",
       "      <td>1.234387</td>\n",
       "      <td>2.842782</td>\n",
       "      <td>2.804193</td>\n",
       "      <td>1.012760</td>\n",
       "      <td>1.440866</td>\n",
       "      <td>1.102760</td>\n",
       "      <td>3.321365</td>\n",
       "      <td>2.605348</td>\n",
       "      <td>2.796486</td>\n",
       "      <td>2.404348</td>\n",
       "      <td>1.720169</td>\n",
       "      <td>2.040143</td>\n",
       "      <td>2.255823</td>\n",
       "      <td>2.565703</td>\n",
       "      <td>2.431303</td>\n",
       "      <td>2.486438</td>\n",
       "      <td>1.955160</td>\n",
       "      <td>2.526212</td>\n",
       "      <td>2.231838</td>\n",
       "      <td>1.689876</td>\n",
       "      <td>2.160114</td>\n",
       "      <td>1.793030</td>\n",
       "      <td>2.600227</td>\n",
       "      <td>2.718190</td>\n",
       "      <td>1.729667</td>\n",
       "      <td>12.529495</td>\n",
       "      <td>7.071042</td>\n",
       "      <td>6.817184</td>\n",
       "      <td>6.082324</td>\n",
       "      <td>5.928411</td>\n",
       "      <td>6.343081</td>\n",
       "      <td>16.282924</td>\n",
       "      <td>6.985425</td>\n",
       "      <td>6.272354</td>\n",
       "      <td>6.565583</td>\n",
       "      <td>6.909752</td>\n",
       "      <td>6.158987</td>\n",
       "      <td>1.406236</td>\n",
       "      <td>1.410381</td>\n",
       "      <td>1.572079</td>\n",
       "      <td>1.078214</td>\n",
       "      <td>1.781908</td>\n",
       "      <td>0.493345</td>\n",
       "      <td>1.953614</td>\n",
       "      <td>1.744981</td>\n",
       "      <td>1.549377</td>\n",
       "      <td>1.954790</td>\n",
       "      <td>1.79415</td>\n",
       "      <td>2.156260</td>\n",
       "      <td>2.066148</td>\n",
       "      <td>2.295339</td>\n",
       "      <td>0.977727</td>\n",
       "      <td>2.460367</td>\n",
       "      <td>1.722648</td>\n",
       "      <td>0.958828</td>\n",
       "      <td>0.631441</td>\n",
       "      <td>14.426043</td>\n",
       "      <td>6.642497</td>\n",
       "      <td>6.501613</td>\n",
       "      <td>6.09809</td>\n",
       "      <td>5.624022</td>\n",
       "      <td>6.627344</td>\n",
       "      <td>1.380762</td>\n",
       "      <td>1.500331</td>\n",
       "      <td>1.182448</td>\n",
       "      <td>1.556004</td>\n",
       "      <td>1.743512</td>\n",
       "      <td>17.017103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.730000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>156.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>156.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.020000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>17.390000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.00000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.00000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>280.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>280.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.110000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.00000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.370000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.110000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>14.945000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.00000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>18.00000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>12.960000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>19.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>410.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>410.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.430000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.080000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>23.080000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>18.750000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>15.630000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.00000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.00000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.670000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>25.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.500000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>10.00000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.00000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>85.858586</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>50.00000</td>\n",
       "      <td>22.220000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               iid       gender       condtn         wave        round  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean    284.513310     0.500000     1.831743    11.387996    16.895530   \n",
       "std     158.796091     0.500031     0.374118     6.007228     4.338223   \n",
       "min       1.000000     0.000000     1.000000     1.000000     5.000000   \n",
       "25%     156.000000     0.000000     2.000000     7.000000    14.000000   \n",
       "50%     280.000000     0.500000     2.000000    11.000000    18.000000   \n",
       "75%     410.000000     1.000000     2.000000    15.000000    20.000000   \n",
       "max     552.000000     1.000000     2.000000    21.000000    22.000000   \n",
       "\n",
       "          position        order          pid        match     int_corr  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      9.073832     8.943621   284.513310     0.165244     0.194619   \n",
       "std       5.525921     5.474034   158.796091     0.371424     0.303636   \n",
       "min       1.000000     1.000000     1.000000     0.000000    -0.730000   \n",
       "25%       4.000000     4.000000   156.000000     0.000000    -0.020000   \n",
       "50%       8.000000     8.000000   280.000000     0.000000     0.210000   \n",
       "75%      13.000000    13.000000   410.000000     0.000000     0.430000   \n",
       "max      22.000000    22.000000   552.000000     1.000000     0.910000   \n",
       "\n",
       "          samerace        age_o       race_o     pf_o_att     pf_o_sin  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7924.000000  7924.000000   \n",
       "mean      0.395028    26.270844     2.780889    22.443131    17.419560   \n",
       "std       0.488887     3.315523     1.254236    12.529495     7.071042   \n",
       "min       0.000000    18.000000     1.000000     0.000000     0.000000   \n",
       "25%       0.000000    24.000000     2.000000    15.000000    15.000000   \n",
       "50%       0.000000    26.000000     2.000000    20.000000    18.370000   \n",
       "75%       1.000000    28.000000     4.000000    25.000000    20.000000   \n",
       "max       1.000000    36.000000     6.000000   100.000000    60.000000   \n",
       "\n",
       "          pf_o_int     pf_o_fun     pf_o_amb     pf_o_sha        dec_o  \\\n",
       "count  7924.000000  7915.000000  7906.000000  7884.000000  7964.000000   \n",
       "mean     20.269719    17.424695    10.663709    11.883570     0.418508   \n",
       "std       6.817184     6.082324     5.928411     6.343081     0.493345   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%      17.390000    15.000000     5.000000    10.000000     0.000000   \n",
       "50%      20.000000    18.000000    10.000000    11.110000     0.000000   \n",
       "75%      23.080000    20.000000    15.000000    16.000000     1.000000   \n",
       "max      50.000000    50.000000    30.000000    30.000000     1.000000   \n",
       "\n",
       "            attr_o       sinc_o      intel_o        fun_o       amb_o  \\\n",
       "count  7785.000000  7713.000000  7700.000000  7659.000000  7304.00000   \n",
       "mean      6.175389     7.159017     7.360649     6.399465     6.77437   \n",
       "std       1.954206     1.744981     1.549377     1.955485     1.79415   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.00000   \n",
       "25%       5.000000     6.000000     6.000000     5.000000     6.00000   \n",
       "50%       6.000000     7.000000     7.000000     7.000000     7.00000   \n",
       "75%       8.000000     8.000000     8.000000     8.000000     8.00000   \n",
       "max      10.500000    10.000000    10.000000    11.000000    10.00000   \n",
       "\n",
       "            shar_o       like_o       prob_o        met_o          age  \\\n",
       "count  6963.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      5.464096     5.961728     5.028692     1.956429    26.217353   \n",
       "std       2.156260     2.066148     2.295339     0.204152     3.270753   \n",
       "min       0.000000     0.000000     0.000000     1.000000    18.000000   \n",
       "25%       4.000000     5.000000     4.000000     2.000000    24.000000   \n",
       "50%       6.000000     6.000000     5.000000     2.000000    26.000000   \n",
       "75%       7.000000     7.000000     7.000000     2.000000    28.000000   \n",
       "max      10.000000    10.000000    10.000000     2.000000    36.000000   \n",
       "\n",
       "          field_cd         race      imprace     imprelig         goal  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7924.000000   \n",
       "mean      7.642265     2.760799     3.779633     3.661602     3.451666   \n",
       "std       3.751910     1.234387     2.842782     2.804193     1.012760   \n",
       "min       1.000000     1.000000     0.000000     1.000000     1.000000   \n",
       "25%       5.000000     2.000000     1.000000     1.000000     3.000000   \n",
       "50%       8.000000     2.000000     3.000000     3.000000     3.000000   \n",
       "75%      10.000000     4.000000     6.000000     6.000000     4.000000   \n",
       "max      18.000000     6.000000    10.000000    10.000000     6.000000   \n",
       "\n",
       "              date       go_out     career_c       sports     tvsports  \\\n",
       "count  7964.000000  7924.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      2.997489     5.857017     5.228403     6.445756     4.578980   \n",
       "std       1.440866     1.102760     3.321365     2.605348     2.796486   \n",
       "min       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "25%       2.000000     5.000000     2.000000     5.000000     2.000000   \n",
       "50%       3.000000     6.000000     6.000000     7.000000     4.000000   \n",
       "75%       4.000000     7.000000     7.000000     9.000000     7.000000   \n",
       "max       7.000000     7.000000    17.000000    10.000000    10.000000   \n",
       "\n",
       "          exercise       dining      museums          art       hiking  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      6.251507     7.807007     6.982170     6.706429     5.731793   \n",
       "std       2.404348     1.720169     2.040143     2.255823     2.565703   \n",
       "min       1.000000     1.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000     7.000000     6.000000     5.000000     4.000000   \n",
       "50%       7.000000     8.000000     7.000000     7.000000     6.000000   \n",
       "75%       8.000000     9.000000     8.000000     8.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "            gaming     clubbing      reading           tv      theater  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      3.777624     5.751758     7.651683     5.296082     6.764817   \n",
       "std       2.431303     2.486438     1.955160     2.526212     2.231838   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       2.000000     4.000000     7.000000     3.000000     5.000000   \n",
       "50%       3.000000     6.000000     8.000000     6.000000     7.000000   \n",
       "75%       6.000000     8.000000     9.000000     7.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "            movies     concerts        music     shopping         yoga  \\\n",
       "count  7964.000000  7964.000000  7964.000000  7964.000000  7964.000000   \n",
       "mean      7.918257     6.821447     7.841411     5.637117     4.329357   \n",
       "std       1.689876     2.160114     1.793030     2.600227     2.718190   \n",
       "min       0.000000     0.000000     1.000000     1.000000     0.000000   \n",
       "25%       7.000000     5.000000     7.000000     4.000000     2.000000   \n",
       "50%       8.000000     7.000000     8.000000     6.000000     4.000000   \n",
       "75%       9.000000     8.000000     9.000000     8.000000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.000000    10.000000   \n",
       "\n",
       "          exphappy      attr1_1      sinc1_1     intel1_1       fun1_1  \\\n",
       "count  7964.000000  7924.000000  7924.000000  7924.000000  7915.000000   \n",
       "mean      5.521597    22.443131    17.419560    20.269719    17.424695   \n",
       "std       1.729667    12.529495     7.071042     6.817184     6.082324   \n",
       "min       1.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000    15.000000    15.000000    17.390000    15.000000   \n",
       "50%       6.000000    20.000000    18.370000    20.000000    18.000000   \n",
       "75%       7.000000    25.000000    20.000000    23.080000    20.000000   \n",
       "max      10.000000   100.000000    60.000000    50.000000    50.000000   \n",
       "\n",
       "            amb1_1      shar1_1      attr2_1      sinc2_1     intel2_1  \\\n",
       "count  7906.000000  7884.000000  7924.000000  7924.000000  7924.000000   \n",
       "mean     10.663709    11.883570    30.333417    13.267618    14.443982   \n",
       "std       5.928411     6.343081    16.282924     6.985425     6.272354   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       5.000000    10.000000    20.000000    10.000000    10.000000   \n",
       "50%      10.000000    11.110000    25.000000    14.945000    15.000000   \n",
       "75%      15.000000    16.000000    40.000000    18.750000    20.000000   \n",
       "max      30.000000    30.000000   100.000000    50.000000    40.000000   \n",
       "\n",
       "            fun2_1       amb2_1      shar2_1      attr3_1      sinc3_1  \\\n",
       "count  7924.000000  7915.000000  7915.000000  7899.000000  7899.000000   \n",
       "mean     18.385351    11.751721    11.845391     7.086213     8.298645   \n",
       "std       6.565583     6.909752     6.158987     1.406236     1.410381   \n",
       "min       0.000000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.000000     5.555556    10.000000     6.000000     8.000000   \n",
       "50%      20.000000    10.000000    10.000000     7.000000     8.000000   \n",
       "75%      20.000000    15.000000    15.630000     8.000000     9.000000   \n",
       "max      50.000000    50.000000    30.000000    10.000000    10.000000   \n",
       "\n",
       "            fun3_1     intel3_1       amb3_1          dec         attr  \\\n",
       "count  7899.000000  7899.000000  7899.000000  7964.000000  7785.000000   \n",
       "mean      7.709710     8.413597     7.575516     0.418508     6.174939   \n",
       "std       1.572079     1.078214     1.781908     0.493345     1.953614   \n",
       "min       2.000000     3.000000     2.000000     0.000000     0.000000   \n",
       "25%       7.000000     8.000000     7.000000     0.000000     5.000000   \n",
       "50%       8.000000     8.000000     8.000000     0.000000     6.000000   \n",
       "75%       9.000000     9.000000     9.000000     1.000000     8.000000   \n",
       "max      10.000000    10.000000    10.000000     1.000000    10.000000   \n",
       "\n",
       "              sinc        intel          fun         amb         shar  \\\n",
       "count  7713.000000  7700.000000  7659.000000  7304.00000  6963.000000   \n",
       "mean      7.159017     7.360649     6.398942     6.77437     5.464096   \n",
       "std       1.744981     1.549377     1.954790     1.79415     2.156260   \n",
       "min       0.000000     0.000000     0.000000     0.00000     0.000000   \n",
       "25%       6.000000     6.000000     5.000000     6.00000     4.000000   \n",
       "50%       7.000000     7.000000     7.000000     7.00000     6.000000   \n",
       "75%       8.000000     8.000000     8.000000     8.00000     7.000000   \n",
       "max      10.000000    10.000000    10.000000    10.00000    10.000000   \n",
       "\n",
       "              like         prob          met     match_es      satis_2  \\\n",
       "count  7964.000000  7964.000000  7964.000000  6854.000000  7964.000000   \n",
       "mean      5.961728     5.028692     0.975515     3.222717     5.729282   \n",
       "std       2.066148     2.295339     0.977727     2.460367     1.722648   \n",
       "min       0.000000     0.000000     0.000000     0.000000     1.000000   \n",
       "25%       5.000000     4.000000     0.000000     2.000000     5.000000   \n",
       "50%       6.000000     5.000000     1.000000     3.000000     6.000000   \n",
       "75%       7.000000     7.000000     2.000000     4.000000     7.000000   \n",
       "max      10.000000    10.000000     2.000000    18.000000    10.000000   \n",
       "\n",
       "            length     numdat_2      attr1_2      sinc1_2     intel1_2  \\\n",
       "count  7964.000000  7964.000000  7109.000000  7126.000000  7126.000000   \n",
       "mean      1.757534     2.410347    26.177172    15.857460    17.790569   \n",
       "std       0.958828     0.631441    14.426043     6.642497     6.501613   \n",
       "min       1.000000     1.000000     5.000000     0.000000     0.000000   \n",
       "25%       1.000000     2.000000    16.670000    10.000000    15.000000   \n",
       "50%       1.000000     2.000000    20.000000    16.666667    19.000000   \n",
       "75%       3.000000     3.000000    30.000000    20.000000    20.000000   \n",
       "max       3.000000     3.000000    85.858586    50.000000    40.000000   \n",
       "\n",
       "           fun1_2       amb1_2      shar1_2      attr3_2      sinc3_2  \\\n",
       "count  7126.00000  7126.000000  7126.000000  7126.000000  7126.000000   \n",
       "mean     17.56748     9.906301    12.763983     7.124333     7.937693   \n",
       "std       6.09809     5.624022     6.627344     1.380762     1.500331   \n",
       "min       0.00000     0.000000     0.000000     2.000000     2.000000   \n",
       "25%      15.00000     5.000000    10.000000     7.000000     7.000000   \n",
       "50%      18.00000    10.000000    12.960000     7.000000     8.000000   \n",
       "75%      20.00000    15.000000    16.670000     8.000000     9.000000   \n",
       "max      50.00000    22.220000    35.000000    10.000000    10.000000   \n",
       "\n",
       "          intel3_2       fun3_2       amb3_2   match_es_r  \n",
       "count  7126.000000  7126.000000  7126.000000  7964.000000  \n",
       "mean      8.243334     7.593882     7.490177    20.905422  \n",
       "std       1.182448     1.556004     1.743512    17.017103  \n",
       "min       4.000000     1.000000     2.000000     0.000000  \n",
       "25%       8.000000     7.000000     7.000000    10.000000  \n",
       "50%       8.000000     8.000000     8.000000    19.047619  \n",
       "75%       9.000000     9.000000     9.000000    25.000000  \n",
       "max      10.000000    10.000000    10.000000   100.000000  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7964, 99)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_clean.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clean.to_csv('./src/speed_dating_clean.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "eda",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}