test_separation_ell_sh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3e3c45b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2007b3b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/binary/ell_sh_data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "06be3bf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pad_sen_id</th>\n",
       "      <th>year</th>\n",
       "      <th>NotitieID</th>\n",
       "      <th>batch</th>\n",
       "      <th>annotator</th>\n",
       "      <th>background_sent</th>\n",
       "      <th>target_sent</th>\n",
       "      <th>text_raw</th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>...</th>\n",
       "      <th>secondary_preds</th>\n",
       "      <th>new_secondary_preds</th>\n",
       "      <th>icf-domains_combined_prim_sec</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_10_epochs</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_half_10ep/best_model</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_all_10ep</th>\n",
       "      <th>icf-domains_incl_primary_no_none_10ep/best_model</th>\n",
       "      <th>tmp_ellemijn/icf-domains_incl_primary_freeze_upto9_10ep</th>\n",
       "      <th>icf-domains_incl_primary_freeze_all_10ep/best_model</th>\n",
       "      <th>icf-domains_incl_primary_freeze_upto9_10ep/best_model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_best_model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>28136</td>\n",
       "      <td>...</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "      <td>20934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>1965</td>\n",
       "      <td>...</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "      <td>1148</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 pad_sen_id   year  NotitieID  batch  annotator  \\\n",
       "pred_best_model                                                   \n",
       "0                     28136  28136      28136  28136      28136   \n",
       "1                      1965   1965       1965   1965       1965   \n",
       "\n",
       "                 background_sent  target_sent  text_raw  labels   text  ...  \\\n",
       "pred_best_model                                                         ...   \n",
       "0                          28136        28136     28136   28136  28136  ...   \n",
       "1                           1965         1965      1965    1965   1965  ...   \n",
       "\n",
       "                 secondary_preds  new_secondary_preds  \\\n",
       "pred_best_model                                         \n",
       "0                          20934                20934   \n",
       "1                           1148                 1148   \n",
       "\n",
       "                 icf-domains_combined_prim_sec  \\\n",
       "pred_best_model                                  \n",
       "0                                        20934   \n",
       "1                                         1148   \n",
       "\n",
       "                 /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_10_epochs  \\\n",
       "pred_best_model                                                                     \n",
       "0                                                            20934                  \n",
       "1                                                             1148                  \n",
       "\n",
       "                 /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_half_10ep/best_model  \\\n",
       "pred_best_model                                                                                       \n",
       "0                                                            20934                                    \n",
       "1                                                             1148                                    \n",
       "\n",
       "                 /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_all_10ep  \\\n",
       "pred_best_model                                                                           \n",
       "0                                                            20934                        \n",
       "1                                                             1148                        \n",
       "\n",
       "                 icf-domains_incl_primary_no_none_10ep/best_model  \\\n",
       "pred_best_model                                                     \n",
       "0                                                           20934   \n",
       "1                                                            1148   \n",
       "\n",
       "                 tmp_ellemijn/icf-domains_incl_primary_freeze_upto9_10ep  \\\n",
       "pred_best_model                                                            \n",
       "0                                                            20934         \n",
       "1                                                             1148         \n",
       "\n",
       "                 icf-domains_incl_primary_freeze_all_10ep/best_model  \\\n",
       "pred_best_model                                                        \n",
       "0                                                            20934     \n",
       "1                                                             1148     \n",
       "\n",
       "                 icf-domains_incl_primary_freeze_upto9_10ep/best_model  \n",
       "pred_best_model                                                         \n",
       "0                                                            20934      \n",
       "1                                                             1148      \n",
       "\n",
       "[2 rows x 30 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('pred_best_model').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "70df41c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = df[(df['pred_best_model']== 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c9f8ae6f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pad_sen_id</th>\n",
       "      <th>year</th>\n",
       "      <th>NotitieID</th>\n",
       "      <th>batch</th>\n",
       "      <th>annotator</th>\n",
       "      <th>background_sent</th>\n",
       "      <th>target_sent</th>\n",
       "      <th>text_raw</th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "      <th>...</th>\n",
       "      <th>new_secondary_preds</th>\n",
       "      <th>icf-domains_combined_prim_sec</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_10_epochs</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_half_10ep/best_model</th>\n",
       "      <th>/data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_all_10ep</th>\n",
       "      <th>icf-domains_incl_primary_no_none_10ep/best_model</th>\n",
       "      <th>tmp_ellemijn/icf-domains_incl_primary_freeze_upto9_10ep</th>\n",
       "      <th>icf-domains_incl_primary_freeze_all_10ep/best_model</th>\n",
       "      <th>icf-domains_incl_primary_freeze_upto9_10ep/best_model</th>\n",
       "      <th>pred_best_model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0109003_0003</th>\n",
       "      <td>0109003_0003</td>\n",
       "      <td>2020</td>\n",
       "      <td>0109003</td>\n",
       "      <td>ergotherapie</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>al ze even klaar mee is , wordt ze moe , zware...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>al ze even klaar mee is , wordt ze moe , zware...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0119003_0001</th>\n",
       "      <td>0119003_0001</td>\n",
       "      <td>2020</td>\n",
       "      <td>0119003</td>\n",
       "      <td>ergotherapie</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>beoordeling therapeut vermoeidheid neemt iets ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>beoordeling therapeut vermoeidheid neemt iets ...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0131001_0005</th>\n",
       "      <td>0131001_0005</td>\n",
       "      <td>2020</td>\n",
       "      <td>0131001</td>\n",
       "      <td>fysiotherapie</td>\n",
       "      <td>meskers</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>met 2 uur werken per dag . komt</td>\n",
       "      <td>[0, 0, 1, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>met 2 uur werken per dag . komt</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0131001_0011</th>\n",
       "      <td>0131001_0011</td>\n",
       "      <td>2020</td>\n",
       "      <td>0131001</td>\n",
       "      <td>fysiotherapie</td>\n",
       "      <td>meskers</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>moe , zwaar werk vandaag gehad , ook iets lang...</td>\n",
       "      <td>[0, 0, 1, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>moe , zwaar werk vandaag gehad , ook iets lang...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0131001_0012</th>\n",
       "      <td>0131001_0012</td>\n",
       "      <td>2020</td>\n",
       "      <td>0131001</td>\n",
       "      <td>fysiotherapie</td>\n",
       "      <td>meskers</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>vragen of hij nog een weekje 2 uur per dag kan...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>vragen of hij nog een weekje 2 uur per dag kan...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79472723_0010</th>\n",
       "      <td>79472723_0010</td>\n",
       "      <td>2016</td>\n",
       "      <td>79472723</td>\n",
       "      <td>lung_patients</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Daarnaast geadviseerd de warme maaltijd niet z...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>Daarnaast geadviseerd de warme maaltijd niet z...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79472723_0012</th>\n",
       "      <td>79472723_0012</td>\n",
       "      <td>2016</td>\n",
       "      <td>79472723</td>\n",
       "      <td>lung_patients</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Man met zeer slechte voedingstoestand , wastin...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 1, 0]</td>\n",
       "      <td>Man met zeer slechte voedingstoestand , wastin...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79472723_0017</th>\n",
       "      <td>79472723_0017</td>\n",
       "      <td>2016</td>\n",
       "      <td>79472723</td>\n",
       "      <td>lung_patients</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Huidige intake &lt; 50 % vd geschatte energie- &amp; ...</td>\n",
       "      <td>[0, 0, 0, 0, 1, 0, 0, 0, 0]</td>\n",
       "      <td>Huidige intake &lt; 50 % vd geschatte energie- &amp; ...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79472723_0021</th>\n",
       "      <td>79472723_0021</td>\n",
       "      <td>2016</td>\n",
       "      <td>79472723</td>\n",
       "      <td>lung_patients</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Intake conform geschatte energie- &amp; eiwitbehoe...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 1, 0]</td>\n",
       "      <td>Intake conform geschatte energie- &amp; eiwitbehoe...</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84588639_0001</th>\n",
       "      <td>84588639_0001</td>\n",
       "      <td>2016</td>\n",
       "      <td>84588639</td>\n",
       "      <td>lung_patient_batch3</td>\n",
       "      <td>edwin</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>loopt zonder problemen in .</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>loopt zonder problemen in .</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1965 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  pad_sen_id  year NotitieID                batch annotator  \\\n",
       "0109003_0003    0109003_0003  2020   0109003         ergotherapie     edwin   \n",
       "0119003_0001    0119003_0001  2020   0119003         ergotherapie     edwin   \n",
       "0131001_0005    0131001_0005  2020   0131001        fysiotherapie   meskers   \n",
       "0131001_0011    0131001_0011  2020   0131001        fysiotherapie   meskers   \n",
       "0131001_0012    0131001_0012  2020   0131001        fysiotherapie   meskers   \n",
       "...                      ...   ...       ...                  ...       ...   \n",
       "79472723_0010  79472723_0010  2016  79472723        lung_patients     edwin   \n",
       "79472723_0012  79472723_0012  2016  79472723        lung_patients     edwin   \n",
       "79472723_0017  79472723_0017  2016  79472723        lung_patients     edwin   \n",
       "79472723_0021  79472723_0021  2016  79472723        lung_patients     edwin   \n",
       "84588639_0001  84588639_0001  2016  84588639  lung_patient_batch3     edwin   \n",
       "\n",
       "               background_sent  target_sent  \\\n",
       "0109003_0003             False        False   \n",
       "0119003_0001             False        False   \n",
       "0131001_0005             False        False   \n",
       "0131001_0011             False        False   \n",
       "0131001_0012             False         True   \n",
       "...                        ...          ...   \n",
       "79472723_0010            False        False   \n",
       "79472723_0012            False        False   \n",
       "79472723_0017            False        False   \n",
       "79472723_0021            False         True   \n",
       "84588639_0001            False        False   \n",
       "\n",
       "                                                        text_raw  \\\n",
       "0109003_0003   al ze even klaar mee is , wordt ze moe , zware...   \n",
       "0119003_0001   beoordeling therapeut vermoeidheid neemt iets ...   \n",
       "0131001_0005                     met 2 uur werken per dag . komt   \n",
       "0131001_0011   moe , zwaar werk vandaag gehad , ook iets lang...   \n",
       "0131001_0012   vragen of hij nog een weekje 2 uur per dag kan...   \n",
       "...                                                          ...   \n",
       "79472723_0010  Daarnaast geadviseerd de warme maaltijd niet z...   \n",
       "79472723_0012  Man met zeer slechte voedingstoestand , wastin...   \n",
       "79472723_0017  Huidige intake < 50 % vd geschatte energie- & ...   \n",
       "79472723_0021  Intake conform geschatte energie- & eiwitbehoe...   \n",
       "84588639_0001                        loopt zonder problemen in .   \n",
       "\n",
       "                                    labels  \\\n",
       "0109003_0003   [0, 0, 0, 1, 0, 0, 0, 0, 0]   \n",
       "0119003_0001   [1, 0, 0, 1, 0, 0, 0, 0, 0]   \n",
       "0131001_0005   [0, 0, 1, 0, 0, 0, 0, 0, 0]   \n",
       "0131001_0011   [0, 0, 1, 1, 0, 0, 0, 0, 0]   \n",
       "0131001_0012   [0, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "...                                    ...   \n",
       "79472723_0010  [0, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "79472723_0012  [0, 0, 0, 0, 0, 0, 0, 1, 0]   \n",
       "79472723_0017  [0, 0, 0, 0, 1, 0, 0, 0, 0]   \n",
       "79472723_0021  [0, 0, 0, 0, 0, 0, 0, 1, 0]   \n",
       "84588639_0001  [0, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "\n",
       "                                                            text  ...  \\\n",
       "0109003_0003   al ze even klaar mee is , wordt ze moe , zware...  ...   \n",
       "0119003_0001   beoordeling therapeut vermoeidheid neemt iets ...  ...   \n",
       "0131001_0005                     met 2 uur werken per dag . komt  ...   \n",
       "0131001_0011   moe , zwaar werk vandaag gehad , ook iets lang...  ...   \n",
       "0131001_0012   vragen of hij nog een weekje 2 uur per dag kan...  ...   \n",
       "...                                                          ...  ...   \n",
       "79472723_0010  Daarnaast geadviseerd de warme maaltijd niet z...  ...   \n",
       "79472723_0012  Man met zeer slechte voedingstoestand , wastin...  ...   \n",
       "79472723_0017  Huidige intake < 50 % vd geschatte energie- & ...  ...   \n",
       "79472723_0021  Intake conform geschatte energie- & eiwitbehoe...  ...   \n",
       "84588639_0001                        loopt zonder problemen in .  ...   \n",
       "\n",
       "               new_secondary_preds icf-domains_combined_prim_sec  \\\n",
       "0109003_0003                   NaN                           NaN   \n",
       "0119003_0001                   NaN                           NaN   \n",
       "0131001_0005                   NaN                           NaN   \n",
       "0131001_0011                   NaN                           NaN   \n",
       "0131001_0012                   NaN                           NaN   \n",
       "...                            ...                           ...   \n",
       "79472723_0010                  NaN                           NaN   \n",
       "79472723_0012                  NaN                           NaN   \n",
       "79472723_0017                  NaN                           NaN   \n",
       "79472723_0021                  NaN                           NaN   \n",
       "84588639_0001                  NaN                           NaN   \n",
       "\n",
       "              /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_10_epochs  \\\n",
       "0109003_0003                                                 NaN                 \n",
       "0119003_0001                                                 NaN                 \n",
       "0131001_0005                                                 NaN                 \n",
       "0131001_0011                                                 NaN                 \n",
       "0131001_0012                                                 NaN                 \n",
       "...                                                          ...                 \n",
       "79472723_0010                                                NaN                 \n",
       "79472723_0012                                                NaN                 \n",
       "79472723_0017                                                NaN                 \n",
       "79472723_0021                                                NaN                 \n",
       "84588639_0001                                                NaN                 \n",
       "\n",
       "              /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_half_10ep/best_model  \\\n",
       "0109003_0003                                                 NaN                                   \n",
       "0119003_0001                                                 NaN                                   \n",
       "0131001_0005                                                 NaN                                   \n",
       "0131001_0011                                                 NaN                                   \n",
       "0131001_0012                                                 NaN                                   \n",
       "...                                                          ...                                   \n",
       "79472723_0010                                                NaN                                   \n",
       "79472723_0012                                                NaN                                   \n",
       "79472723_0017                                                NaN                                   \n",
       "79472723_0021                                                NaN                                   \n",
       "84588639_0001                                                NaN                                   \n",
       "\n",
       "               /data2/Documents/tmp_ellemijn/icf-domains_incl_primary_freeze_all_10ep  \\\n",
       "0109003_0003                                                 NaN                        \n",
       "0119003_0001                                                 NaN                        \n",
       "0131001_0005                                                 NaN                        \n",
       "0131001_0011                                                 NaN                        \n",
       "0131001_0012                                                 NaN                        \n",
       "...                                                          ...                        \n",
       "79472723_0010                                                NaN                        \n",
       "79472723_0012                                                NaN                        \n",
       "79472723_0017                                                NaN                        \n",
       "79472723_0021                                                NaN                        \n",
       "84588639_0001                                                NaN                        \n",
       "\n",
       "              icf-domains_incl_primary_no_none_10ep/best_model  \\\n",
       "0109003_0003                                               NaN   \n",
       "0119003_0001                                               NaN   \n",
       "0131001_0005                                               NaN   \n",
       "0131001_0011                                               NaN   \n",
       "0131001_0012                                               NaN   \n",
       "...                                                        ...   \n",
       "79472723_0010                                              NaN   \n",
       "79472723_0012                                              NaN   \n",
       "79472723_0017                                              NaN   \n",
       "79472723_0021                                              NaN   \n",
       "84588639_0001                                              NaN   \n",
       "\n",
       "              tmp_ellemijn/icf-domains_incl_primary_freeze_upto9_10ep  \\\n",
       "0109003_0003                                                 NaN        \n",
       "0119003_0001                                                 NaN        \n",
       "0131001_0005                                                 NaN        \n",
       "0131001_0011                                                 NaN        \n",
       "0131001_0012                                                 NaN        \n",
       "...                                                          ...        \n",
       "79472723_0010                                                NaN        \n",
       "79472723_0012                                                NaN        \n",
       "79472723_0017                                                NaN        \n",
       "79472723_0021                                                NaN        \n",
       "84588639_0001                                                NaN        \n",
       "\n",
       "              icf-domains_incl_primary_freeze_all_10ep/best_model  \\\n",
       "0109003_0003                                                 NaN    \n",
       "0119003_0001                                                 NaN    \n",
       "0131001_0005                                                 NaN    \n",
       "0131001_0011                                                 NaN    \n",
       "0131001_0012                                                 NaN    \n",
       "...                                                          ...    \n",
       "79472723_0010                                                NaN    \n",
       "79472723_0012                                                NaN    \n",
       "79472723_0017                                                NaN    \n",
       "79472723_0021                                                NaN    \n",
       "84588639_0001                                                NaN    \n",
       "\n",
       "              icf-domains_incl_primary_freeze_upto9_10ep/best_model  \\\n",
       "0109003_0003                                                 NaN      \n",
       "0119003_0001                                                 NaN      \n",
       "0131001_0005                                                 NaN      \n",
       "0131001_0011                                                 NaN      \n",
       "0131001_0012                                                 NaN      \n",
       "...                                                          ...      \n",
       "79472723_0010                                                NaN      \n",
       "79472723_0012                                                NaN      \n",
       "79472723_0017                                                NaN      \n",
       "79472723_0021                                                NaN      \n",
       "84588639_0001                                                NaN      \n",
       "\n",
       "              pred_best_model  \n",
       "0109003_0003                1  \n",
       "0119003_0001                1  \n",
       "0131001_0005                1  \n",
       "0131001_0011                1  \n",
       "0131001_0012                1  \n",
       "...                       ...  \n",
       "79472723_0010               1  \n",
       "79472723_0012               1  \n",
       "79472723_0017               1  \n",
       "79472723_0021               1  \n",
       "84588639_0001               1  \n",
       "\n",
       "[1965 rows x 31 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d321f53f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1.to_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/binary/test_set_pos_ell_sh.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42baa811",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}