2_Experiment_2_part_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Step 1: Install the package\n",
    "# !pip install git+https://github.com/smazzanti/mrmr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2: Load packages\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from mrmr import mrmr_classif\n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Load data\n",
    "file1 = pd.read_csv('Data\\data_clinical_patient.csv')\n",
    "file2 = pd.read_csv('Data\\data_mRNA_median_all_sample_Zscores.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 24368 entries, 0 to 24367\n",
      "Columns: 1906 entries, Hugo_Symbol to MB-4313\n",
      "dtypes: float64(1905), object(1)\n",
      "memory usage: 354.4+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hugo_Symbol</th>\n",
       "      <th>Entrez_Gene_Id</th>\n",
       "      <th>MB-0362</th>\n",
       "      <th>MB-0346</th>\n",
       "      <th>MB-0386</th>\n",
       "      <th>MB-0574</th>\n",
       "      <th>MB-0503</th>\n",
       "      <th>MB-0641</th>\n",
       "      <th>MB-0201</th>\n",
       "      <th>MB-0218</th>\n",
       "      <th>...</th>\n",
       "      <th>MB-6122</th>\n",
       "      <th>MB-6192</th>\n",
       "      <th>MB-4820</th>\n",
       "      <th>MB-5527</th>\n",
       "      <th>MB-5167</th>\n",
       "      <th>MB-5465</th>\n",
       "      <th>MB-5453</th>\n",
       "      <th>MB-5471</th>\n",
       "      <th>MB-5127</th>\n",
       "      <th>MB-4313</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RERE</td>\n",
       "      <td>473.0</td>\n",
       "      <td>-0.7082</td>\n",
       "      <td>1.2179</td>\n",
       "      <td>0.0168</td>\n",
       "      <td>-0.4248</td>\n",
       "      <td>0.4916</td>\n",
       "      <td>0.5156</td>\n",
       "      <td>-1.2105</td>\n",
       "      <td>-0.9309</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5452</td>\n",
       "      <td>-0.4450</td>\n",
       "      <td>1.8429</td>\n",
       "      <td>1.1092</td>\n",
       "      <td>1.1871</td>\n",
       "      <td>-1.8702</td>\n",
       "      <td>1.1299</td>\n",
       "      <td>0.0481</td>\n",
       "      <td>-0.3357</td>\n",
       "      <td>-1.2562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RNF165</td>\n",
       "      <td>494470.0</td>\n",
       "      <td>-0.4419</td>\n",
       "      <td>0.4140</td>\n",
       "      <td>-0.6843</td>\n",
       "      <td>-1.1139</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>-0.2522</td>\n",
       "      <td>-0.4124</td>\n",
       "      <td>-0.0023</td>\n",
       "      <td>...</td>\n",
       "      <td>0.9537</td>\n",
       "      <td>-1.1564</td>\n",
       "      <td>0.9563</td>\n",
       "      <td>0.0470</td>\n",
       "      <td>-0.2570</td>\n",
       "      <td>3.2290</td>\n",
       "      <td>1.3609</td>\n",
       "      <td>0.6291</td>\n",
       "      <td>0.2281</td>\n",
       "      <td>0.6051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CD049690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2236</td>\n",
       "      <td>0.2255</td>\n",
       "      <td>0.5691</td>\n",
       "      <td>0.3545</td>\n",
       "      <td>0.7865</td>\n",
       "      <td>-0.3715</td>\n",
       "      <td>1.9356</td>\n",
       "      <td>-0.1612</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5783</td>\n",
       "      <td>-0.4329</td>\n",
       "      <td>0.5928</td>\n",
       "      <td>-1.0796</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>-0.0018</td>\n",
       "      <td>0.8035</td>\n",
       "      <td>-0.6178</td>\n",
       "      <td>1.0327</td>\n",
       "      <td>0.8558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BC033982</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.1485</td>\n",
       "      <td>0.4763</td>\n",
       "      <td>-0.2446</td>\n",
       "      <td>0.2618</td>\n",
       "      <td>-0.2695</td>\n",
       "      <td>-0.8391</td>\n",
       "      <td>-0.6770</td>\n",
       "      <td>0.9853</td>\n",
       "      <td>...</td>\n",
       "      <td>0.1445</td>\n",
       "      <td>-3.1854</td>\n",
       "      <td>-2.2533</td>\n",
       "      <td>1.1311</td>\n",
       "      <td>0.4819</td>\n",
       "      <td>-2.5749</td>\n",
       "      <td>-1.6314</td>\n",
       "      <td>-0.8435</td>\n",
       "      <td>-1.0429</td>\n",
       "      <td>-0.1023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PHF7</td>\n",
       "      <td>51533.0</td>\n",
       "      <td>-0.3220</td>\n",
       "      <td>-1.0921</td>\n",
       "      <td>0.2830</td>\n",
       "      <td>-0.2864</td>\n",
       "      <td>0.0772</td>\n",
       "      <td>-0.4976</td>\n",
       "      <td>-0.6453</td>\n",
       "      <td>-0.0506</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.9190</td>\n",
       "      <td>-0.0539</td>\n",
       "      <td>0.7454</td>\n",
       "      <td>0.1631</td>\n",
       "      <td>0.8931</td>\n",
       "      <td>-0.9482</td>\n",
       "      <td>-0.0397</td>\n",
       "      <td>0.5491</td>\n",
       "      <td>-0.0115</td>\n",
       "      <td>4.1846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1906 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Hugo_Symbol  Entrez_Gene_Id  MB-0362  MB-0346  MB-0386  MB-0574  MB-0503  \\\n",
       "0        RERE           473.0  -0.7082   1.2179   0.0168  -0.4248   0.4916   \n",
       "1      RNF165        494470.0  -0.4419   0.4140  -0.6843  -1.1139  -0.6875   \n",
       "2    CD049690             NaN   0.2236   0.2255   0.5691   0.3545   0.7865   \n",
       "3    BC033982             NaN  -2.1485   0.4763  -0.2446   0.2618  -0.2695   \n",
       "4        PHF7         51533.0  -0.3220  -1.0921   0.2830  -0.2864   0.0772   \n",
       "\n",
       "   MB-0641  MB-0201  MB-0218  ...  MB-6122  MB-6192  MB-4820  MB-5527  \\\n",
       "0   0.5156  -1.2105  -0.9309  ...  -0.5452  -0.4450   1.8429   1.1092   \n",
       "1  -0.2522  -0.4124  -0.0023  ...   0.9537  -1.1564   0.9563   0.0470   \n",
       "2  -0.3715   1.9356  -0.1612  ...  -0.5783  -0.4329   0.5928  -1.0796   \n",
       "3  -0.8391  -0.6770   0.9853  ...   0.1445  -3.1854  -2.2533   1.1311   \n",
       "4  -0.4976  -0.6453  -0.0506  ...  -0.9190  -0.0539   0.7454   0.1631   \n",
       "\n",
       "   MB-5167  MB-5465  MB-5453  MB-5471  MB-5127  MB-4313  \n",
       "0   1.1871  -1.8702   1.1299   0.0481  -0.3357  -1.2562  \n",
       "1  -0.2570   3.2290   1.3609   0.6291   0.2281   0.6051  \n",
       "2   0.1163  -0.0018   0.8035  -0.6178   1.0327   0.8558  \n",
       "3   0.4819  -2.5749  -1.6314  -0.8435  -1.0429  -0.1023  \n",
       "4   0.8931  -0.9482  -0.0397   0.5491  -0.0115   4.1846  \n",
       "\n",
       "[5 rows x 1906 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Have a quick look at data\n",
    "file2.info()\n",
    "file2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of duplicate values of Hugo_Symbol in data: 192\n",
      "After pre-processing, the number of duplicate values of Hugo_Symbol in data: 0\n",
      "Shape of Gene data: (24176, 1905)\n",
      "New shape of Gene data: (1904, 24177)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PATIENT_ID</th>\n",
       "      <th>RERE</th>\n",
       "      <th>RNF165</th>\n",
       "      <th>CD049690</th>\n",
       "      <th>BC033982</th>\n",
       "      <th>PHF7</th>\n",
       "      <th>CIDEA</th>\n",
       "      <th>PAPD4</th>\n",
       "      <th>AI082173</th>\n",
       "      <th>SLC17A3</th>\n",
       "      <th>...</th>\n",
       "      <th>BX115874</th>\n",
       "      <th>BX107598</th>\n",
       "      <th>UGCGL1</th>\n",
       "      <th>VPS72</th>\n",
       "      <th>CSMD3</th>\n",
       "      <th>CC2D1A</th>\n",
       "      <th>CB986545</th>\n",
       "      <th>IGSF9</th>\n",
       "      <th>DA110839</th>\n",
       "      <th>FAM71A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MB-0362</td>\n",
       "      <td>-0.7082</td>\n",
       "      <td>-0.4419</td>\n",
       "      <td>0.2236</td>\n",
       "      <td>-2.1485</td>\n",
       "      <td>-0.3220</td>\n",
       "      <td>0.0543</td>\n",
       "      <td>-0.7462</td>\n",
       "      <td>-0.4045</td>\n",
       "      <td>0.7777</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.8405</td>\n",
       "      <td>1.1698</td>\n",
       "      <td>0.1744</td>\n",
       "      <td>-0.2875</td>\n",
       "      <td>-0.5855</td>\n",
       "      <td>0.0174</td>\n",
       "      <td>-3.4444</td>\n",
       "      <td>0.4401</td>\n",
       "      <td>-1.0021</td>\n",
       "      <td>-1.1375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MB-0346</td>\n",
       "      <td>1.2179</td>\n",
       "      <td>0.4140</td>\n",
       "      <td>0.2255</td>\n",
       "      <td>0.4763</td>\n",
       "      <td>-1.0921</td>\n",
       "      <td>-1.1534</td>\n",
       "      <td>0.0709</td>\n",
       "      <td>0.5118</td>\n",
       "      <td>-0.5187</td>\n",
       "      <td>...</td>\n",
       "      <td>1.5500</td>\n",
       "      <td>-0.1237</td>\n",
       "      <td>0.4005</td>\n",
       "      <td>0.3579</td>\n",
       "      <td>-0.4784</td>\n",
       "      <td>-0.7659</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>1.2807</td>\n",
       "      <td>0.4464</td>\n",
       "      <td>0.0515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MB-0386</td>\n",
       "      <td>0.0168</td>\n",
       "      <td>-0.6843</td>\n",
       "      <td>0.5691</td>\n",
       "      <td>-0.2446</td>\n",
       "      <td>0.2830</td>\n",
       "      <td>2.9594</td>\n",
       "      <td>-0.6240</td>\n",
       "      <td>-0.3849</td>\n",
       "      <td>0.6866</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.2062</td>\n",
       "      <td>-1.0774</td>\n",
       "      <td>0.6514</td>\n",
       "      <td>0.4954</td>\n",
       "      <td>6.6421</td>\n",
       "      <td>0.0632</td>\n",
       "      <td>1.1914</td>\n",
       "      <td>0.8163</td>\n",
       "      <td>-1.1240</td>\n",
       "      <td>0.6751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 24177 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  PATIENT_ID    RERE  RNF165  CD049690  BC033982    PHF7   CIDEA   PAPD4  \\\n",
       "0    MB-0362 -0.7082 -0.4419    0.2236   -2.1485 -0.3220  0.0543 -0.7462   \n",
       "1    MB-0346  1.2179  0.4140    0.2255    0.4763 -1.0921 -1.1534  0.0709   \n",
       "2    MB-0386  0.0168 -0.6843    0.5691   -0.2446  0.2830  2.9594 -0.6240   \n",
       "\n",
       "   AI082173  SLC17A3  ...  BX115874  BX107598  UGCGL1   VPS72   CSMD3  CC2D1A  \\\n",
       "0   -0.4045   0.7777  ...   -0.8405    1.1698  0.1744 -0.2875 -0.5855  0.0174   \n",
       "1    0.5118  -0.5187  ...    1.5500   -0.1237  0.4005  0.3579 -0.4784 -0.7659   \n",
       "2   -0.3849   0.6866  ...   -1.2062   -1.0774  0.6514  0.4954  6.6421  0.0632   \n",
       "\n",
       "   CB986545   IGSF9  DA110839  FAM71A  \n",
       "0   -3.4444  0.4401   -1.0021 -1.1375  \n",
       "1    0.1327  1.2807    0.4464  0.0515  \n",
       "2    1.1914  0.8163   -1.1240  0.6751  \n",
       "\n",
       "[3 rows x 24177 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop unused column\n",
    "file2 = file2.drop('Entrez_Gene_Id', axis=1)\n",
    "\n",
    "# Drop NA in GeneID\n",
    "file2 = file2[file2['Hugo_Symbol'].notna()]\n",
    "\n",
    "# Check null in GeneID columns\n",
    "file2['Hugo_Symbol'].isnull().sum()\n",
    "\n",
    "# Check duplicate values\n",
    "print('The number of duplicate values of Hugo_Symbol in data:', file2['Hugo_Symbol'].duplicated().sum())\n",
    "\n",
    "# Drop duplicate values for Gene ID\n",
    "file2 = file2.drop_duplicates(subset=['Hugo_Symbol'])\n",
    "print('After pre-processing, the number of duplicate values of Hugo_Symbol in data:',\n",
    "     file2['Hugo_Symbol'].duplicated().sum())\n",
    "print('Shape of Gene data:', file2.shape)\n",
    "\n",
    "# Tranpose patient ID to rows in order to match two data\n",
    "file2 = file2.set_index('Hugo_Symbol').T.rename_axis('PATIENT_ID').rename_axis(None, axis=1).reset_index()\n",
    "print('New shape of Gene data:', file2.shape)\n",
    "file2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge gene data with OS time and status\n",
    "data = pd.merge(file1[['PATIENT_ID','OS_MONTHS','OS_STATUS']],file2, how=\"inner\", on=[\"PATIENT_ID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PATIENT_ID</th>\n",
       "      <th>OS_MONTHS</th>\n",
       "      <th>OS_STATUS</th>\n",
       "      <th>RERE</th>\n",
       "      <th>RNF165</th>\n",
       "      <th>CD049690</th>\n",
       "      <th>BC033982</th>\n",
       "      <th>PHF7</th>\n",
       "      <th>CIDEA</th>\n",
       "      <th>PAPD4</th>\n",
       "      <th>...</th>\n",
       "      <th>BX115874</th>\n",
       "      <th>BX107598</th>\n",
       "      <th>UGCGL1</th>\n",
       "      <th>VPS72</th>\n",
       "      <th>CSMD3</th>\n",
       "      <th>CC2D1A</th>\n",
       "      <th>CB986545</th>\n",
       "      <th>IGSF9</th>\n",
       "      <th>DA110839</th>\n",
       "      <th>FAM71A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MB-0000</td>\n",
       "      <td>140.500000</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>1.3762</td>\n",
       "      <td>0.1172</td>\n",
       "      <td>0.2469</td>\n",
       "      <td>-1.2960</td>\n",
       "      <td>-0.9217</td>\n",
       "      <td>3.8334</td>\n",
       "      <td>0.2327</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.6222</td>\n",
       "      <td>1.5451</td>\n",
       "      <td>-1.1909</td>\n",
       "      <td>-0.4725</td>\n",
       "      <td>-0.1735</td>\n",
       "      <td>-0.3961</td>\n",
       "      <td>0.3665</td>\n",
       "      <td>-1.6152</td>\n",
       "      <td>1.6121</td>\n",
       "      <td>-0.7582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MB-0002</td>\n",
       "      <td>84.633333</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>-0.0226</td>\n",
       "      <td>-0.9290</td>\n",
       "      <td>1.9792</td>\n",
       "      <td>1.0040</td>\n",
       "      <td>-1.0580</td>\n",
       "      <td>-0.1394</td>\n",
       "      <td>0.2341</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.4192</td>\n",
       "      <td>-1.1942</td>\n",
       "      <td>0.0247</td>\n",
       "      <td>-0.5276</td>\n",
       "      <td>-0.4878</td>\n",
       "      <td>-0.0706</td>\n",
       "      <td>1.2126</td>\n",
       "      <td>-0.0789</td>\n",
       "      <td>-1.1503</td>\n",
       "      <td>1.2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MB-0005</td>\n",
       "      <td>163.700000</td>\n",
       "      <td>1:DECEASED</td>\n",
       "      <td>-2.2425</td>\n",
       "      <td>-1.2323</td>\n",
       "      <td>0.3577</td>\n",
       "      <td>-0.1114</td>\n",
       "      <td>-0.4655</td>\n",
       "      <td>0.1429</td>\n",
       "      <td>1.3516</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.5978</td>\n",
       "      <td>-0.1758</td>\n",
       "      <td>-2.0710</td>\n",
       "      <td>-0.3882</td>\n",
       "      <td>-0.1544</td>\n",
       "      <td>-0.1448</td>\n",
       "      <td>0.4565</td>\n",
       "      <td>0.5685</td>\n",
       "      <td>-0.1200</td>\n",
       "      <td>1.4474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MB-0006</td>\n",
       "      <td>164.933333</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>-1.7706</td>\n",
       "      <td>-1.4902</td>\n",
       "      <td>0.7725</td>\n",
       "      <td>-2.0158</td>\n",
       "      <td>-0.4199</td>\n",
       "      <td>-1.0699</td>\n",
       "      <td>0.9961</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5841</td>\n",
       "      <td>0.2872</td>\n",
       "      <td>-1.3090</td>\n",
       "      <td>-1.7143</td>\n",
       "      <td>-0.4777</td>\n",
       "      <td>0.5786</td>\n",
       "      <td>0.0833</td>\n",
       "      <td>-1.2428</td>\n",
       "      <td>1.4114</td>\n",
       "      <td>0.2156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MB-0008</td>\n",
       "      <td>41.366667</td>\n",
       "      <td>1:DECEASED</td>\n",
       "      <td>-2.0498</td>\n",
       "      <td>-1.2677</td>\n",
       "      <td>-0.8829</td>\n",
       "      <td>-0.9718</td>\n",
       "      <td>-0.0172</td>\n",
       "      <td>2.0533</td>\n",
       "      <td>-0.4041</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0591</td>\n",
       "      <td>0.6851</td>\n",
       "      <td>-1.3606</td>\n",
       "      <td>-0.2123</td>\n",
       "      <td>-0.3088</td>\n",
       "      <td>-0.0857</td>\n",
       "      <td>1.5561</td>\n",
       "      <td>-1.0884</td>\n",
       "      <td>-1.3529</td>\n",
       "      <td>-1.2238</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24179 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  PATIENT_ID   OS_MONTHS   OS_STATUS    RERE  RNF165  CD049690  BC033982  \\\n",
       "0    MB-0000  140.500000    0:LIVING  1.3762  0.1172    0.2469   -1.2960   \n",
       "1    MB-0002   84.633333    0:LIVING -0.0226 -0.9290    1.9792    1.0040   \n",
       "2    MB-0005  163.700000  1:DECEASED -2.2425 -1.2323    0.3577   -0.1114   \n",
       "3    MB-0006  164.933333    0:LIVING -1.7706 -1.4902    0.7725   -2.0158   \n",
       "4    MB-0008   41.366667  1:DECEASED -2.0498 -1.2677   -0.8829   -0.9718   \n",
       "\n",
       "     PHF7   CIDEA   PAPD4  ...  BX115874  BX107598  UGCGL1   VPS72   CSMD3  \\\n",
       "0 -0.9217  3.8334  0.2327  ...   -0.6222    1.5451 -1.1909 -0.4725 -0.1735   \n",
       "1 -1.0580 -0.1394  0.2341  ...   -0.4192   -1.1942  0.0247 -0.5276 -0.4878   \n",
       "2 -0.4655  0.1429  1.3516  ...   -0.5978   -0.1758 -2.0710 -0.3882 -0.1544   \n",
       "3 -0.4199 -1.0699  0.9961  ...    0.5841    0.2872 -1.3090 -1.7143 -0.4777   \n",
       "4 -0.0172  2.0533 -0.4041  ...    1.0591    0.6851 -1.3606 -0.2123 -0.3088   \n",
       "\n",
       "   CC2D1A  CB986545   IGSF9  DA110839  FAM71A  \n",
       "0 -0.3961    0.3665 -1.6152    1.6121 -0.7582  \n",
       "1 -0.0706    1.2126 -0.0789   -1.1503  1.2770  \n",
       "2 -0.1448    0.4565  0.5685   -0.1200  1.4474  \n",
       "3  0.5786    0.0833 -1.2428    1.4114  0.2156  \n",
       "4 -0.0857    1.5561 -1.0884   -1.3529 -1.2238  \n",
       "\n",
       "[5 rows x 24179 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Have a quick look at data\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total missing values in the dataset: 10\n",
      "List columns having missing data: Index(['TMPRSS7', 'SLC25A19', 'IDO1', 'CSNK2A1', 'BAMBI', 'MRPL24', 'AK127905',\n",
      "       'FAM71A'],\n",
      "      dtype='object')\n",
      "After preprocessing, the number of missing values: 0\n"
     ]
    }
   ],
   "source": [
    "# Step 4: Preprocess data & Explore data\n",
    "\n",
    "# Step 4.1: Deal with missing values\n",
    "# Check missing values\n",
    "print('Total missing values in the dataset:', data.isnull().sum().sum())\n",
    "cols_missvalue = data.columns[data.isnull().sum()>0]\n",
    "print('List columns having missing data:', cols_missvalue)\n",
    "\n",
    "# Replace missing values with average values\n",
    "data[cols_missvalue] = data[cols_missvalue].fillna(data[cols_missvalue].mean())\n",
    "\n",
    "# Check missing values again\n",
    "print('After preprocessing, the number of missing values:', data.isna().sum().sum())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [03:40<00:00,  4.41s/it]\n"
     ]
    }
   ],
   "source": [
    "# Step 5: Feature extraction\n",
    "\n",
    "# 5.1: Normalise data\n",
    "ss = MinMaxScaler()\n",
    "X_norm = data.drop(['OS_STATUS', 'OS_MONTHS','PATIENT_ID'], axis = 1)\n",
    "X_norm = pd.DataFrame(ss.fit_transform(X_norm), columns=X_norm.columns)\n",
    "\n",
    "\n",
    "# 5.2: Select feature using MRMR\n",
    "y_mrmr = data['OS_MONTHS']\n",
    "\n",
    "features_selected = mrmr_classif(X_norm, y_mrmr, K = 50)\n",
    "X_mrmr = data[features_selected]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.3 Save to csv file\n",
    "df_mrmr = X_mrmr\n",
    "df_mrmr['PATIENT_ID'] = data['PATIENT_ID']\n",
    "df_mrmr.to_csv('Data\\Gene_MRMR_50.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1904, 51)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mrmr.shape"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "183bbf6827d058c2a2fb0f4acdc0420849dda2b4380af0e437e38c64d798d8b7"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}