01_BuildingSequences.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1009,
     "status": "ok",
     "timestamp": 1581580292448,
     "user": {
      "displayName": "Stelios Paras",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCrrVG0C4KKts4mHhl6k_QPGfeBUnd_HugEyl87=s64",
      "userId": "14262754419909116823"
     },
     "user_tz": -60
    },
    "id": "4h351OyK_b4d",
    "outputId": "58f89e2d-ad54-4835-db10-9b04d16eaa51",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir(\"project_dir\")\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oItV6WBT_-ox"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import tensorflow as tf\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import random\n",
    "import glob\n",
    "import os\n",
    "from itertools import islice\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 554
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 527,
     "status": "ok",
     "timestamp": 1581580295598,
     "user": {
      "displayName": "Stelios Paras",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCrrVG0C4KKts4mHhl6k_QPGfeBUnd_HugEyl87=s64",
      "userId": "14262754419909116823"
     },
     "user_tz": -60
    },
    "id": "EcFikESbC-WE",
    "outputId": "4e61bea1-641d-4146-beea-0414617a761c"
   },
   "outputs": [],
   "source": [
    "glob.glob('project_dir/data/'+'*csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SQSumSq(df):\n",
    "    # compute Signal Vector Magnitude per input\n",
    "    return (df**2).sum(axis=1)**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LhPtL6laAEpe"
   },
   "outputs": [],
   "source": [
    "def read_data(name, randomSeed, numberForVal, folder1 = '... /Energy_Expenditure_Measurements/'):\n",
    "###### Reading datafiles. \n",
    "# This function creates dataFrames for train, test and validation datasets for the LOSO-CV.\n",
    "# The read_data function takes two variables, i.e, the name of the participant ('GOTOV05) and their directory. \n",
    "# Then it creates the validation set which contains 2 participants selected at random (one with all data, one with\n",
    "# only indoors data). The random.seed is fixed always to the number of the participants selected for reproducable\n",
    "# results. The participant name passed as a variable above is used to create the test set.\n",
    "# The rest are used in training the model. \n",
    "    # name: the test participant\n",
    "    # folder: the dir of data\n",
    "    # numberForVal: the number of participants you want per group (all_data/only_indoors) as validation set \n",
    "    # random seed is the number of participants' id\n",
    "    # folder1: data directory\n",
    "######    \n",
    "    random.seed(randomSeed)\n",
    "    print('Random seed is:',randomSeed)\n",
    "    df_train, df_val, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()\n",
    "    ankle = ['ankle_x', 'ankle_y', 'ankle_z']\n",
    "    wrist = ['wrist_x', 'wrist_y', 'wrist_z'] \n",
    "    data_to_select_random_ = []\n",
    "    df_gene = pd.DataFrame()\n",
    "    df_cos = pd.DataFrame()\n",
    "\n",
    "    print('loading all datafiles...')\n",
    "    # for all participants\n",
    "    for data in glob.glob(folder1+'*csv'):\n",
    "        data = os.path.splitext(os.path.basename(data))[0]\n",
    "        # skip participants with no cosmoed\n",
    "        if data in ['GOTOV02', 'GOTOV03', 'GOTOV04', 'GOTOV19']:\n",
    "            continue\n",
    "        else:\n",
    "            # read data\n",
    "            df = pd.read_csv(folder1+data+\".csv\", header = 0, index_col = None, low_memory=False)\n",
    "            # keep target data and their timestamps\n",
    "            df_cosmed = df[['EEm', 'time']]\n",
    "            df_cosmed.rename(columns={'time':'time_cosmed'})\n",
    "            # keep timestamps separately\n",
    "            time = df.time\n",
    "\n",
    "            # keep accelerometer data\n",
    "            df_ankle = df[ankle]\n",
    "            # df_ankle = SQSumSq(df_ankle) # this can be used if you want to train with SVMs instead of all x,y,z\n",
    "            df_wrist = df[wrist]\n",
    "            # df_wrist = SQSumSq(df_wrist) # this can be used if you want to train with SVMs instead of all x,y,z\n",
    "\n",
    "            df = pd.concat([time, df_ankle, df_wrist], axis=1)\n",
    "            df = df.rename(columns={0:'ankle',1:'wrist'})\n",
    "\n",
    "            inv_yhat = np.empty((df.shape[0], 2))\n",
    "            inv_yhat.fill(np.nan)\n",
    "            inv_yhat[:df_cosmed.shape[0]] = df_cosmed\n",
    "            df_cosmed = pd.DataFrame(inv_yhat, columns=['EEm','time_cos'])\n",
    "\n",
    "            df = pd.concat([df, df_cosmed], axis=1)\n",
    "            df['participant'] = data\n",
    "\n",
    "            df_gene = df_gene.append(df)\n",
    "            data_to_select_random_.append(data)\n",
    "        #fi\n",
    "    #efor\n",
    "    \n",
    "    # exclude participant for test and build validation set\n",
    "    if name in data_to_select_random_: data_to_select_random_.remove(name) #fi\n",
    "  \n",
    "    # set of participants with all data\n",
    "    all_act_data = ['GOTOV08', 'GOTOV10', 'GOTOV11', 'GOTOV12', \n",
    "                   'GOTOV17', 'GOTOV20', 'GOTOV21', 'GOTOV28',\n",
    "                   'GOTOV29', 'GOTOV31', 'GOTOV33', 'GOTOV35']\n",
    "    # exclude participant for test if in all_data\n",
    "    if name in all_act_data: all_act_data.remove(name) #fi\n",
    "\n",
    "    # set of participants with only indoors activities\n",
    "    indoors_act_data = ['GOTOV22', 'GOTOV23', 'GOTOV13','GOTOV14', \n",
    "                       'GOTOV24', 'GOTOV25', 'GOTOV26', 'GOTOV27', \n",
    "                       'GOTOV30', 'GOTOV32', 'GOTOV34', 'GOTOV36']\n",
    "    # exclude participant for test if in indoors_act_data\n",
    "    if name in indoors_act_data: indoors_act_data.remove(name) #fi\n",
    "    # select partcipants for val_set\n",
    "    validation_data = [random.sample(all_act_data, numberForVal)[0], \n",
    "                       random.sample(indoors_act_data, numberForVal)[0]]\n",
    "\n",
    "    print('Getting val and train data.....')\n",
    "    # build train, val and test sets\n",
    "    for data in data_to_select_random_: \n",
    "        if data in validation_data:\n",
    "            df_val = df_val.append(df_gene.query('participant == \"'+data+'\"'))\n",
    "            print('val_data:', data)\n",
    "        else:\n",
    "            df_train = df_train.append(df_gene.query('participant == \"'+data+'\"'))\n",
    "        #fi\n",
    "    #for\n",
    "    print('Getting test data.....')\n",
    "    df_test = df_test.append(df_gene.query('participant == \"'+name+'\"'))\n",
    "    print('Done creating all dataframes.....')\n",
    "    \n",
    "    return df_train, df_val, df_test\n",
    "#edef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yIUwEkd5IeUu"
   },
   "outputs": [],
   "source": [
    "def standardizing_data(name, Xtrain, Xval, Xtest, SEQUENCE_SIZE, downsample_rate, func, EEm_rate, save_dir):\n",
    "###### Standarize data for training\n",
    "# This function takes as an input the data frames created by read_data() function and uses the StandardScaler() to\n",
    "# scale them in order to create the training sequences needed.\n",
    "    # name: the test participant\n",
    "    # Xtrain: the train dataset\n",
    "    # Xval: the val dataset\n",
    "    # Xtest: the test dataset   \n",
    "    \n",
    "    # the rest of the inputs are passed as input to the  sequence_builder function\n",
    "    # SEQUENCE_SIZE: the length of the sequences to be build\n",
    "        # This input is given in terms of instance, e.g. for a 4min window n= 4*60*83, where 4 is the minutes, \n",
    "        # and 83 the Sampling Rate\n",
    "        # For example 465 is the number of instances for a window of 4 minutes with a sampling rate of SR=2Hz, \n",
    "        # downsampled from orignal data. If the SR is changed it has to be recalculated. \n",
    "        # For more details about the sequences see below\n",
    "    # downsample_rate: the sampling rate to downsample the accelerometer data for the sequences\n",
    "    # func: is the function used for sampling rate\n",
    "        # 'mean': keep the mean of the window selected for downsampling\n",
    "        # 'std': keep the standard deviation of the window selected for downsampling\n",
    "        # 'qdif': keep the given percentiles difference (0.05 and 0.95 for us here) of the window selected for downsampling\n",
    "        # 'iqr_dif': keep the interquartile range of the window selected for downsampling        \n",
    "    # EEm_rate: the downsampled rate of EEm values. Since COSMED give data per breath (unstable rate) we need to \n",
    "    #           fix the EEm rate to be stable.  EEm_rate == 0  no downsampling of target var EEm.\n",
    "    # save_dir: the directory where the sequences are saved\n",
    "######     \n",
    "    print('Scaling data....')\n",
    "\n",
    "    #variables to drop and use as columns later \n",
    "    cols = [ 'time_cos', 'participant','time']\n",
    "    cols_df = Xtrain.columns.tolist()\n",
    "    cols_df = cols_df[1:] + cols_df[:1]\n",
    "    \n",
    "    #scaling accelerometer measurements per set (train, val, test)\n",
    "    X_train = Xtrain.drop(cols, axis=1)\n",
    "    y_train = Xtrain[cols].values\n",
    "\n",
    "    X_val = Xval.drop(cols,  axis=1)\n",
    "    y_val = Xval[cols].values\n",
    "\n",
    "    X_test = Xtest.drop(cols,  axis=1)\n",
    "    y_test = Xtest[cols].values\n",
    "\n",
    "    # load scaler\n",
    "    scaler = StandardScaler()\n",
    "    # with mask we avoid the na's while standarizing\n",
    "    X_train = X_train.values.astype('float32')\n",
    "    X_train = np.ma.array(X_train, mask=np.isnan(X_train))\n",
    "    # fit scaler to train set\n",
    "    scaler.fit(X_train)\n",
    "    #scale train set\n",
    "    X_train = scaler.transform(X_train)\n",
    "    #scale val set\n",
    "    X_val = X_val.values.astype('float32')\n",
    "    X_val = np.ma.array(X_val, mask=np.isnan(X_val))\n",
    "    X_val = scaler.transform(X_val)\n",
    "    #scale test set\n",
    "    X_test = X_test.values.astype('float32')\n",
    "    X_test = np.ma.array(X_test, mask=np.isnan(X_test))\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    #concat again predictors with train data\n",
    "    X_train = pd.DataFrame(X_train)\n",
    "    y_train = pd.DataFrame(y_train)\n",
    "    X_train = pd.concat([X_train, y_train], axis=1)\n",
    "    X_train.columns = cols_df\n",
    "\n",
    "    X_val = pd.DataFrame(X_val)\n",
    "    y_val = pd.DataFrame(y_val)\n",
    "    X_val = pd.concat([X_val, y_val], axis=1)\n",
    "    X_val.columns = cols_df \n",
    "\n",
    "    X_test = pd.DataFrame(X_test)\n",
    "    y_test = pd.DataFrame(y_test)\n",
    "    X_test = pd.concat([X_test, y_test], axis=1)\n",
    "    X_test.columns = cols_df \n",
    "\n",
    "    #read, add and standarize participant level data (age, sex, weight, height)\n",
    "    encoder = LabelEncoder()\n",
    "\n",
    "    part_level_data_file = pd.read_csv('GOTOV_DataSummary.csv')\n",
    "    part_level_data_file_cols = ['ID', 'gender', 'age', 'weight', 'height', 'bmi']\n",
    "    part_level_data_file = part_level_data_file[part_level_data_file_cols]\n",
    "    part_level_data_file.gender = encoder.fit_transform(part_level_data_file.gender)\n",
    "\n",
    "    part_level_data_columns = part_level_data_file.columns.tolist()\n",
    "    part_level_data_columns =  part_level_data_columns[2:]+ part_level_data_columns[:-4]\n",
    "    part_level_data_file_numeric_cols = ['age', 'weight', 'height', 'bmi']\n",
    "    age_wt_ht = part_level_data_file[part_level_data_file_numeric_cols].values\n",
    "    part_level_data_file_non_numeric_cols = ['ID', 'gender']\n",
    "    part_sex_ID = part_level_data_file[part_level_data_file_non_numeric_cols].values\n",
    "\n",
    "    scaler2 = StandardScaler()\n",
    "    age_wt_ht = scaler2.fit_transform(age_wt_ht)\n",
    "\n",
    "    age_wt_ht = pd.DataFrame(age_wt_ht)\n",
    "    part_sex_ID = pd.DataFrame(part_sex_ID)\n",
    "    part_level_data_file = pd.concat([age_wt_ht, part_sex_ID], axis=1)\n",
    "    part_level_data_columns[4] = 'participant'\n",
    "    part_level_data_file.columns = part_level_data_columns\n",
    "\n",
    "    print('Done scaling data....')\n",
    "\n",
    "    # Call functions to build sequences\n",
    "    print('Building Train sequences.....')\n",
    "    X_train, y_train, ytrain_time, bmi_train = sequence_builder(X_train, part_level_data_file, SEQUENCE_SIZE, downsample_rate, func, EEm_rate)\n",
    "    print('Building Val sequences.....')\n",
    "    X_val, y_val, yval_time, bmi_val = sequence_builder(X_val, part_level_data_file, SEQUENCE_SIZE, downsample_rate, func, EEm_rate)\n",
    "    print('Building Test sequences.....')\n",
    "    X_test, y_test, ytest_time, bmi_test = sequence_builder(X_test, part_level_data_file, SEQUENCE_SIZE, downsample_rate, func, EEm_rate)\n",
    "\n",
    "    print('Saving sequences.... ')\n",
    "    with open(save_dir+name+'.pkl','wb') as f:\n",
    "        pickle.dump((X_train, y_train, ytrain_time, bmi_train, \n",
    "                     X_val, y_val, yval_time, bmi_val, \n",
    "                     X_test, y_test, ytest_time, bmi_test, \n",
    "                     scaler), f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VahexFKKBwvv"
   },
   "outputs": [],
   "source": [
    "# Custom downsampling functions added to the mean and std\n",
    "def dif_quantiles(df, limits = [.05, .95]):\n",
    "####\n",
    "# keep the given percentiles difference (0.05 and 0.95 for us here) of the df\n",
    "####\n",
    "    q = list(df.quantile(limits))\n",
    "#     diff_q = np.abs(q[1]-q[0])\n",
    "    diff_q = q[1]-q[0]\n",
    "    return diff_q\n",
    "\n",
    "def iqr_dif(df, limits = [.25, .75]):\n",
    "####\n",
    "# keep the interquartile range of the df\n",
    "####\n",
    "    q = list(df.quantile(limits))\n",
    "#     diff_q = np.abs(q[1]-q[0])\n",
    "    diff_q = q[1]-q[0]\n",
    "    return diff_q\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bsLmmI-4EB_M"
   },
   "source": [
    "#### Build sequences functions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sequence_builder(input_dataset, part_level_data, SEQUENCE_SIZE, downsample_rate, func, EEm_rate):\n",
    "###### \n",
    "# The sequence_builder function accepts a dataFrame returned by the standardizing_data function above and \n",
    "# transforms the data inputs into sequences required for training the machine learning model. \n",
    "# The to_sequences function (to_sequences_data()) is called within the sequence_builder function. \n",
    "    # input_dataset: the standardised input dataset with predictors and target data\n",
    "    # part_level_data: the standardised participant level data\n",
    "    # SEQUENCE_SIZE: the length of the sequences to be build\n",
    "        # This input is given in terms of instance, e.g. for a 4min window n= 4*60*83, where 4 is the minutes, \n",
    "        # and 83 the Sampling Rate. For example 465 is the number of instances for a window of 4 minutes with \n",
    "        # a sampling rate of SR=2Hz, downsampled from orignal data. If the SR is changed it has to be recalculated. \n",
    "    # downsample_rate: the sampling rate to downsample the accelerometer data for the sequences\n",
    "    # func: is the function used for sampling rate\n",
    "        # 'mean': keep the mean of the window selected for downsampling\n",
    "        # 'std': keep the standard deviation of the window selected for downsampling\n",
    "        # 'qdif': keep the given percentiles difference (0.05 and 0.95 for us here) of the window selected for downsampling\n",
    "        # 'iqr_dif': keep the interquartile range of the window selected for downsampling        \n",
    "    # EEm_rate: the downsampled rate of EEm values. Since COSMED give data per breath (unstable rate) we need to \n",
    "    #           fix the EEm rate to be stable. EEm_rate == 0  no downsampling of target var EEm.\n",
    "###### \n",
    "    #initialize\n",
    "    seqX, seqY, seqYtime, seqB = [], [], [], []\n",
    "    cols = ['time_cos', 'EEm']\n",
    "    k=1\n",
    "    for data in input_dataset['participant'].unique(): \n",
    "        # print('building sequence', k,'/', df_train['participant'].nunique(), ', participant - ',data)\n",
    "        k = k+1\n",
    "        df_input_dataset = input_dataset.query('participant == \"'+data+'\"')\n",
    "\n",
    "        part_level_data_part = part_level_data[part_level_data.participant==data]\n",
    "        part_level_data_part = part_level_data_part.drop('participant', axis=1).values.tolist()\n",
    "\n",
    "        df_accelData = df_input_dataset.drop(cols, axis=1)\n",
    "        df_cosmed = df_input_dataset[cols]\n",
    "\n",
    "        df_accelData.set_index('time', inplace=True)\n",
    "        df_accelData.index = pd.to_datetime(df_accelData.index, unit = \"ms\")\n",
    "\n",
    "        df_cosmed.set_index('time_cos', inplace=True)\n",
    "        df_cosmed.index = pd.to_datetime(df_cosmed.index, unit = \"ms\")\n",
    "\n",
    "        x_values = df_accelData.drop(['participant'], axis=1).sort_index()\n",
    "        idx = df_cosmed['EEm'].notnull()\n",
    "        y_values = df_cosmed['EEm'][idx].sort_index()\n",
    "\n",
    "        # downsampling accelerometer\n",
    "        key = str(round((1/downsample_rate), 3)) + 'S' # sampling rate for downsampling\n",
    "        if func == 'mean':\n",
    "            x_values = x_values.resample(key).mean()\n",
    "        elif func == 'std':\n",
    "            x_values = x_values.resample(key).std()\n",
    "        elif func == 'qdif':\n",
    "            x_values = x_values.resample(key).apply(dif_quantiles)\n",
    "        elif func == 'iqr_dif':\n",
    "            x_values = x_values.resample(key).apply(dif_quantiles)            \n",
    "                 \n",
    "        # delete created na values between indoors and outdoors data\n",
    "        idx1 = x_values['ankle_x'].notnull()\n",
    "        x_values = x_values[idx1]\n",
    "        \n",
    "        # downsampling EEm\n",
    "        if EEm_rate == 0: # no downsampling of target var EEm\n",
    "            y_values = df_cosmed['EEm'][idx].sort_index()\n",
    "        else:\n",
    "            key2 = str(EEm_rate) + 'S' # sampling rate for downsampling\n",
    "            y_values = y_values.resample(key2).mean()\n",
    "            idx2 = y_values.notnull()\n",
    "            y_values = y_values[idx2]\n",
    "        #fi\n",
    "        \n",
    "        # create sequences per window of data using function to_sequences_data()\n",
    "        x, y, bmi, y_time = to_sequences_data(SEQUENCE_SIZE, x_values, y_values, part_level_data_part, downsample_rate)\n",
    "        seqX.append(x)\n",
    "        seqY.append(y)\n",
    "        seqYtime.append(y_time)\n",
    "        seqB.append(bmi)\n",
    "    #efor\n",
    "    \n",
    "    # reshaping sequences  \n",
    "    x = np.vstack(seqX)\n",
    "    bmi = np.vstack(seqB)\n",
    "    y = [item for sublist in seqY for item in sublist]\n",
    "    y = np.array(y)\n",
    "    y_time = [item for sublist in seqYtime for item in sublist]\n",
    "    y_time = np.array(y_time)\n",
    "    x = x.reshape(x.shape[0], x.shape[2], x.shape[3])\n",
    "    print(x.shape, y.shape, bmi.shape)\n",
    "    return x, y, y_time, bmi\n",
    "#edef\n",
    "\n",
    "def to_sequences_data(SEQUENCE_SIZE, obs, yobs, part_level_data, downsample_rate):\n",
    "######\n",
    "# for every EEm value: \n",
    "# 1)take its timestamp, \n",
    "# 2)take all the accel timestamps, before that one, \n",
    "# 3)keep the last accel timestamps(accel instances) in the size of the window given\n",
    "    # SEQUENCE_SIZE: the length of the sequences to be build\n",
    "    # obs: accel predictors\n",
    "    # yobs: target (EEm)\n",
    "    # part_level_data: participants' level data\n",
    "    # downsample_rate: the sampling rate to downsample the accelerometer data for the sequences    \n",
    "######\n",
    "\n",
    "    x, y, y_time = [], [], []\n",
    "    for i in yobs.index.tolist():\n",
    "        # keep the window of data to sequence\n",
    "        window = obs.loc[:i].iloc[-SEQUENCE_SIZE:]\n",
    "        windowInMinutes = round(SEQUENCE_SIZE/(downsample_rate*60))  \n",
    "        if windowInMinutes < 1:\n",
    "            if window.shape[0]==0:\n",
    "                continue\n",
    "            else:\n",
    "                windowInSeconds = round((SEQUENCE_SIZE/(downsample_rate*60))*60)  \n",
    "    #             print('Window smaller than a minute, equal to: '+ str(SEQUENCE_SIZE/(downsample_rate*60))+' in sec: '+str(windowInSeconds))       \n",
    "                windowSize = (window.index[-1]-window.index[0])/np.timedelta64(1,'s') #calculate window size in seconds\n",
    "                if window.shape[0] == SEQUENCE_SIZE :\n",
    "                    after_window = yobs[i]\n",
    "                    x.append(window)\n",
    "                    y.append(after_window)\n",
    "                    y_time.append(i)\n",
    "\n",
    "                #fi\n",
    "            #fi\n",
    "\n",
    "        else:\n",
    "            if window.shape[0]==0:\n",
    "                continue\n",
    "            else:\n",
    "                windowSize = (window.index[-1]-window.index[0])/np.timedelta64(1,'m') #calculate window size in minutes\n",
    "                window = window.values.reshape(-1, window.shape[0], window.shape[1])\n",
    "                if window.shape[1] == SEQUENCE_SIZE and windowSize < windowInMinutes:\n",
    "                    after_window = yobs[i]\n",
    "                    x.append(window)\n",
    "                    y.append(after_window)\n",
    "                    y_time.append(i)\n",
    "                #fi\n",
    "            #fi\n",
    "        #fi\n",
    "\n",
    "    # BMI here are the demographics data and y is the number of sequences (not the length of a seq)\n",
    "    BMI = np.empty((len(y), np.array(part_level_data).shape[1]))\n",
    "    BMI = part_level_data*len(y)\n",
    "    return x, y, BMI, y_time\n",
    "#edef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 623,
     "status": "ok",
     "timestamp": 1581580332586,
     "user": {
      "displayName": "Stelios Paras",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCrrVG0C4KKts4mHhl6k_QPGfeBUnd_HugEyl87=s64",
      "userId": "14262754419909116823"
     },
     "user_tz": -60
    },
    "id": "Xp3914eTdBEZ",
    "outputId": "d1cb7065-d22d-42f4-c1ff-96a2f6176a1e"
   },
   "outputs": [],
   "source": [
    "# # # # # # # # # # # # FIXED VARs # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # \n",
    "numberForVal = 1 # the number of participants you want per group as validation set (random seed is the number of participants' id)\n",
    "SEQUENCE_SIZE = 50 # fixed size of esquences\n",
    "windowSize = 2 # in minutes\n",
    "func = 'std'\n",
    "# # # # # # # # # # # # GIVEN VARS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # \n",
    "# downsample_rate = 0.666666 # sampling rate of predictors given in Hz (2Hz create a window 4min, 4Hz -> 2min etc. )\n",
    "EEm_rate = 0 # sampling rate of target in seconds e.g. EEm=10 means one input every 10 seconds\n",
    "\n",
    "downsample_rate = round(SEQUENCE_SIZE/(windowSize*60),3)\n",
    "\n",
    "print('downsample_rate is', downsample_rate, 'Hz')\n",
    "# windowSize = round(SEQUENCE_SIZE/(downsample_rate*60),3) # size of window calculated from given downsample_rate \n",
    "print('windowSize is', windowSize, 'minutes')\n",
    "if windowSize < 1:\n",
    "    save_dir = '/preprocessedData/sequences/'+str(int(round(windowSize*60,0)))+'secs_seqs_'+str(numberForVal)+'_EEm_'+str(EEm_rate)+'ds_'+func+'_'+str(SEQUENCE_SIZE)+'/'\n",
    "else:\n",
    "    save_dir = '/preprocessedData/sequences/'+str(int(round(windowSize,0)))+'min_seqs_'+str(numberForVal)+'_EEm_'+str(EEm_rate)+'ds_'+func+'_'+str(SEQUENCE_SIZE)+'/'\n",
    "print('Save directory:', save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 329
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 183490,
     "status": "ok",
     "timestamp": 1581581757313,
     "user": {
      "displayName": "Stelios Paras",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mCrrVG0C4KKts4mHhl6k_QPGfeBUnd_HugEyl87=s64",
      "userId": "14262754419909116823"
     },
     "user_tz": -60
    },
    "id": "-yLLat0vJorf",
    "outputId": "ee784b85-d93c-46ea-f3a7-ff20568b734c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# calling the functions above to read data, standarize them, build seqs and save files\n",
    "if not os.path.exists(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "\n",
    "print('===========================================')\n",
    "# names = [6,10,14,15,18,23,25,29,36]\n",
    "# for i in names:\n",
    "for i in range(2,37):\n",
    "    if i in [2,3,4,19]: #[12, 19, 23, 16]: \n",
    "        continue\n",
    "    else: \n",
    "        if len(str(i)) == 1:\n",
    "            name = 'GOTOV0'+str(i)\n",
    "        else: \n",
    "            name = 'GOTOV'+str(i)\n",
    "    print('Build sequences for test patient', name)\n",
    "    df_train, df_val, df_test = read_data(name, i, numberForVal)\n",
    "    standardizing_data(name, df_train, df_val, df_test, SEQUENCE_SIZE, downsample_rate, func, EEm_rate, save_dir)\n",
    "    print('===========================================')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "BuildingSequences_ds_EEm.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}