create-counts-for-clicks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "88d669f0",
   "metadata": {
    "papermill": {
     "duration": 0.006875,
     "end_time": "2023-03-17T22:08:19.489453",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.482578",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Exact next click-to-click co-visitation matrix and some per aid calculations for clicks\n",
    "This notebook prepares click-to-click co-visitation matrix. It is used to and calculate features for the clicks model.\n",
    "Additionally, some per aid counts are performed in this notebook, including:\n",
    "* median time users view aid\n",
    "* average per day clicks per aid\n",
    "* return rate, counting how often users return for a new click or other actions with the same aid.\n",
    "\n",
    "These counts are used to engineer features for all the models."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c430879d",
   "metadata": {
    "papermill": {
     "duration": 0.005243,
     "end_time": "2023-03-17T22:08:19.500595",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.495352",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f3c90412",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.513831Z",
     "iopub.status.busy": "2023-03-17T22:08:19.513033Z",
     "iopub.status.idle": "2023-03-17T22:08:19.552652Z",
     "shell.execute_reply": "2023-03-17T22:08:19.551614Z"
    },
    "papermill": {
     "duration": 0.049324,
     "end_time": "2023-03-17T22:08:19.555417",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.506093",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b7bfb40e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.568674Z",
     "iopub.status.busy": "2023-03-17T22:08:19.568197Z",
     "iopub.status.idle": "2023-03-17T22:08:19.579484Z",
     "shell.execute_reply": "2023-03-17T22:08:19.578382Z"
    },
    "papermill": {
     "duration": 0.020553,
     "end_time": "2023-03-17T22:08:19.581674",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.561121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CalculateCovisitationMatrixExactNext(otto_common.CalculateCovisitationMatrix):\n",
    "    '''\n",
    "    Child class with logic specific to this type of covisitation matrix.\n",
    "    It counts all aids that were clicked exact after some event with another aid.\n",
    "    Weight value is calculated in a way that makes later events more important.\n",
    "    '''\n",
    "    \n",
    "    def __init__(self, *args, timebase, **kwargs):\n",
    "        self.timebase = timebase\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "    def calculate_weights(self, df_i):\n",
    "        df_i = df_i.sort_values(['session','ts'],ascending=[True,True])\n",
    "        df_i = df_i.reset_index(drop=True)\n",
    "        df_i['n'] = df_i.groupby('session').cumcount()\n",
    "        df_i2 = df_i.copy()\n",
    "        df_i2['n'] = df_i2['n'] - 1\n",
    "        df_j = df_i.merge(df_i2,how='inner',on=['session', 'n'])\n",
    "        df_j = df_j[['session', 'aid_x', 'aid_y','ts_x']].drop_duplicates(['session', 'aid_x', 'aid_y']) \n",
    "        df_j['wgt'] = (1 + 3*(df_j.ts_x - self.timebase)/3024000)   \n",
    "        df_j = df_j[['aid_x','aid_y','wgt']]\n",
    "        del df_i, df_i2\n",
    "        gc.collect()\n",
    "        df_j = df_j.loc[df_j['aid_x'].isin(self.aid_list)]\n",
    "        df_j.wgt = df_j.wgt.astype('float32')\n",
    "        df_j = self.groupby_reset_and_reduce(df_j)\n",
    "        return df_j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbed208f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.595041Z",
     "iopub.status.busy": "2023-03-17T22:08:19.594593Z",
     "iopub.status.idle": "2023-03-17T22:08:19.601099Z",
     "shell.execute_reply": "2023-03-17T22:08:19.599947Z"
    },
    "papermill": {
     "duration": 0.015839,
     "end_time": "2023-03-17T22:08:19.603270",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.587431",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This function normalizes matrix before calculating features.\n",
    "# Normalized means that all weights are divided by sum of weights per aid_x.\n",
    "def normalize_matrice(df, column_name):\n",
    "    print('start normalizing')\n",
    "    sum_name = column_name + '_sum'\n",
    "    df[sum_name] = df.groupby('aid_x')[column_name].transform('sum')\n",
    "    df[column_name] = 100 * df[column_name]/df[sum_name]\n",
    "    df = df.drop(sum_name, axis=1)\n",
    "    df[column_name] = df[column_name].fillna(0).astype(np.float32)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5ae7447c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.616961Z",
     "iopub.status.busy": "2023-03-17T22:08:19.615837Z",
     "iopub.status.idle": "2023-03-17T22:08:19.627640Z",
     "shell.execute_reply": "2023-03-17T22:08:19.626637Z"
    },
    "papermill": {
     "duration": 0.02077,
     "end_time": "2023-03-17T22:08:19.629843",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.609073",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count how often users return to each aid.\n",
    "def count_returns(train_data):\n",
    "    n_splits = 4\n",
    "    df_click_data = pd.read_parquet(train_data)\n",
    "    df_click_data = df_click_data.loc[df_click_data['type'] == 0]\n",
    "    for i in range(n_splits):\n",
    "        df_i = otto_common.divide_df_by_column(df_click_data, n_splits, i, 'session')\n",
    "        df_i = df_i.groupby(['session','aid']).size()\n",
    "        df_i.name = 'aid_counts'\n",
    "        df_i = df_i.reset_index()\n",
    "        if i == 0:\n",
    "            df_all = df_i\n",
    "        else:\n",
    "            df_all = pd.concat([df_all, df_i], axis=0)\n",
    "    df_total_aid = df_all.groupby('aid').size()\n",
    "    df_total_aid.name = 'total'\n",
    "    df_total_aid = df_total_aid.reset_index()\n",
    "    df_returns = df_all.loc[df_all['aid_counts'] > 1].groupby('aid').size()\n",
    "    df_returns.name = 'returns'\n",
    "    df_returns = df_returns.reset_index()\n",
    "    del df_all\n",
    "    gc.collect()\n",
    "    df_total_aid = pd.merge(df_total_aid, df_returns, how='left', on='aid')\n",
    "    df_total_aid['returns'] = df_total_aid['returns'].fillna(0)\n",
    "\n",
    "    df_total_aid['return_rate'] = (3 + df_total_aid['returns'])/(10 + df_total_aid['total'])\n",
    "    df_total_aid = df_total_aid[['aid', 'return_rate']]\n",
    "    df_total_aid['return_rate'] = df_total_aid['return_rate'].astype(np.float32)\n",
    "    df_total_aid['aid'] = df_total_aid['aid'].astype(np.int32)\n",
    "    return df_total_aid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a02f56ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.642961Z",
     "iopub.status.busy": "2023-03-17T22:08:19.642562Z",
     "iopub.status.idle": "2023-03-17T22:08:19.652983Z",
     "shell.execute_reply": "2023-03-17T22:08:19.652116Z"
    },
    "papermill": {
     "duration": 0.019567,
     "end_time": "2023-03-17T22:08:19.655056",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.635489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count for how long users wait after clicking each item.\n",
    "def median_time_clicked(train_data):\n",
    "    n_splits = 5\n",
    "    for i in range(n_splits):\n",
    "        df_i = pd.read_parquet(train_data)\n",
    "        df_i = otto_common.divide_df_by_column(df_i, n_splits, i, 'session')\n",
    "        df_i['n'] = df_i.groupby('session').cumcount().astype(np.int16)\n",
    "        df_i_plus_1 = df_i.copy()\n",
    "        df_i_plus_1['n'] = df_i_plus_1['n'] - 1\n",
    "        df_i = pd.merge(df_i, df_i_plus_1, how='inner', on=['session', 'n'])\n",
    "        del df_i_plus_1\n",
    "        gc.collect()\n",
    "        if i == 0:\n",
    "            df_all = df_i\n",
    "        else:\n",
    "            df_all = pd.concat([df_all, df_i])\n",
    "    del df_i\n",
    "    gc.collect()\n",
    "    print('merge_successfull')\n",
    "    df_all['time_viewed'] = df_all['ts_y'] - df_all['ts_x']\n",
    "    df_all = df_all[['session', 'aid_x', 'time_viewed', 'type_x']]\n",
    "    gc.collect()\n",
    "      \n",
    "    df_all['time_viewed'] = df_all['time_viewed'].clip(0,180)\n",
    "    df_all = df_all.loc[df_all['type_x'] == 0]\n",
    "    gc.collect()\n",
    "    df_all = df_all.groupby(['session','aid_x']).agg({'time_viewed':'median'})\n",
    "    df_all = df_all.reset_index()\n",
    "    df_all = df_all.groupby('aid_x').agg({'time_viewed':'median'})\n",
    "    df_all = df_all.rename(columns={'time_viewed':'time_viewed_clipped'})\n",
    "    df_all['time_viewed_clipped'] = df_all['time_viewed_clipped'].astype(np.float32)\n",
    "    gc.collect()\n",
    "    df_all = df_all.reset_index()\n",
    "    \n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08f5b39e",
   "metadata": {
    "papermill": {
     "duration": 0.005321,
     "end_time": "2023-03-17T22:08:19.666015",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.660694",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrice and other counts for cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f4e061c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:19.680398Z",
     "iopub.status.busy": "2023-03-17T22:08:19.679780Z",
     "iopub.status.idle": "2023-03-17T22:08:25.594145Z",
     "shell.execute_reply": "2023-03-17T22:08:25.592973Z"
    },
    "papermill": {
     "duration": 5.923988,
     "end_time": "2023-03-17T22:08:25.596950",
     "exception": false,
     "start_time": "2023-03-17T22:08:19.672962",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Make a list of AIDs, that appear in cross-validation sets.\n",
    "# aid_x is filtered to only include aids from the list, while aid_y are not filtered\n",
    "# This reduces size of resulting matrix.\n",
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "trunked_sessions_path2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "cv_list = otto_common.build_aid_list(trunked_sessions_path, trunked_sessions_path2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ad2eb2d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:08:25.611439Z",
     "iopub.status.busy": "2023-03-17T22:08:25.610554Z",
     "iopub.status.idle": "2023-03-17T22:21:43.121597Z",
     "shell.execute_reply": "2023-03-17T22:21:43.120237Z"
    },
    "papermill": {
     "duration": 797.521362,
     "end_time": "2023-03-17T22:21:43.124346",
     "exception": false,
     "start_time": "2023-03-17T22:08:25.602984",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "start normalizing\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Input parameters and matrix calculation itself.\n",
    "\n",
    "Unlike other co-visitation matrixes, this one is normalized right after it is calculated.\n",
    "Normalized means that all weights are divided by sum of weights per aid_x.\n",
    "'''\n",
    "\n",
    "click_data_cv = '/kaggle/input/otto-prepare-cv/cv_train.parquet'\n",
    "n_splits = 80\n",
    "n_splits_concat = 3\n",
    "timebase_cv = 1658700000\n",
    "\n",
    "covisitation_cv = CalculateCovisitationMatrixExactNext(n_splits, n_splits_concat, cv_list, timebase=timebase_cv)\n",
    "df_matrix_cv = covisitation_cv.generate_covisitation_matrix(click_data_cv)\n",
    "df_matrix_cv = normalize_matrice(df_matrix_cv, 'wgt')\n",
    "df_matrix_cv.to_parquet('matrix_exact_next_counts_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89e09138",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:21:43.147524Z",
     "iopub.status.busy": "2023-03-17T22:21:43.147069Z",
     "iopub.status.idle": "2023-03-17T22:21:43.299658Z",
     "shell.execute_reply": "2023-03-17T22:21:43.298528Z"
    },
    "papermill": {
     "duration": 0.167215,
     "end_time": "2023-03-17T22:21:43.302212",
     "exception": false,
     "start_time": "2023-03-17T22:21:43.134997",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_matrix_cv, cv_list, covisitation_cv\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9dda2db1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:21:43.325619Z",
     "iopub.status.busy": "2023-03-17T22:21:43.325188Z",
     "iopub.status.idle": "2023-03-17T22:26:28.772080Z",
     "shell.execute_reply": "2023-03-17T22:26:28.770778Z"
    },
    "papermill": {
     "duration": 285.461953,
     "end_time": "2023-03-17T22:26:28.774994",
     "exception": false,
     "start_time": "2023-03-17T22:21:43.313041",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculating number of clicks per aid each day.\n",
    "# To make it possible to compare data for days, when we know full sessions and days, when we only have trunked sessions, \n",
    "# click counts are divided by total number of clicks each day.\n",
    "\n",
    "df_daily_counts_train = otto_common.create_average_daily_counts(click_data_cv, False)\n",
    "df_daily_counts_train.to_parquet('daily_counts_train.parquet')\n",
    "\n",
    "df_daily_counts_cv1 = otto_common.create_average_daily_counts(trunked_sessions_path, True)\n",
    "df_daily_counts_cv1.to_parquet('daily_counts_cv1.parquet')\n",
    "\n",
    "df_daily_counts_cv2 = otto_common.create_average_daily_counts(trunked_sessions_path2, True)\n",
    "df_daily_counts_cv2.to_parquet('daily_counts_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2a4d76c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:26:28.799107Z",
     "iopub.status.busy": "2023-03-17T22:26:28.798271Z",
     "iopub.status.idle": "2023-03-17T22:26:28.922161Z",
     "shell.execute_reply": "2023-03-17T22:26:28.920977Z"
    },
    "papermill": {
     "duration": 0.138283,
     "end_time": "2023-03-17T22:26:28.924500",
     "exception": false,
     "start_time": "2023-03-17T22:26:28.786217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_daily_counts_train, df_daily_counts_cv1, df_daily_counts_cv2\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4af5d20d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:26:28.948161Z",
     "iopub.status.busy": "2023-03-17T22:26:28.947445Z",
     "iopub.status.idle": "2023-03-17T22:28:25.293026Z",
     "shell.execute_reply": "2023-03-17T22:28:25.285814Z"
    },
    "papermill": {
     "duration": 116.380033,
     "end_time": "2023-03-17T22:28:25.315342",
     "exception": false,
     "start_time": "2023-03-17T22:26:28.935309",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Calculating return rate - how often users who viewed this aid view it again.\n",
    "df_return_rate = count_returns(click_data_cv)\n",
    "df_return_rate.to_parquet('return_rate_cv.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eebb7839",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:28:25.474662Z",
     "iopub.status.busy": "2023-03-17T22:28:25.474243Z",
     "iopub.status.idle": "2023-03-17T22:28:25.814527Z",
     "shell.execute_reply": "2023-03-17T22:28:25.806912Z"
    },
    "papermill": {
     "duration": 0.42725,
     "end_time": "2023-03-17T22:28:25.831217",
     "exception": false,
     "start_time": "2023-03-17T22:28:25.403967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_return_rate\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f7d22dc7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:28:25.975728Z",
     "iopub.status.busy": "2023-03-17T22:28:25.970991Z",
     "iopub.status.idle": "2023-03-17T22:33:43.372113Z",
     "shell.execute_reply": "2023-03-17T22:33:43.370821Z"
    },
    "papermill": {
     "duration": 317.490014,
     "end_time": "2023-03-17T22:33:43.386515",
     "exception": false,
     "start_time": "2023-03-17T22:28:25.896501",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "merge_successfull\n",
      "cv_median_time_ready\n"
     ]
    }
   ],
   "source": [
    "# Calculating amount of time each aid was viewed.\n",
    "df_time_viewed_agg = median_time_clicked(click_data_cv)\n",
    "df_time_viewed_agg.to_parquet('time_viewed_agg_cv.parquet')\n",
    "print('cv_median_time_ready')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "134ec2e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:33:43.411051Z",
     "iopub.status.busy": "2023-03-17T22:33:43.410608Z",
     "iopub.status.idle": "2023-03-17T22:33:43.519028Z",
     "shell.execute_reply": "2023-03-17T22:33:43.517961Z"
    },
    "papermill": {
     "duration": 0.123708,
     "end_time": "2023-03-17T22:33:43.521529",
     "exception": false,
     "start_time": "2023-03-17T22:33:43.397821",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_time_viewed_agg\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9bdbcff",
   "metadata": {
    "papermill": {
     "duration": 0.011194,
     "end_time": "2023-03-17T22:33:43.544414",
     "exception": false,
     "start_time": "2023-03-17T22:33:43.533220",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Co-visitation matrice and other counts for test dataset\n",
    "Repeat exactly the same steps, but for the test dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "10626ae0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:33:43.569114Z",
     "iopub.status.busy": "2023-03-17T22:33:43.568684Z",
     "iopub.status.idle": "2023-03-17T22:33:46.639536Z",
     "shell.execute_reply": "2023-03-17T22:33:46.638004Z"
    },
    "papermill": {
     "duration": 3.086772,
     "end_time": "2023-03-17T22:33:46.642592",
     "exception": false,
     "start_time": "2023-03-17T22:33:43.555820",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trunked_sessions_path = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "aid_list =  otto_common.build_aid_list(trunked_sessions_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a65be683",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:33:46.667808Z",
     "iopub.status.busy": "2023-03-17T22:33:46.667379Z",
     "iopub.status.idle": "2023-03-17T22:55:58.725414Z",
     "shell.execute_reply": "2023-03-17T22:55:58.724197Z"
    },
    "papermill": {
     "duration": 1332.074317,
     "end_time": "2023-03-17T22:55:58.728641",
     "exception": false,
     "start_time": "2023-03-17T22:33:46.654324",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "start normalizing\n"
     ]
    }
   ],
   "source": [
    "click_data_test = '/kaggle/input/otto-prepare-cv/train_full.parquet'\n",
    "n_splits = 120\n",
    "n_splits_concat = 5\n",
    "timebase_test = 1659304800\n",
    "\n",
    "covisitation_test = CalculateCovisitationMatrixExactNext(n_splits, n_splits_concat, aid_list, timebase=timebase_test)\n",
    "df_matrix_test = covisitation_test.generate_covisitation_matrix(click_data_test)\n",
    "df_matrix_test = normalize_matrice(df_matrix_test, 'wgt')\n",
    "df_matrix_test.to_parquet('matrix_exact_next_counts_full.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d88f6287",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:55:58.769162Z",
     "iopub.status.busy": "2023-03-17T22:55:58.768117Z",
     "iopub.status.idle": "2023-03-17T22:55:58.912292Z",
     "shell.execute_reply": "2023-03-17T22:55:58.911290Z"
    },
    "papermill": {
     "duration": 0.167227,
     "end_time": "2023-03-17T22:55:58.914818",
     "exception": false,
     "start_time": "2023-03-17T22:55:58.747591",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_matrix_test, aid_list, covisitation_test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "817e5270",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T22:55:58.954838Z",
     "iopub.status.busy": "2023-03-17T22:55:58.954165Z",
     "iopub.status.idle": "2023-03-17T23:00:00.826793Z",
     "shell.execute_reply": "2023-03-17T23:00:00.825449Z"
    },
    "papermill": {
     "duration": 241.896349,
     "end_time": "2023-03-17T23:00:00.829988",
     "exception": false,
     "start_time": "2023-03-17T22:55:58.933639",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_daily_counts_test_full = otto_common.create_average_daily_counts(click_data_test, False)\n",
    "df_daily_counts_test_full.to_parquet('daily_counts_test_full.parquet')\n",
    "\n",
    "df_daily_counts_test_trunked = otto_common.create_average_daily_counts(trunked_sessions_path, True)\n",
    "df_daily_counts_test_trunked.to_parquet('daily_counts_test_trunked.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "35f10977",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T23:00:00.870982Z",
     "iopub.status.busy": "2023-03-17T23:00:00.870547Z",
     "iopub.status.idle": "2023-03-17T23:00:00.995172Z",
     "shell.execute_reply": "2023-03-17T23:00:00.993956Z"
    },
    "papermill": {
     "duration": 0.147749,
     "end_time": "2023-03-17T23:00:00.997764",
     "exception": false,
     "start_time": "2023-03-17T23:00:00.850015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_daily_counts_test_full, df_daily_counts_test_trunked\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2bd5f909",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T23:00:01.041424Z",
     "iopub.status.busy": "2023-03-17T23:00:01.040699Z",
     "iopub.status.idle": "2023-03-17T23:02:47.505335Z",
     "shell.execute_reply": "2023-03-17T23:02:47.504272Z"
    },
    "papermill": {
     "duration": 166.488841,
     "end_time": "2023-03-17T23:02:47.508593",
     "exception": false,
     "start_time": "2023-03-17T23:00:01.019752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_return_rate = count_returns(click_data_test)\n",
    "df_return_rate.to_parquet('return_rate_test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cb167c5e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T23:02:47.549086Z",
     "iopub.status.busy": "2023-03-17T23:02:47.548127Z",
     "iopub.status.idle": "2023-03-17T23:02:47.651348Z",
     "shell.execute_reply": "2023-03-17T23:02:47.650250Z"
    },
    "papermill": {
     "duration": 0.1257,
     "end_time": "2023-03-17T23:02:47.653816",
     "exception": false,
     "start_time": "2023-03-17T23:02:47.528116",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df_return_rate\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "81e6a4b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-17T23:02:47.693549Z",
     "iopub.status.busy": "2023-03-17T23:02:47.693126Z",
     "iopub.status.idle": "2023-03-17T23:09:16.348804Z",
     "shell.execute_reply": "2023-03-17T23:09:16.347847Z"
    },
    "papermill": {
     "duration": 388.678713,
     "end_time": "2023-03-17T23:09:16.351655",
     "exception": false,
     "start_time": "2023-03-17T23:02:47.672942",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "merge_successfull\n"
     ]
    }
   ],
   "source": [
    "df_time_viewed_agg = median_time_clicked(click_data_test)\n",
    "df_time_viewed_agg.to_parquet('time_viewed_agg_test.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3667.375282,
   "end_time": "2023-03-17T23:09:17.107300",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-17T22:08:09.732018",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}