otto-generate-candidates-orders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c5d3727f",
   "metadata": {
    "papermill": {
     "duration": 0.004337,
     "end_time": "2023-03-18T13:00:53.761471",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.757134",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Generate candidates for orders\n",
    "This notebook uses the click2buy matrix to generate candidates. It produces 4 outputs: candidates for the first cross-validation set (used to train the re-ranking model), candidates for the second cross-validation dataset (used to train the re-ranking model), top 75 candidates for the test dataset (used as test dataset for re-ranking model to produce the final results) and top 20 candidates for the test dataset (used to check the results on leaderboard).\n",
    "\n",
    "Until the very end of competition, carts model used 50 candidates, and all the constants for 50 candidates are still there. But the final submission pipeline only uses 75 candidates.\n",
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "69a533cb",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.770455Z",
     "iopub.status.busy": "2023-03-18T13:00:53.769813Z",
     "iopub.status.idle": "2023-03-18T13:00:53.813689Z",
     "shell.execute_reply": "2023-03-18T13:00:53.812643Z"
    },
    "papermill": {
     "duration": 0.051608,
     "end_time": "2023-03-18T13:00:53.816389",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.764781",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "        \n",
    "import gc\n",
    "from datetime import datetime\n",
    "from humanize import naturalsize\n",
    "import itertools\n",
    "from collections import Counter\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9c259357",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.824258Z",
     "iopub.status.busy": "2023-03-18T13:00:53.823884Z",
     "iopub.status.idle": "2023-03-18T13:00:53.839639Z",
     "shell.execute_reply": "2023-03-18T13:00:53.838445Z"
    },
    "papermill": {
     "duration": 0.022039,
     "end_time": "2023-03-18T13:00:53.841832",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.819793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def suggest_buys(df, n_candidates, top_dict, click2buy_dict):\n",
    "    # Set some constants depending on number of candidates.\n",
    "    # n_candidates should be 20, 50 or 75\n",
    "    if n_candidates == 20:\n",
    "        aid_length = 18\n",
    "        max_from_buys = 5\n",
    "    if n_candidates == 50:\n",
    "        aid_length = 32\n",
    "        max_from_buys = 10\n",
    "    if n_candidates == 75:\n",
    "        aid_length = 35\n",
    "        max_from_buys = 12\n",
    "        \n",
    "    # Select all last AIDs and some other info from dataframe.\n",
    "    aids=df.aid.tolist()\n",
    "    unique_aids = list(dict.fromkeys(aids[::-1] ))\n",
    "    day_of_week = max(df.day_of_week.tolist())\n",
    "    df = df.loc[(df['type']==1)|(df['type']==2)]\n",
    "    unique_buys = list(dict.fromkeys( df.aid.tolist()[::-1] ))\n",
    "\n",
    "    # Use AIDs, suggested from buys.\n",
    "    if len(unique_buys) > 0:\n",
    "        root_aids = unique_aids[:max_from_buys]\n",
    "        result = unique_buys[:aid_length]\n",
    "        for aid in unique_aids[:aid_length]:\n",
    "            if aid not in result:\n",
    "                result.append(aid)\n",
    "        if len(result) >= n_candidates:\n",
    "            return result[:n_candidates]\n",
    "        dict_limit = min(5, (n_candidates - len(result)))\n",
    "        if dict_limit > 0:\n",
    "            aids2 = list(itertools.chain(*[click2buy_dict[aid][:dict_limit] for aid in unique_buys if\n",
    "                                           aid in click2buy_dict]))\n",
    "            top_aids2 = [aid2 for aid2, cnt in Counter(aids2).most_common(n_candidates) if aid2 not in result]\n",
    "        else:\n",
    "            top_aids2 = []\n",
    "    else:\n",
    "        root_aids = unique_aids[:n_candidates]\n",
    "        top_aids2 = []\n",
    "        result = unique_aids[:aid_length]\n",
    "    \n",
    "    # Select AIDs, suggested from all the last clicks.\n",
    "    n_left = n_candidates - len(result) - len(unique_aids[:aid_length])\n",
    "    aids3 = list(itertools.chain(*[click2buy_dict[aid][:n_left] for aid in root_aids if aid in click2buy_dict]))\n",
    "    top_aids3 = [aid3 for aid3, cnt in Counter(aids3).most_common(n_candidates) if aid3 not in result]\n",
    "    \n",
    "    # Check for duplicates and merge all the previously selected AIDs.\n",
    "    suggested_aids = top_aids2[:max_from_buys] + top_aids3[:n_candidates]\n",
    "    i = 0\n",
    "    while (len(result) < n_candidates) & (i < len(suggested_aids)):\n",
    "        if suggested_aids[i] not in result:\n",
    "            result.append(suggested_aids[i])\n",
    "        i+=1\n",
    "        \n",
    "    # Add items from day top, if free slots.\n",
    "    if (n_candidates-len(result)) > 0:\n",
    "        i = 0\n",
    "        days_top = top_dict[day_of_week]\n",
    "        while len(result) < n_candidates:\n",
    "            if days_top[i] not in result:\n",
    "                result.append(days_top[i])\n",
    "            i+=1\n",
    "    return result   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "53f8c319",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.849509Z",
     "iopub.status.busy": "2023-03-18T13:00:53.849140Z",
     "iopub.status.idle": "2023-03-18T13:00:53.860244Z",
     "shell.execute_reply": "2023-03-18T13:00:53.859263Z"
    },
    "papermill": {
     "duration": 0.017661,
     "end_time": "2023-03-18T13:00:53.862521",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.844860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Prepare all the data for candidate generation.\n",
    "\n",
    "def generate_order_candidates(click2buy_matrix_path, sessions_path, n_candidates, answers_path=None):\n",
    "    # Load covisitation matrix and transform it to dictionary (to speed up candidate generation)\n",
    "    click2buy_dict = otto_common.matrix_to_dict(click2buy_matrix_path, n_candidates)\n",
    "    \n",
    "    # Add a weekday column and calculate top orders for each day\n",
    "    df = pd.read_parquet(sessions_path)\n",
    "    df = otto_common.add_datetime(df)\n",
    "    df['day_of_week'] = df['time'].dt.dayofweek.astype(np.int8)\n",
    "    \n",
    "    del df['time']\n",
    "    gc.collect()\n",
    "    \n",
    "    top_dict = otto_common.build_top_dict(df, n_candidates, 2)\n",
    "    \n",
    "    # In case we generate candidates for cross-validation, reduce inputs to those that have positive results as ground truth.\n",
    "    # Also prepare some data for statistics.\n",
    "    if answers_path:\n",
    "        df, df_answers = otto_common.reduce_df_prepare_answers(df, answers_path, 'orders')\n",
    "        \n",
    "        unique_session_aids = df.groupby([\"session\"]).aid.nunique()\n",
    "        unique_session_aids = pd.DataFrame({'session':unique_session_aids.index,\n",
    "                                            'all_aids':unique_session_aids.values})\n",
    "        unique_session_buys = df.loc[df['type'] > 0].groupby([\"session\"]).aid.nunique()\n",
    "        unique_session_buys = pd.DataFrame({'session':unique_session_buys.index,\n",
    "                                            'buys':unique_session_buys.values})\n",
    "        \n",
    "    # The candidate generation itself.\n",
    "    pred_df_clicks = df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n",
    "        lambda x: suggest_buys(x, n_candidates, top_dict, click2buy_dict)\n",
    "        )\n",
    "    \n",
    "    del df\n",
    "    gc.collect()\n",
    "    \n",
    "    pred_df_clicks = pd.DataFrame({'session':pred_df_clicks.index, 'order_predictions':pred_df_clicks.values})\n",
    " \n",
    "    # If generating candidates for a cross-validation set, use ground truth to calculate some statistics.\n",
    "    # Else just remove columns we do not need anymore.\n",
    "    if answers_path:\n",
    "        pred_df_clicks = otto_common.calculate_stats(pred_df_clicks, df_answers, unique_session_aids,\n",
    "                                         unique_session_buys, 'orders', n_candidates)\n",
    "        #pred_df_clicks = calculate_stats(pred_df_clicks, df_answers, unique_session_aids, unique_session_buys,\n",
    "        #                                 'orders', n_candidates)\n",
    "    else:\n",
    "        pred_df_clicks = pred_df_clicks[['session', 'order_predictions']]\n",
    "    \n",
    "    return pred_df_clicks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "12a2c3b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.870380Z",
     "iopub.status.busy": "2023-03-18T13:00:53.870011Z",
     "iopub.status.idle": "2023-03-18T13:00:53.875354Z",
     "shell.execute_reply": "2023-03-18T13:00:53.874293Z"
    },
    "papermill": {
     "duration": 0.011865,
     "end_time": "2023-03-18T13:00:53.877615",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.865750",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# All the paths.\n",
    "# Tried to use all the co-visitation matrixes, but final version only uses \"regular\" matrix.\n",
    "\n",
    "click2buy_matrix_path = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_cv_top_n.parquet'\n",
    "click2buy_full_matrix_path = '/kaggle/input/otto-prepare-candidates-buys/matrix_click2buy_test_top_n.parquet'\n",
    "\n",
    "sessions_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_inputs.parquet'\n",
    "sessions_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "\n",
    "answers_path_cv1 = '/kaggle/input/otto-prepare-cv/cv_labels.parquet'\n",
    "answers_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_labels2.parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8d1a182a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.884930Z",
     "iopub.status.busy": "2023-03-18T13:00:53.884615Z",
     "iopub.status.idle": "2023-03-18T13:00:53.890698Z",
     "shell.execute_reply": "2023-03-18T13:00:53.889700Z"
    },
    "papermill": {
     "duration": 0.013243,
     "end_time": "2023-03-18T13:00:53.893973",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.880730",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n",
      "Wall time: 7.15 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# This cell was used for tests and checks.\n",
    "\n",
    "#n_candidates = 75\n",
    "\n",
    "#df_result =  generate_order_candidates(click2buy_matrix_path, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "#df_result =  generate_order_candidates(click2buy_matrix_path, sessions_path_cv2, n_candidates, answers_path_cv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca1350f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:00:53.902357Z",
     "iopub.status.busy": "2023-03-18T13:00:53.901637Z",
     "iopub.status.idle": "2023-03-18T13:04:49.832717Z",
     "shell.execute_reply": "2023-03-18T13:04:49.831421Z"
    },
    "papermill": {
     "duration": 235.938745,
     "end_time": "2023-03-18T13:04:49.836002",
     "exception": false,
     "start_time": "2023-03-18T13:00:53.897257",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total orders:  311302\n",
      "Total orders clipped:  311302\n",
      "Total orders guessed:  214629\n",
      "Total orders with buys in history:  227882\n",
      "Orders with buys in history guessed:  166746\n",
      "Total orders with no buys in history:  83420\n",
      "Orders with no buys in history guessed:  47883\n",
      "Total:  68.95%\n",
      "Buys:  73.17%\n",
      "No buys:  57.40%\n",
      "CPU times: user 3min 46s, sys: 7.32 s, total: 3min 54s\n",
      "Wall time: 3min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the first cross-validation.\n",
    "\n",
    "n_candidates = 75\n",
    "\n",
    "df_result =  generate_order_candidates(click2buy_matrix_path, sessions_path_cv1, n_candidates, answers_path_cv1)\n",
    "df_result.to_parquet('candidates_order_cv1.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7c5a915d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:04:49.845083Z",
     "iopub.status.busy": "2023-03-18T13:04:49.844709Z",
     "iopub.status.idle": "2023-03-18T13:08:45.031147Z",
     "shell.execute_reply": "2023-03-18T13:08:45.030265Z"
    },
    "papermill": {
     "duration": 235.194325,
     "end_time": "2023-03-18T13:08:45.034231",
     "exception": false,
     "start_time": "2023-03-18T13:04:49.839906",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total orders:  311762\n",
      "Total orders clipped:  311762\n",
      "Total orders guessed:  214972\n",
      "Total orders with buys in history:  228320\n",
      "Orders with buys in history guessed:  167111\n",
      "Total orders with no buys in history:  83442\n",
      "Orders with no buys in history guessed:  47861\n",
      "Total:  68.95%\n",
      "Buys:  73.19%\n",
      "No buys:  57.36%\n",
      "CPU times: user 3min 49s, sys: 5.96 s, total: 3min 55s\n",
      "Wall time: 3min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the second cross-validation.\n",
    "\n",
    "df_result =  generate_order_candidates(click2buy_matrix_path, sessions_path_cv2, n_candidates, answers_path_cv2)\n",
    "df_result.to_parquet('candidates_order_cv2.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6eca1e38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:08:45.044705Z",
     "iopub.status.busy": "2023-03-18T13:08:45.044097Z",
     "iopub.status.idle": "2023-03-18T13:38:41.717969Z",
     "shell.execute_reply": "2023-03-18T13:38:41.715708Z"
    },
    "papermill": {
     "duration": 1796.687362,
     "end_time": "2023-03-18T13:38:41.726166",
     "exception": false,
     "start_time": "2023-03-18T13:08:45.038804",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max candidates test ready\n",
      "CPU times: user 29min 30s, sys: 29.8 s, total: 30min\n",
      "Wall time: 29min 56s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate candidates for the test dataset.\n",
    "\n",
    "df_result =  generate_order_candidates(click2buy_full_matrix_path, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_order_test.parquet')\n",
    "print('max candidates test ready')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "38c7b24f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-18T13:38:41.740396Z",
     "iopub.status.busy": "2023-03-18T13:38:41.739538Z",
     "iopub.status.idle": "2023-03-18T14:04:17.755533Z",
     "shell.execute_reply": "2023-03-18T14:04:17.754321Z"
    },
    "papermill": {
     "duration": 1536.029623,
     "end_time": "2023-03-18T14:04:17.761280",
     "exception": false,
     "start_time": "2023-03-18T13:38:41.731657",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 25min 24s, sys: 20.2 s, total: 25min 45s\n",
      "Wall time: 25min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Generate 20 candidates for the test dataset (to check on leaderboard).\n",
    "\n",
    "n_candidates = 20\n",
    "df_result =  generate_order_candidates(click2buy_full_matrix_path, sessions_path_test, n_candidates)\n",
    "df_result.to_parquet('candidates_order_test20.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3816.352336,
   "end_time": "2023-03-18T14:04:20.694359",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-18T13:00:44.342023",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}