otto-orders-w2vec-part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4aaa6822",
   "metadata": {
    "papermill": {
     "duration": 0.00463,
     "end_time": "2023-03-21T12:14:38.302139",
     "exception": false,
     "start_time": "2023-03-21T12:14:38.297509",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# W2vec features for orders (cross-validation2 and part_1 of test)\n",
    "\n",
    "In this notebook, the w2vec features for the orders model are built using w2vec model from \"W2vec model for carts and orders\" notebook. There are total 4 w2vec features used by the orders model, two of them are calculated for the last 5 session aids and candidate, another two - for last 20 session aids and candidate. Each group of features is calculated in a single cycle and also using pandarallel library, but anyway this calculation takes time. To decrease the calculation time even further, the task is performed in two notebooks in parallel - this one, and another one with very similar code that calculates the same features for another cross-validation dataset and for another chunk of test dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6d98fd2b",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:38.311798Z",
     "iopub.status.busy": "2023-03-21T12:14:38.311249Z",
     "iopub.status.idle": "2023-03-21T12:14:39.832679Z",
     "shell.execute_reply": "2023-03-21T12:14:39.831377Z"
    },
    "papermill": {
     "duration": 1.53011,
     "end_time": "2023-03-21T12:14:39.836060",
     "exception": false,
     "start_time": "2023-03-21T12:14:38.305950",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import gc\n",
    "from humanize import naturalsize\n",
    "from gensim.models import Word2Vec\n",
    "from pandarallel import pandarallel\n",
    "\n",
    "# functions and classes common for several notebooks of current project\n",
    "import otto_common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6458e50a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:39.846000Z",
     "iopub.status.busy": "2023-03-21T12:14:39.845104Z",
     "iopub.status.idle": "2023-03-21T12:14:39.852413Z",
     "shell.execute_reply": "2023-03-21T12:14:39.851393Z"
    },
    "papermill": {
     "duration": 0.015268,
     "end_time": "2023-03-21T12:14:39.855023",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.839755",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns min and max w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_min_max(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.order_predictions))\n",
    "    return [np.min(similarities), np.max(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "613a3498",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:39.864464Z",
     "iopub.status.busy": "2023-03-21T12:14:39.863959Z",
     "iopub.status.idle": "2023-03-21T12:14:39.870809Z",
     "shell.execute_reply": "2023-03-21T12:14:39.869511Z"
    },
    "papermill": {
     "duration": 0.014856,
     "end_time": "2023-03-21T12:14:39.873517",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.858661",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Returns mean and min w2vec similarity for the candidate aid and last session aids.\n",
    "def agg_mean_min(x, w2v_model):\n",
    "    similarities = []\n",
    "    for item in x.aid:\n",
    "        similarities.append(w2v_model.wv.similarity(item, x.order_predictions))\n",
    "    return [np.mean(similarities), np.min(similarities)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "808c5dff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:39.883393Z",
     "iopub.status.busy": "2023-03-21T12:14:39.882906Z",
     "iopub.status.idle": "2023-03-21T12:14:39.898096Z",
     "shell.execute_reply": "2023-03-21T12:14:39.896770Z"
    },
    "papermill": {
     "duration": 0.02349,
     "end_time": "2023-03-21T12:14:39.900986",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.877496",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import candidates and last N aids for each session, then calculate the w2vec features for them in chunks.\n",
    "def calculate_w2vec_features(train_path, sessions_path, w2v_model, time_delta, n_max, feature_type, feature_name1, feature_name2):\n",
    "    \n",
    "    # Import all the required data and merge it together.\n",
    "    print(feature_type)\n",
    "    df_sessions = pd.read_parquet(sessions_path)\n",
    "    df_sessions = otto_common.filter_by_time_and_n_max(df_sessions, time_delta, n_max)\n",
    "    df_sessions = (df_sessions.groupby('session').agg({'aid': lambda x: x.tolist()}))\n",
    "    df = pd.read_parquet(train_path)\n",
    "    df = pd.merge(df, df_sessions, how='left', on='session')\n",
    "    \n",
    "    # Remove all the columns not relevant to this calculation.\n",
    "    df = df[['order_predictions', 'aid']]\n",
    "    del df_sessions\n",
    "    gc.collect()\n",
    "    \n",
    "    # Prepare data chunks.\n",
    "    i = 0\n",
    "    chunk_size = 10000000\n",
    "    while i < 1000000000:\n",
    "        df_chunk = df.iloc[i:i+chunk_size,:].copy()\n",
    "        \n",
    "        # Calculate the features.\n",
    "        #pandarallel.initialize(nb_workers=4, progress_bar=True)\n",
    "        pandarallel.initialize(nb_workers=4)\n",
    "        if feature_type == 'mean_min':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_mean_min(x, w2v_model), axis=1)\n",
    "        elif feature_type == 'min_max':\n",
    "            df_chunk['features']  = df_chunk.parallel_apply(\n",
    "                lambda x: agg_min_max(x, w2v_model), axis=1)\n",
    "        else:\n",
    "            print('feature_type_unknown')\n",
    "            \n",
    "        # Format the calculated features.\n",
    "        df_chunk[[feature_name1,feature_name2]] = pd.DataFrame(df_chunk.features.tolist(), index=df_chunk.index)\n",
    "        df_chunk = df_chunk[[feature_name1, feature_name2]]\n",
    "        df_chunk[feature_name1] = df_chunk[feature_name1].astype(np.float32)\n",
    "        df_chunk[feature_name2] = df_chunk[feature_name2].astype(np.float32)\n",
    "        gc.collect()\n",
    "        \n",
    "        # Merge the chunks together.\n",
    "        if i == 0:\n",
    "            df_all = df_chunk.copy()\n",
    "        else:\n",
    "            df_all = pd.concat([df_all, df_chunk])\n",
    "        print(i)\n",
    "        i += chunk_size\n",
    "        gc.collect()\n",
    "        if df_chunk.shape[0] < chunk_size:\n",
    "            return df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "db7fc1ab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:39.911930Z",
     "iopub.status.busy": "2023-03-21T12:14:39.910688Z",
     "iopub.status.idle": "2023-03-21T12:14:39.919197Z",
     "shell.execute_reply": "2023-03-21T12:14:39.917759Z"
    },
    "papermill": {
     "duration": 0.017038,
     "end_time": "2023-03-21T12:14:39.922248",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.905210",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define all the parameters for the w2vec features, except for paths, that are different for cross-validation and test datasets.\n",
    "def add_w2vec_data(train_path, sessions_path, w2vec_path):\n",
    "    w2v_model = Word2Vec.load(w2vec_path)\n",
    "    df_w2v_20 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 3 * 60 * 60, 20, 'mean_min', 'w2v_20_mean', 'w2v_20_min')  \n",
    "    gc.collect()\n",
    "    df_w2v_5 = calculate_w2vec_features(train_path, sessions_path, w2v_model, 5 * 24 * 60 * 60, 5, 'min_max', 'w2v_5_min', 'w2v_5_max')\n",
    "    gc.collect()\n",
    "    df_all = pd.read_parquet(train_path)\n",
    "    df_all = pd.concat([df_all, df_w2v_20, df_w2v_5], axis=1)\n",
    "    return df_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58dd1075",
   "metadata": {
    "papermill": {
     "duration": 0.003496,
     "end_time": "2023-03-21T12:14:39.929718",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.926222",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## W2vec features for the second cross-validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0a0920d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:14:39.940318Z",
     "iopub.status.busy": "2023-03-21T12:14:39.939434Z",
     "iopub.status.idle": "2023-03-21T12:50:34.723649Z",
     "shell.execute_reply": "2023-03-21T12:50:34.722021Z"
    },
    "papermill": {
     "duration": 2154.79371,
     "end_time": "2023-03-21T12:50:34.727512",
     "exception": false,
     "start_time": "2023-03-21T12:14:39.933802",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_min\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "min_max\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# All the paths and w2vec feature calculation.\n",
    "train_path_cv2 = '/kaggle/input/otto-feature-engineering-orders/cv2_features_order.parquet'\n",
    "sessions_path_cv2 = '/kaggle/input/otto-prepare-cv/cv_inputs2.parquet'\n",
    "w2vec_path_cv = '/kaggle/input/otto-word2vec-exp/word2vec_cv_exp.wordvectors'\n",
    "\n",
    "df_cv2 = add_w2vec_data(train_path_cv2, sessions_path_cv2, w2vec_path_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c4355f2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:50:34.739449Z",
     "iopub.status.busy": "2023-03-21T12:50:34.738864Z",
     "iopub.status.idle": "2023-03-21T12:50:42.167940Z",
     "shell.execute_reply": "2023-03-21T12:50:42.166460Z"
    },
    "papermill": {
     "duration": 7.439399,
     "end_time": "2023-03-21T12:50:42.171405",
     "exception": false,
     "start_time": "2023-03-21T12:50:34.732006",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 GB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check size and export to file.\n",
    "size = df_cv2.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "df_cv2.to_parquet('train_features_with_w2v_cv2.parquet')\n",
    "\n",
    "del df_cv2, train_path_cv2, sessions_path_cv2, w2vec_path_cv\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e50a366",
   "metadata": {
    "papermill": {
     "duration": 0.004747,
     "end_time": "2023-03-21T12:50:42.181199",
     "exception": false,
     "start_time": "2023-03-21T12:50:42.176452",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## W2vec features for Part_1 of test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "02a29b87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T12:50:42.192842Z",
     "iopub.status.busy": "2023-03-21T12:50:42.192317Z",
     "iopub.status.idle": "2023-03-21T15:22:21.975396Z",
     "shell.execute_reply": "2023-03-21T15:22:21.973813Z"
    },
    "papermill": {
     "duration": 9099.792955,
     "end_time": "2023-03-21T15:22:21.978822",
     "exception": false,
     "start_time": "2023-03-21T12:50:42.185867",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_min\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n",
      "min_max\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "0\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "10000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "20000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "30000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "40000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "50000000\n",
      "INFO: Pandarallel will run on 4 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n",
      "60000000\n"
     ]
    }
   ],
   "source": [
    "# All the paths and w2vec feature calculation.\n",
    "train_path_test_1 = '/kaggle/input/otto-feature-engineering-orders/test_features_order_part_1.parquet'\n",
    "sessions_path_test = '/kaggle/input/otto-prepare-cv/test.parquet'\n",
    "w2vec_path_test = '/kaggle/input/otto-word2vec-exp/word2vec_test_exp.wordvectors'\n",
    "\n",
    "df_test_1 = add_w2vec_data(train_path_test_1, sessions_path_test, w2vec_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1fd7c0b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-03-21T15:22:21.994059Z",
     "iopub.status.busy": "2023-03-21T15:22:21.993063Z",
     "iopub.status.idle": "2023-03-21T15:22:59.895356Z",
     "shell.execute_reply": "2023-03-21T15:22:59.894182Z"
    },
    "papermill": {
     "duration": 37.913172,
     "end_time": "2023-03-21T15:22:59.898355",
     "exception": false,
     "start_time": "2023-03-21T15:22:21.985183",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.5 GB\n"
     ]
    }
   ],
   "source": [
    "# Check size and export to file.\n",
    "size = df_test_1.memory_usage(deep='True').sum()\n",
    "print(naturalsize(size))\n",
    "\n",
    "df_test_1.to_parquet('train_features_with_w2v_part_1.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 11314.88722,
   "end_time": "2023-03-21T15:23:01.354904",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-03-21T12:14:26.467684",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}