notebooks/TextAnalytics - Amazon Book Reviews with Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## TextAnalytics - Amazon Book Reviews with Word2Vec\n",
    "\n",
    "Yet again, now using the `Word2Vec` Estimator from Spark.  We can use the tree-based\n",
    "learners from spark in this scenario due to the lower dimensionality representation of\n",
    "features."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import os\r\n",
    "if os.environ.get(\"AZURE_SERVICE\", None) == \"Microsoft.ProjectArcadia\":\r\n",
    "    from pyspark.sql import SparkSession\r\n",
    "    spark = SparkSession.builder.getOrCreate()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import pandas as pd\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "data = spark.read.parquet(\"wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet\")\r\n",
    "data.limit(10).toPandas()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Modify the label column to predict a rating greater than 3."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "processedData = data.withColumn(\"label\", data[\"rating\"] > 3) \\\r\n",
    "                    .select([\"text\", \"label\"])\r\n",
    "processedData.limit(5).toPandas()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Split the dataset into train, test and validation sets."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "train, test, validation = processedData.randomSplit([0.60, 0.20, 0.20])"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Use `Tokenizer` and `Word2Vec` to generate the features."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from pyspark.ml import Pipeline\r\n",
    "from pyspark.ml.feature import Tokenizer, Word2Vec\r\n",
    "tokenizer = Tokenizer(inputCol=\"text\", outputCol=\"words\")\r\n",
    "partitions = train.rdd.getNumPartitions()\r\n",
    "word2vec = Word2Vec(maxIter=4, seed=42, inputCol=\"words\", outputCol=\"features\",\r\n",
    "                    numPartitions=partitions)\r\n",
    "textFeaturizer = Pipeline(stages = [tokenizer, word2vec]).fit(train)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Transform each of the train, test and validation datasets."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "ptrain = textFeaturizer.transform(train).select([\"label\", \"features\"])\r\n",
    "ptest = textFeaturizer.transform(test).select([\"label\", \"features\"])\r\n",
    "pvalidation = textFeaturizer.transform(validation).select([\"label\", \"features\"])\r\n",
    "ptrain.limit(5).toPandas()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Generate several models with different parameters from the training data."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from pyspark.ml.classification import LogisticRegression, RandomForestClassifier, GBTClassifier\r\n",
    "from mmlspark.train import TrainClassifier\r\n",
    "import itertools\r\n",
    "\r\n",
    "lrHyperParams       = [0.05, 0.2]\r\n",
    "logisticRegressions = [LogisticRegression(regParam = hyperParam)\r\n",
    "                       for hyperParam in lrHyperParams]\r\n",
    "lrmodels            = [TrainClassifier(model=lrm, labelCol=\"label\").fit(ptrain)\r\n",
    "                       for lrm in logisticRegressions]\r\n",
    "\r\n",
    "rfHyperParams       = itertools.product([5, 10], [2, 3])\r\n",
    "randomForests       = [RandomForestClassifier(numTrees=hyperParam[0], maxDepth=hyperParam[1])\r\n",
    "                       for hyperParam in rfHyperParams]\r\n",
    "rfmodels            = [TrainClassifier(model=rfm, labelCol=\"label\").fit(ptrain)\r\n",
    "                       for rfm in randomForests]\r\n",
    "\r\n",
    "gbtHyperParams      = itertools.product([8, 16], [2, 3])\r\n",
    "gbtclassifiers      = [GBTClassifier(maxBins=hyperParam[0], maxDepth=hyperParam[1])\r\n",
    "                       for hyperParam in gbtHyperParams]\r\n",
    "gbtmodels           = [TrainClassifier(model=gbt, labelCol=\"label\").fit(ptrain)\r\n",
    "                       for gbt in gbtclassifiers]\r\n",
    "\r\n",
    "trainedModels       = lrmodels + rfmodels + gbtmodels"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Find the best model for the given test dataset."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from mmlspark.automl import FindBestModel\r\n",
    "bestModel = FindBestModel(evaluationMetric=\"AUC\", models=trainedModels).fit(ptest)\r\n",
    "bestModel.getRocCurve().show()\r\n",
    "bestModel.getBestModelMetrics().show()\r\n",
    "bestModel.getAllModelMetrics().show()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "Get the accuracy from the validation dataset."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from mmlspark.train import ComputeModelStatistics\r\n",
    "predictions = bestModel.transform(pvalidation)\r\n",
    "metrics = ComputeModelStatistics().transform(predictions)\r\n",
    "print(\"Best model's accuracy on validation set = \"\r\n",
    "      + \"{0:.2f}%\".format(metrics.first()[\"accuracy\"] * 100))\r\n",
    "print(\"Best model's AUC on validation set = \"\r\n",
    "      + \"{0:.2f}%\".format(metrics.first()[\"AUC\"] * 100))"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}