notebooks/CLS-JA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/pczapla/workspace/ulmfit-multilingual\n"
     ]
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "from multifit.datasets import ULMFiTDataset, Dataset\n",
    "import multifit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\") # to ignore pytorch 1.3 warnings triggered by older fastai"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CLS JA music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#!python prepare_cls.py https://storage.googleapis.com/ulmfit/cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = multifit.from_pretrained('ja_multifit_paper_version')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ULMFiTArchitecture(tokenizer_type='sp', max_vocab=15000, lang='ja', emb_sz=400, n_hid=1550, n_layers=4, qrnn=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.arch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from /home/pczapla/.fastai/models/ja_multifit_paper_version to data/cls/ja-music/models/sp15k\n"
     ]
    }
   ],
   "source": [
    "cls_dataset = exp.arch.dataset(Path('data/cls/ja-music'), exp.pretrain_lm.tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ja'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_dataset.lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data tst, trn: 200, val: 1999\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>▁ xxbos ▁ xxfld ▁ 1 ▁一 音 が すべて 、 すべてが 一 音 。 &lt;unk&gt; し み を 包 んで く れる 宇 宙 その もの ▁ xxfld ▁ 2 ▁ xxup ▁ &lt;unk&gt; かな 世 界 が 、 cd を かけた と たん 拡 が る 。 「 打 波 」 だ 。 それは &lt;unk&gt; 気 と &lt;unk&gt; 発 力 をもって はじまり 、 美 しい 小 波 が</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 様 々 な バージョン 違 い や レア 音 源 を 聴 き 散 ら か した 上 で 聴 く と 、 どう にも &lt;unk&gt; 求 不 満 になる アルバム 。 イン スト 部 分 が 多 い のも 「 歌 」 を 充 分 聴 きた い 人 間 には 物 足 り ない かな 、 と 。 ▁ xxfld ▁</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁ xxbos ▁ xxfld ▁ 1 ▁ xxup ▁ 今 度 は 野 球 の 松 坂 ファンを ex ファンに させ ようと してる ▁ xxrep ▁4 ▁・ ▁ xxfld ▁ 2 ▁ xxup ▁第 2 章 になって すべての シングル 、 アルバムの dvd 付 き を 購 入 した 方 。 よく 考 え ましょう 。 ▁ xxup ▁ cd 収 録 の ▁ xxup ▁ w on ▁ ' ▁</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 粒 &lt;unk&gt; い の light な d ance ▁number が &lt;unk&gt; 能 出 来 る b est 盤 ▁ xxfld ▁ 2 ▁ sky y は 80 年 代 を pe ak に 大 活 躍 した new - y or k のお &lt;unk&gt; 落 な fu nk ▁ xxmaj ▁ band である 。 ▁ただ 、 fu nk ▁ band とは 言 え</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>▁ xxbos ▁ xxfld ▁ 1 ▁ 少 々 物 足 り ない 内 容 ▁ xxfld ▁ 2 ▁ レイ ・ デイヴィス 自 身 が 監 修 した キン クス 待 望 の ボックス ・ セット の 登 場 です 。 p ye 〜 rca 〜 ari sta 〜 lon don 〜 co lum bia 〜 ko nk の 全 時 代 を 網 羅 した 6 枚 組</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cls_dataset.load_clas_databunch(bs=exp.finetune_lm.bs).show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting LM weights seed seed to 0\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Training args:  {'drop_mult': 0.3, 'true_wd': False, 'wd': 1e-07, 'pretrained': False, 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15, 'tie_weights': True, 'out_bias': True}\n",
      "Setting LM training seed seed to 0\n",
      "Loading pretrained weights:  [PosixPath('/home/pczapla/.fastai/models/ja_multifit_paper_version/lm_best'), PosixPath('/home/pczapla/.fastai/models/ja_multifit_paper_version/itos')]\n",
      "Experiment data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "Fitting using 2 cycle fit schedule\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 02:14 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.815562</td>\n",
       "      <td>5.125886</td>\n",
       "      <td>0.214914</td>\n",
       "      <td>02:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Total time: 1:04:16 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.962217</td>\n",
       "      <td>4.831137</td>\n",
       "      <td>0.238743</td>\n",
       "      <td>03:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.559075</td>\n",
       "      <td>4.432836</td>\n",
       "      <td>0.278943</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.131804</td>\n",
       "      <td>4.128801</td>\n",
       "      <td>0.312514</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.960420</td>\n",
       "      <td>3.925446</td>\n",
       "      <td>0.332600</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.833594</td>\n",
       "      <td>3.798564</td>\n",
       "      <td>0.346086</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.664375</td>\n",
       "      <td>3.708547</td>\n",
       "      <td>0.354543</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.592303</td>\n",
       "      <td>3.634840</td>\n",
       "      <td>0.362600</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.448824</td>\n",
       "      <td>3.587191</td>\n",
       "      <td>0.367714</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.456172</td>\n",
       "      <td>3.549420</td>\n",
       "      <td>0.374629</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.353690</td>\n",
       "      <td>3.518477</td>\n",
       "      <td>0.378029</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.329306</td>\n",
       "      <td>3.500824</td>\n",
       "      <td>0.381371</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>3.270015</td>\n",
       "      <td>3.491282</td>\n",
       "      <td>0.382200</td>\n",
       "      <td>03:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.166214</td>\n",
       "      <td>3.475223</td>\n",
       "      <td>0.384343</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.155823</td>\n",
       "      <td>3.467666</td>\n",
       "      <td>0.387829</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.136682</td>\n",
       "      <td>3.457926</td>\n",
       "      <td>0.389714</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.022880</td>\n",
       "      <td>3.463706</td>\n",
       "      <td>0.388857</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>3.058692</td>\n",
       "      <td>3.461311</td>\n",
       "      <td>0.389086</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>3.008492</td>\n",
       "      <td>3.461708</td>\n",
       "      <td>0.388857</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>2.982608</td>\n",
       "      <td>3.462014</td>\n",
       "      <td>0.389171</td>\n",
       "      <td>03:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>2.988810</td>\n",
       "      <td>3.461781</td>\n",
       "      <td>0.389457</td>\n",
       "      <td>03:13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from /home/pczapla/.fastai/models/ja_multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "Language model saved to data/cls/ja-music/models/sp15k/multifit_paper_version\n"
     ]
    }
   ],
   "source": [
    "exp.finetune_lm.train_(cls_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing bs '18' with '20\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing early_stopping 'None' with 'accuracy\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ULMFiTFinetuning(seed=0, name='multifit_paper_version', experiment_path=PosixPath('data/cls/ja-music/models/sp15k/multifit_paper_version'), dataset_path=PosixPath('data/cls/ja-music'), num_epochs=20, bs=20, bptt=70, drop_mult=0.3, dropout_values={'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}, label_smoothing_eps=0.0, label_smoothing_eps_norm_by_classes=True, use_adam_08=False, true_wd=False, wd=1e-07, clip=0.12, fp16=False, lr=0.001)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.load_(cls_dataset.cache_path/exp.pretrain_lm.name).finetune_lm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 0\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 0\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_version\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.551692</td>\n",
       "      <td>0.544244</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.514899</td>\n",
       "      <td>0.617670</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.443367</td>\n",
       "      <td>0.596144</td>\n",
       "      <td>0.785000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.368091</td>\n",
       "      <td>0.567668</td>\n",
       "      <td>0.790000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.315757</td>\n",
       "      <td>0.635034</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.271595</td>\n",
       "      <td>0.515227</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.241717</td>\n",
       "      <td>0.523387</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.237796</td>\n",
       "      <td>0.518763</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.75.\n",
      "Better model found at epoch 2 with accuracy value: 0.7850000262260437.\n",
      "Better model found at epoch 3 with accuracy value: 0.7900000214576721.\n",
      "Better model found at epoch 4 with accuracy value: 0.8050000071525574.\n",
      "Better model found at epoch 5 with accuracy value: 0.8149999976158142.\n",
      "Better model found at epoch 6 with accuracy value: 0.8199999928474426.\n",
      "Better model found at epoch 7 with accuracy value: 0.824999988079071.\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_version/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 1\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 1\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.553418</td>\n",
       "      <td>0.493184</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.517754</td>\n",
       "      <td>0.603011</td>\n",
       "      <td>0.745000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.444649</td>\n",
       "      <td>0.451021</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.378163</td>\n",
       "      <td>0.488551</td>\n",
       "      <td>0.845000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.296332</td>\n",
       "      <td>0.722608</td>\n",
       "      <td>0.745000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.267031</td>\n",
       "      <td>0.528215</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.248884</td>\n",
       "      <td>0.522899</td>\n",
       "      <td>0.830000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.234614</td>\n",
       "      <td>0.512867</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.800000011920929.\n",
      "Better model found at epoch 2 with accuracy value: 0.8500000238418579.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed1/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 2\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 2\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.522099</td>\n",
       "      <td>0.587937</td>\n",
       "      <td>0.780000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.501093</td>\n",
       "      <td>0.749337</td>\n",
       "      <td>0.755000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.431629</td>\n",
       "      <td>0.610693</td>\n",
       "      <td>0.765000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.369956</td>\n",
       "      <td>0.518467</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.300080</td>\n",
       "      <td>0.621023</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.273065</td>\n",
       "      <td>0.548695</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.244343</td>\n",
       "      <td>0.521141</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.228544</td>\n",
       "      <td>0.508984</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.7799999713897705.\n",
      "Better model found at epoch 3 with accuracy value: 0.8199999928474426.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed2/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 3\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 3\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:36 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.540789</td>\n",
       "      <td>0.530250</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.494581</td>\n",
       "      <td>0.509151</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.446126</td>\n",
       "      <td>0.529051</td>\n",
       "      <td>0.785000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.376928</td>\n",
       "      <td>0.582771</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.322823</td>\n",
       "      <td>0.492915</td>\n",
       "      <td>0.835000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.270082</td>\n",
       "      <td>0.499838</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.249619</td>\n",
       "      <td>0.512006</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.240476</td>\n",
       "      <td>0.515156</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.7749999761581421.\n",
      "Better model found at epoch 1 with accuracy value: 0.8149999976158142.\n",
      "Better model found at epoch 4 with accuracy value: 0.8349999785423279.\n",
      "Better model found at epoch 5 with accuracy value: 0.8500000238418579.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed3/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 4\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 4\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.557735</td>\n",
       "      <td>0.474914</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.497810</td>\n",
       "      <td>0.519276</td>\n",
       "      <td>0.785000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.445226</td>\n",
       "      <td>0.632691</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.376329</td>\n",
       "      <td>0.523593</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.315562</td>\n",
       "      <td>0.491826</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.277676</td>\n",
       "      <td>0.531970</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.241653</td>\n",
       "      <td>0.491889</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.240878</td>\n",
       "      <td>0.505452</td>\n",
       "      <td>0.835000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.8050000071525574.\n",
      "Better model found at epoch 3 with accuracy value: 0.8100000023841858.\n",
      "Better model found at epoch 4 with accuracy value: 0.8149999976158142.\n",
      "Better model found at epoch 6 with accuracy value: 0.8399999737739563.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed4/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 5\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 5\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:35 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.539067</td>\n",
       "      <td>0.524193</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.504846</td>\n",
       "      <td>0.563058</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.455630</td>\n",
       "      <td>0.554939</td>\n",
       "      <td>0.785000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.366007</td>\n",
       "      <td>0.535870</td>\n",
       "      <td>0.810000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.302207</td>\n",
       "      <td>0.502417</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.274910</td>\n",
       "      <td>0.543057</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.257135</td>\n",
       "      <td>0.517289</td>\n",
       "      <td>0.815000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.234617</td>\n",
       "      <td>0.505733</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.8199999928474426.\n",
      "Better model found at epoch 4 with accuracy value: 0.824999988079071.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed5/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 6\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 6\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:38 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.529963</td>\n",
       "      <td>0.533260</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.501183</td>\n",
       "      <td>0.518192</td>\n",
       "      <td>0.785000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.429616</td>\n",
       "      <td>0.633020</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.382384</td>\n",
       "      <td>0.526554</td>\n",
       "      <td>0.830000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.315402</td>\n",
       "      <td>0.520821</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.274772</td>\n",
       "      <td>0.551267</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.249956</td>\n",
       "      <td>0.519960</td>\n",
       "      <td>0.830000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.226764</td>\n",
       "      <td>0.516419</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.75.\n",
      "Better model found at epoch 1 with accuracy value: 0.7850000262260437.\n",
      "Better model found at epoch 2 with accuracy value: 0.800000011920929.\n",
      "Better model found at epoch 3 with accuracy value: 0.8299999833106995.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed6/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k\n",
      "Data lm-notst, trn: 31800, val: 200\n",
      "Size of vocabulary: 15000\n",
      "First 20 words in vocab: ['xxunk', 'xxpad', 'xxbos', 'xxfld', 'xxmaj', 'xxup', 'xxrep', 'xxwrep', '<unk>', '▁', '▁、', '▁。', '▁の', '▁に', '▁を', '▁年', 'の', '▁は', '▁・', '▁)']\n",
      "Data cls, trn: 1800, val: 200\n",
      "Data tst, trn: 200, val: 1999\n",
      "Using Label smoothing with eps =  0.05\n",
      "Setting Classifier weights seed seed to 7\n",
      "Training args:  {'drop_mult': 0.5, 'wd': 0.1, 'pretrained': False, 'bptt': 70, 'loss_func': FlattenedLoss of LabelSmoothingCrossEntropy(), 'clip': 0.12} config:  {'emb_sz': 400, 'n_hid': 1550, 'n_layers': 4, 'pad_token': 1, 'qrnn': True, 'bidir': False, 'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "Loading pretrained model /home/pczapla/workspace/ulmfit-multilingual/data/cls/ja-music/models/sp15k/multifit_paper_version/enc_best\n",
      "Setting Classifier training seed seed to 7\n",
      "Training: data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 01:36 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.565436</td>\n",
       "      <td>0.517827</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.517789</td>\n",
       "      <td>0.710051</td>\n",
       "      <td>0.795000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.456400</td>\n",
       "      <td>0.502067</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.367990</td>\n",
       "      <td>0.560270</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.305195</td>\n",
       "      <td>0.585719</td>\n",
       "      <td>0.805000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.276869</td>\n",
       "      <td>0.496807</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>00:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.244716</td>\n",
       "      <td>0.502482</td>\n",
       "      <td>0.845000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.238059</td>\n",
       "      <td>0.496951</td>\n",
       "      <td>0.845000</td>\n",
       "      <td>00:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.8050000071525574.\n",
      "Better model found at epoch 5 with accuracy value: 0.8399999737739563.\n",
      "Better model found at epoch 6 with accuracy value: 0.8450000286102295.\n",
      "Copy sp model from data/cls/ja-music/models/sp15k/multifit_paper_version to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
      "Classifier model saved to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
      "Saving dump to data/cls/ja-music/models/sp15k/multifit_paper_versionseed7/classifier.json\n",
      "this Learner object self-destroyed - it still exists, but no longer usable\n"
     ]
    }
   ],
   "source": [
    "exp.classifier.train_(seed=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed5/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '5\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed5' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed5\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed4/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '4\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed4' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed4\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed3/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '3\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed3' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed3\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed1/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '1\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed1' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed1\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed2/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '2\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed2' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed2\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed6/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '6\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed6' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed6\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_versionseed7/classifier.json\n",
      "Loading data/cls/ja-music/models/sp15k/multifit_paper_version/finetuning.json\n",
      "ULMFiTFinetuning Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_version\n",
      "ULMFiTFinetuning Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTFinetuning Replacing num_epochs '10' with '20\n",
      "ULMFiTFinetuning Replacing drop_mult '1.0' with '0.3\n",
      "ULMFiTFinetuning Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTFinetuning Replacing true_wd 'True' with 'False\n",
      "ULMFiTFinetuning Replacing wd '0.01' with '1e-07\n",
      "ULMFiTFinetuning Replacing clip 'None' with '0.12\n",
      "ULMFiTArchitecture Replacing tokenizer_type 'f' with 'sp\n",
      "ULMFiTArchitecture Replacing max_vocab '60000' with '15000\n",
      "ULMFiTArchitecture Replacing lang 'None' with 'ja\n",
      "ULMFiTArchitecture Replacing n_hid '1150' with '1550\n",
      "ULMFiTArchitecture Replacing n_layers '3' with '4\n",
      "ULMFiTArchitecture Replacing qrnn 'False' with 'True\n",
      "ULMFiTClassifier Replacing seed '0' with '7\n",
      "ULMFiTClassifier Replacing name 'multifit_paper_versionseed7' with 'multifit_paper_version\n",
      "ULMFiTClassifier Replacing experiment_path 'None' with 'data/cls/ja-music/models/sp15k/multifit_paper_versionseed7\n",
      "ULMFiTClassifier Replacing dataset_path 'None' with 'data/cls/ja-music\n",
      "ULMFiTClassifier Replacing num_epochs '10' with '8\n",
      "ULMFiTClassifier Replacing dropout_values '{}' with '{'output_p': 0.25, 'hidden_p': 0.1, 'input_p': 0.2, 'embed_p': 0.02, 'weight_p': 0.15}\n",
      "ULMFiTClassifier Replacing wd '0.01' with '0.1\n",
      "ULMFiTClassifier Replacing clip 'None' with '0.12\n",
      "ULMFiTClassifier Replacing label_smoothing_eps '0.0' with '0.1\n",
      "ULMFiTClassifier Replacing label_smoothing_eps_norm_by_classes 'False' with 'True\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>seed</th>\n",
       "      <th>test accuracy</th>\n",
       "      <th>valid accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>2</td>\n",
       "      <td>0.848424</td>\n",
       "      <td>0.820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>5</td>\n",
       "      <td>0.851926</td>\n",
       "      <td>0.825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>6</td>\n",
       "      <td>0.842421</td>\n",
       "      <td>0.830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870435</td>\n",
       "      <td>0.840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>7</td>\n",
       "      <td>0.861431</td>\n",
       "      <td>0.845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>3</td>\n",
       "      <td>0.854427</td>\n",
       "      <td>0.850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multifit_paper_version</td>\n",
       "      <td>1</td>\n",
       "      <td>0.860930</td>\n",
       "      <td>0.850</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     name  seed  test accuracy  valid accuracy\n",
       "4  multifit_paper_version     2       0.848424           0.820\n",
       "0  multifit_paper_version     5       0.851926           0.825\n",
       "5  multifit_paper_version     6       0.842421           0.830\n",
       "1  multifit_paper_version     4       0.870435           0.840\n",
       "6  multifit_paper_version     7       0.861431           0.845\n",
       "2  multifit_paper_version     3       0.854427           0.850\n",
       "3  multifit_paper_version     1       0.860930           0.850"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_results(exp_path):\n",
    "    exp = multifit.ULMFiT().load_(exp_path, silent=False).classifier\n",
    "    results = exp.validate(use_cache=True) \n",
    "    results.update(seed=exp.seed, fp16=exp.fp16)\n",
    "    return results\n",
    "results = [get_results(exp_path) for exp_path in cls_dataset.cache_path.glob(exp.pretrain_lm.name+\"seed*\")]\n",
    "results_df = pd.DataFrame.from_records(results)\n",
    "results_df.sort_values([\"valid accuracy\"])[[\"name\", \"seed\", \"test accuracy\", \"valid accuracy\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}