analyze_iron_maiden.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "403b0d96",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from spacy.tokens import DocBin, Doc, Span, Token\n",
    "from spacy.matcher import Matcher\n",
    "from spacy import Language\n",
    "import random\n",
    "from utils import string, punct, label_sent, label_many_sents, LabelHolder, my_sentencizer\n",
    "# from text_to_num import alpha2digit # python -m pip install text2num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "85c946f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('ner', <spacy.pipeline.ner.EntityRecognizer at 0x1a4f82c07b0>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp = spacy.load('en_core_web_md')\n",
    "nlp.remove_pipe('ner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "725d852d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tok2vec', <spacy.pipeline.tok2vec.Tok2Vec at 0x1a4f8457280>),\n",
       " ('tagger', <spacy.pipeline.tagger.Tagger at 0x1a4f8457f40>),\n",
       " ('parser', <spacy.pipeline.dep_parser.DependencyParser at 0x1a4f82c0970>),\n",
       " ('attribute_ruler',\n",
       "  <spacy.pipeline.attributeruler.AttributeRuler at 0x1a4f850d700>),\n",
       " ('lemmatizer', <spacy.lang.en.lemmatizer.EnglishLemmatizer at 0x1a4f851c100>)]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c90e6b1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron_labels = ['DEATH', 'TIME', 'LIGHT']\n",
    "iron_label_holder = LabelHolder(iron_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6008ae9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "matcher = Matcher(nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "561591ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://spacy.io/usage/rule-based-matching\n",
    "with open('death_words.txt') as f:\n",
    "    death_words = f.read().split()\n",
    "\n",
    "def on_death_match(matcher, doc, i, matches):\n",
    "    match_id, start, end = matches[i]\n",
    "    ent = Span(doc, start, end, label='DEATH')\n",
    "    try:\n",
    "        doc.ents += (ent,)\n",
    "    except ValueError:\n",
    "        # due to trying to add an entity where one already exists\n",
    "        pass\n",
    "\n",
    "matcher.add('DEATH_RULES', [\n",
    "        [{'LEMMA': 'die', 'POS': 'VERB'}], # because of \"die\" as in \"dice\"\n",
    "        [{'LEMMA': 'grave', 'POS': 'NOUN'}] # because of \"grave\" as in \"grave accusation\" \n",
    "    ] + \n",
    "    [[{'LEMMA': x}] for x in death_words], \n",
    "            on_match=on_death_match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "46e2516f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"time_words.txt\") as f:\n",
    "    time_words = f.read().split()\n",
    "\n",
    "def on_time_match(matcher, doc, i, matches):\n",
    "    match_id, start, end = matches[i]\n",
    "    ent = Span(doc, start, end, label='TIME')\n",
    "    try:\n",
    "        doc.ents += (ent,)\n",
    "    except ValueError:\n",
    "        pass\n",
    "    \n",
    "matcher.add('TIME_RULES', [\n",
    "        [{'LEMMA': x}] for x in time_words\n",
    "    ] + [[{'LEMMA': 'second', 'POS': 'NOUN'}]], # because of \"second\" as in \"I second that emotion\" or \"a second bite\" \n",
    "            on_match=on_time_match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d60e41e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('light_words.txt') as f:\n",
    "    light_words = f.read().split()\n",
    "\n",
    "def on_light_match(matcher, doc, i, matches):\n",
    "    match_id, start, end = matches[i]\n",
    "    ent = Span(doc, start, end, label='LIGHT')\n",
    "    try:\n",
    "        doc.ents += (ent,)\n",
    "    except ValueError:\n",
    "        pass\n",
    "    \n",
    "matcher.add('LIGHT_RULES', [\n",
    "    [{'LEMMA': x}] for x in light_words\n",
    "] + [[{'LEMMA': 'light', 'POS': 'NOUN'}],\n",
    "     [{'LEMMA': 'light', 'POS': 'VERB'}], # have to exclude 'light' when used as an adjective\n",
    "    ],\n",
    "on_match=on_light_match\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80cec90a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# @Language.component('light_dark_time')\n",
    "# def light_dark_time_match(doc):\n",
    "#     matcher(doc)\n",
    "#     return doc\n",
    "# nlp_special = spacy.load('en_core_web_md')\n",
    "# nlp_special.add_pipe('light_dark_time', after='lemmatizer')\n",
    "# nlp_special.remove_pipe('ner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "20c8a717",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('dead', 'DEATH'), ('dying', 'DEATH'), ('soul', 'DEATH'), ('dies', 'DEATH'), ('years', 'TIME'), ('ancient', 'TIME'), ('dark', 'LIGHT'), ('died', 'DEATH'), ('seconds', 'TIME'), ('hours', 'TIME'), ('light', 'LIGHT'), ('darkness', 'LIGHT'), ('night', 'LIGHT')]\n"
     ]
    }
   ],
   "source": [
    "ex = nlp('''He is dead, we're dying, roll a die, it's deadly, the soul dies.\n",
    "After hundreds of years, the ancient dark demon died in seconds.\n",
    "It took hours to climb its carcass.\n",
    "His heart was light, but once the light failed, buzzards swooped out of the darkness of night''')\n",
    "matcher(ex)\n",
    "print([(e.text, e.label_) for e in ex.ents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "beca82d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron = pd.read_csv('iron_maiden_songs.tsv', delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a604382c",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron = iron[iron.lyrics.apply(type) == str]\n",
    "# there are four pure-instrumental songs, so there's no lyrics listed\n",
    "# it wasn't a web-scraping error!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "116fe6e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron['lyrics'] = iron.lyrics.apply(\n",
    "    lambda x: my_sentencizer([x.split() for x in x.split('\\\\n')[:-1]])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5131848e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"One, two, three, four...  Hahaha! Oh! They even got the music to go with it, thatâ\\x80\\x99s lovely! I can't be compromising in my thoughts no more, ha ha, oh yeah, ah. I can't remember whats-er-name the name my anger fills my heart. I can't be sympathising with a new lost fart. Hahaha! Ohâ\\x80¦ Ohhh... I can\""
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iron[iron.album == 'No More Lies - Dance Of Death'].lyrics.iat[0][:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "711e5af8",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron = iron[iron.album != 'No More Lies - Dance Of Death']\n",
    "# that's a live show where Bruce Dickinson is babbling about some random stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15471b5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['album', 'year', 'num', 'title', 'lyrics'], dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iron.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "318dc836",
   "metadata": {},
   "outputs": [],
   "source": [
    "albums = set(iron.album)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2941768a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'A Matter Of Life And Death',\n",
       " 'Be Quick Or Be Dead',\n",
       " 'Brave New World',\n",
       " 'Can I Play With Madness',\n",
       " 'Dance Of Death',\n",
       " 'Fear Of The Dark',\n",
       " 'From Here For Eternity',\n",
       " 'Iron Maiden',\n",
       " 'Killers',\n",
       " 'Man On The Edge',\n",
       " 'No Prayer For The Dying',\n",
       " 'Piece Of Mind',\n",
       " 'Powerslave',\n",
       " 'Rainmaker',\n",
       " 'Running Free',\n",
       " 'Senjutsu',\n",
       " 'Seventh Son Of A Seventh Son',\n",
       " 'Somewhere In Time',\n",
       " 'The Book Of Souls',\n",
       " 'The Final Frontier',\n",
       " 'The Number Of The Beast',\n",
       " 'The X-Factor',\n",
       " 'Virtual XI',\n",
       " 'Virus',\n",
       " 'Wasted Years',\n",
       " 'Wildest Dreams',\n",
       " 'Women In Uniform'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "albums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e136b973",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyr_ctx = [(r.lyrics, r.iloc[:-1].to_dict()) for _,r in iron.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b65d0ef2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"As I lay here lying on my bed, sweet voices come into my head. Oh what it is, I wanna know, please won't you tell me it's got to go. There's a feeling that's inside me, telling me to get away. But I'm so tired of living, I might as well end today. \",\n",
       " {'album': 'Killers', 'year': 1981, 'num': 4, 'title': 'Another Life'})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lyr_ctx[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "70cddc93",
   "metadata": {},
   "outputs": [],
   "source": [
    "for ext in ['album', 'year', 'song_num', 'title']:\n",
    "    Doc.set_extension(ext, default=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "792befe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sents = []\n",
    "for doc, ctx in nlp.pipe(lyr_ctx, as_tuples=True):\n",
    "    # we first run this through the naive model that doesn't recognize entities\n",
    "    # just so that we can use its sentencizer\n",
    "    for sent in doc.sents:\n",
    "        s = sent.as_doc()\n",
    "        all_sents.append((s.text, ctx))\n",
    "del lyr_ctx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "49d2492c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test_inds = list(range(len(all_sents)))\n",
    "random.shuffle(train_test_inds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ec206437",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_cutoff = int(len(all_sents) * 0.9)\n",
    "train_cutoff = int(len(all_sents) * 0.4)\n",
    "train_inds = train_test_inds[:train_cutoff]\n",
    "test_inds = train_test_inds[train_cutoff:val_cutoff]\n",
    "val_inds = train_test_inds[val_cutoff:]\n",
    "train_sents = [all_sents[ii] for ii in train_inds]\n",
    "test_sents = [all_sents[ii] for ii in test_inds]\n",
    "val_sents = [all_sents[ii] for ii in val_inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "74dfa0c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_docs = []\n",
    "for doc, ctx in nlp.pipe(train_sents, as_tuples=True):\n",
    "    # now we our pre-defined labeling rules to label only the training set\n",
    "    # we will also hand-label a lot of the training set to get some things that slipped through our initial rules\n",
    "    doc._.album = ctx['album']\n",
    "    doc._.song_num = ctx['num']\n",
    "    doc._.title = ctx['title']\n",
    "    doc._.year = ctx['year']\n",
    "    matcher(doc)\n",
    "    train_docs.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "12bb7f7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_docs = []\n",
    "for doc, ctx in nlp.pipe(test_sents, as_tuples=True):\n",
    "    doc._.album = ctx['album']\n",
    "    doc._.song_num = ctx['num']\n",
    "    doc._.title = ctx['title']\n",
    "    doc._.year = ctx['year']\n",
    "    matcher(doc)\n",
    "    test_docs.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "12245e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_docs = []\n",
    "for doc, ctx in nlp.pipe(val_sents, as_tuples=True):\n",
    "    doc._.album = ctx['album']\n",
    "    doc._.song_num = ctx['num']\n",
    "    doc._.year = ctx['year']\n",
    "    doc._.title = ctx['title']\n",
    "    matcher(doc)\n",
    "    val_docs.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "852402df",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_by_album = iron.groupby(['album', 'year']) \\\n",
    "                     .apply(lambda x: nlp('\\n'.join(x.lyrics))) \\\n",
    "                     .reset_index() \\\n",
    "                     .rename({0: 'lyrics'}, axis=1) \\\n",
    "                     .sort_values('year')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2165c119",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron['lyrics'] = iron.lyrics.apply(lambda x: nlp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d293d1aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "alchemist = iron[iron.title == 'The Alchemist'].iat[0, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "97cfc408",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>album</th>\n",
       "      <th>year</th>\n",
       "      <th>lyrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Women In Uniform</td>\n",
       "      <td>1980</td>\n",
       "      <td>(Beehive, hairdo, ,, 45, on, the, hip, ., Patr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Running Free</td>\n",
       "      <td>1980</td>\n",
       "      <td>(So, you, think, you, can, own, me, ,, well, y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The Book Of Souls</td>\n",
       "      <td>2015</td>\n",
       "      <td>(Here, is, the, soul, of, a, man, ., Here, in,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Senjutsu</td>\n",
       "      <td>2021</td>\n",
       "      <td>(Beat, the, warning, the, sound, of, the, drum...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                album  year                                             lyrics\n",
       "26   Women In Uniform  1980  (Beehive, hairdo, ,, 45, on, the, hip, ., Patr...\n",
       "14       Running Free  1980  (So, you, think, you, can, own, me, ,, well, y...\n",
       "18  The Book Of Souls  2015  (Here, is, the, soul, of, a, man, ., Here, in,...\n",
       "15           Senjutsu  2021  (Beat, the, warning, the, sound, of, the, drum..."
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_by_album.iloc[[0,1,-2, -1], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c1a10b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron_doc = docs_by_album[docs_by_album.album == 'Iron Maiden'].lyrics.iat[0]\n",
    "senju_doc = docs_by_album[docs_by_album.album == 'Senjutsu'].lyrics.iat[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "01acb20e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# iron.to_pickle('all_songs_df__en_core_web_md.pkl.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b64e4346",
   "metadata": {},
   "outputs": [],
   "source": [
    "# docs_by_album.to_pickle('songs_by_album_df__en_core_web_md.pkl.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d2ff8d35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(light,), (light, second), (death,), ()]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mysents = [\n",
    "    nlp(\"I am a light and i love flight!\"),\n",
    "    nlp(\"I eat light every second.\"),\n",
    "    nlp(\"Squirrels also eat a lot of death\"),\n",
    "    nlp(\"I second that emotion.\")\n",
    "]\n",
    "for sent in mysents:\n",
    "    matcher(sent)\n",
    "[s.ents for s in mysents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f65860ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2671 3340 668\n",
      "train\n",
      " [('time', 'TIME'), ('hour', 'TIME'), ('white', 'LIGHT'), ('light', 'LIGHT'), ('kill', 'DEATH'), ('dead', 'DEATH'), ('dead', 'DEATH'), ('Waiting', 'TIME')]\n",
      "test\n",
      " [('Time', 'TIME'), ('waits', 'TIME'), ('dead', 'DEATH'), ('time', 'TIME'), ('time', 'TIME'), ('fates', 'DEATH'), ('dead', 'DEATH'), ('kill', 'DEATH'), ('Dark', 'LIGHT'), ('dark', 'LIGHT'), ('Kill', 'DEATH'), ('dead', 'DEATH'), ('graves', 'DEATH'), ('dark', 'LIGHT'), ('time', 'TIME'), ('time', 'TIME'), ('Waiting', 'TIME')]\n"
     ]
    }
   ],
   "source": [
    "print(len(train_docs), len(test_docs), len(val_docs))\n",
    "print('train\\n', [(e.text, e.label_) for sent in train_docs[:60] for e in sent.ents])\n",
    "print('test\\n', [(e.text, e.label_) for sent in test_docs[:60] for e in sent.ents])\n",
    "# to make sure the train sentences are being labeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7f79ae57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# label_many_sents(mysents, ['animal', 'food'])\n",
    "# [[(e.text, e.label_) for e in s.ents] for s in mysents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bbb8877d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# random_sents = random.sample(train_docs, 500)\n",
    "# label_many_sents(random_sents, iron_label_holder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7476d75f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_doc_bin = DocBin()\n",
    "for d in train_docs:\n",
    "    train_doc_bin.add(d)\n",
    "train_doc_bin.to_disk('train_docs__en_core_web_md.spacy')\n",
    "# note that this is much faster than pickling the DataFrames\n",
    "test_doc_bin = DocBin()\n",
    "for d in test_docs:\n",
    "    test_doc_bin.add(d)\n",
    "test_doc_bin.to_disk('test_docs__en_core_web_md.spacy')\n",
    "val_doc_bin = DocBin()\n",
    "for d in val_docs:\n",
    "    val_doc_bin.add(d)\n",
    "val_doc_bin.to_disk('val_docs__en_core_web_md.spacy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "07458033",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Generated config template specific for your use case\n",
      "- Language: en\n",
      "- Pipeline: tagger, parser, ner\n",
      "- Optimize for: efficiency\n",
      "- Hardware: CPU\n",
      "- Transformer: None\n",
      "[+] Auto-filled config with all values\n",
      "[+] Saved config\n",
      "config.cfg\n",
      "You can now add your data and train your pipeline:\n",
      "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-03 14:37:04.403082: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found\n",
      "2022-05-03 14:37:04.403119: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy init config -F ./config.cfg --lang en --pipeline tagger,parser,ner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b073ba2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Saving to output directory: output"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-03 14:37:11.791570: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'cudart64_110.dll'; dlerror: cudart64_110.dll not found\n",
      "2022-05-03 14:37:11.791608: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "[2022-05-03 14:37:16,171] [INFO] Set up nlp object from config\n",
      "[2022-05-03 14:37:16,194] [INFO] Pipeline: ['tok2vec', 'tagger', 'parser', 'ner']\n",
      "[2022-05-03 14:37:16,202] [INFO] Created vocabulary\n",
      "[2022-05-03 14:37:16,203] [INFO] Finished initializing nlp object\n",
      "[2022-05-03 14:37:18,549] [INFO] Initialized pipeline components: ['tok2vec', 'tagger', 'parser', 'ner']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[i] Using CPU\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "[+] Initialized pipeline\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "[i] Pipeline: ['tok2vec', 'tagger', 'parser', 'ner']\n",
      "[i] Initial learn rate: 0.001\n",
      "E    #       LOSS TOK2VEC  LOSS TAGGER  LOSS PARSER  LOSS NER  TAG_ACC  DEP_UAS  DEP_LAS  SENTS_F  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
      "---  ------  ------------  -----------  -----------  --------  -------  -------  -------  -------  ------  ------  ------  ------\n",
      "  0       0          0.00        90.89       252.60      7.64    32.27    23.83    11.94     0.04    0.00    0.00    0.00    0.17\n",
      "  0     200       2158.39      8647.25     16985.99   1170.19    85.53    80.04    71.72    95.73   67.49   73.42   62.45    0.76\n",
      "  2     400       3129.96      3653.84      9351.59    513.23    92.28    86.04    79.57    98.96   87.62   86.80   88.45    0.88\n",
      "  3     600       3614.08      2100.48      7602.37    221.77    93.60    87.23    82.16    99.19   91.24   89.24   93.33    0.90\n",
      "  5     800       4209.61      1638.17      6439.77    144.34    93.97    88.02    83.33    98.85   93.81   94.09   93.53    0.91\n",
      "  7    1000       4709.26      1284.70      5392.92     96.57    94.36    88.79    84.43    98.35   93.91   91.90   96.02    0.92\n",
      "  9    1200       5310.88      1111.15      4644.91     77.92    94.43    89.07    84.72    98.60   96.61   98.05   95.22    0.93\n",
      " 12    1400       5993.75       906.32      3997.14     95.89    94.64    88.97    84.93    99.10   95.96   96.01   95.92    0.92\n",
      " 16    1600       6225.72       800.79      3424.82     71.06    94.45    89.16    84.76    99.60   96.88   97.96   95.82    0.93\n",
      " 21    1800       6368.68       722.46      2948.93     84.63    94.57    89.59    85.28    99.33   96.40   96.69   96.12    0.93\n",
      " 27    2000       7365.09       607.30      2725.09     84.62    94.71    89.23    85.10    99.46   95.62   94.55   96.71    0.92\n",
      " 34    2200       6784.58       559.64      2374.84     57.54    94.67    89.22    85.12    98.84   97.29   98.17   96.41    0.93\n",
      " 42    2400       7354.29       556.92      2285.34     59.92    94.72    89.29    85.25    99.46   95.45   93.76   97.21    0.92\n",
      " 51    2600       6531.12       392.47      1965.20     41.74    94.70    89.35    85.25    98.90   97.49   98.38   96.61    0.93\n",
      " 60    2800       7618.76       381.78      1911.88     31.52    94.65    89.18    84.90    98.88   96.76   96.81   96.71    0.93\n",
      " 68    3000       6619.51       337.79      1625.43     39.93    94.75    89.31    85.15    98.82   97.44   98.08   96.81    0.93\n",
      " 77    3200       6744.25       289.28      1525.59     38.20    94.77    89.64    85.55    99.06   96.66   96.90   96.41    0.93\n",
      " 86    3400       7164.00       285.17      1424.37     49.52    94.84    89.29    85.04    99.30   97.24   98.07   96.41    0.93\n",
      " 94    3600       8238.37       280.09      1536.68     58.40    94.79    89.47    85.18    99.09   97.15   97.59   96.71    0.93\n",
      "103    3800       7509.11       274.17      1423.08     36.39    94.83    89.95    85.84    98.97   96.91   96.91   96.91    0.93\n",
      "112    4000       7348.85       230.18      1286.01     51.65    95.01    89.55    85.58    98.84   96.96   97.01   96.91    0.93\n",
      "121    4200       6965.23       226.85      1327.00     12.21    94.75    89.67    85.60    99.02   96.92   96.82   97.01    0.93\n",
      "129    4400       7084.03       210.09      1191.32     31.33    94.89    89.59    85.22    98.66   97.06   97.01   97.11    0.93\n",
      "138    4600       6700.91       208.49      1135.85     24.13    94.84    89.89    85.65    99.64   95.73   94.39   97.11    0.93\n",
      "147    4800       6748.94       195.25       954.49     35.40    94.86    89.67    85.47    99.63   97.70   98.29   97.11    0.93\n",
      "155    5000       7318.42       171.93      1170.88     26.12    94.75    89.71    85.58    98.51   97.64   98.58   96.71    0.93\n",
      "164    5200       6200.16       210.51       879.54     17.35    94.72    89.80    85.53    99.69   97.70   98.19   97.21    0.93\n",
      "173    5400       8394.00       179.52      1154.96     18.04    94.86    89.42    85.27    99.55   97.84   98.88   96.81    0.93\n",
      "181    5600       6574.20       156.67       889.52     19.13    94.78    89.74    85.35    99.21   96.43   96.14   96.71    0.93\n",
      "190    5800       8723.60       167.87      1014.82     34.16    94.60    89.34    84.92    99.06   97.43   98.67   96.22    0.93\n",
      "199    6000       7405.40       168.33       909.23     22.36    94.63    89.56    85.51    98.84   97.07   96.92   97.21    0.93\n",
      "207    6200       8235.14       153.44       946.30     25.20    94.67    89.54    85.32    99.42   97.29   97.98   96.61    0.93\n",
      "216    6400       7781.61       160.60       852.99     33.47    94.74    89.82    85.41    98.93   97.94   98.88   97.01    0.93\n",
      "225    6600       7404.67       163.13       852.36     21.58    94.69    89.89    85.74    99.27   97.45   97.89   97.01    0.93\n",
      "234    6800       7588.01       153.70       867.17      9.54    94.66    89.55    85.34    99.67   97.88   99.28   96.51    0.93\n",
      "242    7000       8140.63       144.88       840.07     25.18    94.73    89.56    85.37    99.15   97.20   97.69   96.71    0.93\n",
      "251    7200       9135.66       128.80       848.79     38.50    94.67    89.59    85.43    99.54   96.92   96.73   97.11    0.93\n",
      "260    7400       8448.93       144.17       764.94     21.69    94.65    89.81    85.64    98.96   97.06   97.20   96.91    0.93\n",
      "268    7600       9020.53       100.01       845.79     24.86    94.72    89.66    85.39    99.19   96.62   96.34   96.91    0.93\n",
      "277    7800       7912.70       106.59       797.74     12.83    94.67    89.68    85.53    99.19   97.01   97.11   96.91    0.93\n",
      "286    8000       8495.42       111.40       761.25     23.43    94.80    89.72    85.75    99.19   97.11   97.30   96.91    0.93\n",
      "[+] Saved pipeline to output directory\n",
      "output\\model-last\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy train ./config.cfg --output ./output --paths.train train_docs__en_core_web_md.spacy --paths.dev test_docs__en_core_web_md.spacy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}