scrape_iron_maiden.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6ba33fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import re\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ef5f44fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "baseurl = \"http://www.darklyrics.com/i/ironmaiden.html\"\n",
    "soup = bs(requests.get(baseurl).text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "14de3cdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "song_link_selector = 'div.album a'\n",
    "song_links = soup.select(song_link_selector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cd6e8e93",
   "metadata": {},
   "outputs": [],
   "source": [
    "album_soups = {}\n",
    "for link in song_links:\n",
    "    album = link.get_attribute_list('href')[0].split('/')[-1]\n",
    "    album = re.sub('\\.html#\\d+', '', album)\n",
    "    if album not in album_soups:\n",
    "        alblink = f'http://www.darklyrics.com/lyrics/ironmaiden/{album}.html#1'\n",
    "        albsoup = bs(requests.get(alblink).text)\n",
    "        album_soups[album] = albsoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b33ce88f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Iron Maiden', 1980)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_year = album_soups['ironmaiden'].select_one('h2').text\n",
    "title, year = re.findall('\\\"(.+)\\\".+\\((\\d+)\\)', title_year)[0]\n",
    "year = int(year)\n",
    "title, year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9d090532",
   "metadata": {},
   "outputs": [],
   "source": [
    "iron = album_soups['ironmaiden']\n",
    "iron_lyrics = iron.select_one('div.lyrics')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "259e2b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "title = None\n",
    "iron_songs = {}\n",
    "for child in iron_lyrics.children:\n",
    "    if child.name == 'h3':\n",
    "        title = re.sub('\\d+\\.', '', child.text).strip()\n",
    "        continue\n",
    "    if title and not hasattr(child, 'text'):\n",
    "        text = child.strip()\n",
    "        if text:\n",
    "            iron_songs.setdefault(title, [])\n",
    "            iron_songs[title].append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bab5a2df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['Prowler', 'Sanctuary', 'Remember Tomorrow', 'Running Free', 'Phantom Of The Opera', 'Strange World', 'Charlotte The Harlot', 'Iron Maiden'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iron_songs.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "abc9d6db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Walking through the city, looking oh so pretty,',\n",
       " \"I've just got to find my way.\",\n",
       " 'See the ladies flashing. All their legs and lashes.',\n",
       " \"I've just got to find my way.\",\n",
       " 'Well you see me crawling through the bushes with it open wide.',\n",
       " 'What you seeing girl?',\n",
       " \"Can't you believe that feeling, can't you believe it,\",\n",
       " \"Can't you believe your eyes?\",\n",
       " \"It's the real thing girl.\",\n",
       " 'Got me feeling myself and reeling around,',\n",
       " 'Got me talking but feel like walking around.',\n",
       " 'Got me feeling myself and reeling a...',\n",
       " \"Got me talking but nothing's with me...\",\n",
       " 'Got me feeling myself and reeling around.']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iron_songs['Prowler']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2edb7c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "songdata = []\n",
    "songs_seen = set()\n",
    "for albsoup in album_soups.values():\n",
    "    title = None\n",
    "    this_song = {}\n",
    "    lyrics = None\n",
    "    title_year = albsoup.select_one('h2').text\n",
    "    album_title, year = re.findall('\\\"(.+)\\\".+\\((\\d+)\\)', title_year)[0]\n",
    "    year = int(year)\n",
    "    song_num = None\n",
    "    song = None\n",
    "    all_lyrics = albsoup.select_one('div.lyrics')\n",
    "    for child in all_lyrics.children:\n",
    "        if child.name == 'h3':\n",
    "            if song:\n",
    "                songdata.append(song)\n",
    "            title = re.sub('\\d+\\.', '', child.text).strip()\n",
    "            song_num = int(re.findall('^\\d+',  child.text)[0])\n",
    "            stripped_title = re.sub('\\(.+\\)', '', title).strip()\n",
    "            # gets rid of things like \"Public Enema Number One (Live)\"\n",
    "            # by converting them to the base song (\"Public Enema Number One\")\n",
    "            if stripped_title in songs_seen:\n",
    "                song = None\n",
    "            else:\n",
    "                songs_seen.add(title)\n",
    "                song = {\n",
    "                    'album': album_title,\n",
    "                    'year': year,\n",
    "                    'num': song_num,\n",
    "                    'title': title,\n",
    "                    'lyrics': '',\n",
    "                }\n",
    "            continue\n",
    "        if not song:\n",
    "            continue\n",
    "        \n",
    "        if title and not hasattr(child, 'text'):\n",
    "            text = child.strip()\n",
    "            if text:\n",
    "                song['lyrics'] += text + '\\\\n'\n",
    "    if song:\n",
    "        songdata.append(song)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f1cb0c54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Public Enema Number One (Live)' in songs_seen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "61c57269",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "187"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(songdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c8ffbc6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "187"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(songs_seen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "642b122b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'album': 'Iron Maiden', 'year': 1980, 'num': 9, 'title': 'Iron Maiden', 'lyrics': \"Won't you come into my room, I wanna show you all my wares.\\\\nI just want to see your blood, I just want to stand and stare.\\\\nSee the blood begin to flow as it falls upon the floor.\\\\nIron Maiden can't be fought, Iron Maiden can't be sought.\\\\nOh Well, wherever, wherever you are,\\\\nIron Maiden's gonna get you, no matter how far.\\\\nSee the blood flow watching it shed up above my head.\\\\nIron Maiden wants you for dead.\\\\nWon't you come into my room, I wanna show you all my wares.\\\\nI just want to see your blood, I just want to stand and stare.\\\\nSee the blood begin to flow as it falls upon the floor.\\\\nIron Maiden can't be fought, Iron Maiden can't be sought.\\\\n\"}\n"
     ]
    }
   ],
   "source": [
    "print(songdata[8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7a486faf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'2 A.M.',\n",
       " '2 Minutes To Midnight',\n",
       " '22 Acacia Avenue',\n",
       " 'Aces High',\n",
       " 'Afraid To Shoot Strangers',\n",
       " 'Age Of Innocence',\n",
       " 'Age Of Innocence... How Old?',\n",
       " 'Alexander The Great',\n",
       " 'Another Life',\n",
       " 'Back In The Village',\n",
       " 'Be Quick Or Be Dead',\n",
       " 'Black Bart Blues',\n",
       " 'Blood Brothers',\n",
       " \"Blood On The World's Hands\",\n",
       " 'Brave New World',\n",
       " 'Brighter Than A Thousand Suns',\n",
       " 'Bring Your Daughter... To The Slaughter',\n",
       " 'Burning Ambition',\n",
       " 'Can I Play With Madness',\n",
       " 'Can I Play With Madness?',\n",
       " 'Caught Somewhere In Time',\n",
       " 'Chains Of Misery',\n",
       " 'Charlotte The Harlot',\n",
       " \"Childhood's End\",\n",
       " 'Children Of The Damned',\n",
       " 'Coming Home',\n",
       " 'Como Estais Amigos',\n",
       " 'Dance Of Death',\n",
       " 'Darkest Hour',\n",
       " 'Days Of Future Past',\n",
       " 'Death Of The Celts',\n",
       " 'Death Or Glory',\n",
       " 'Deja-Vu',\n",
       " 'Die With Your Boots On',\n",
       " 'Different World',\n",
       " 'Doctor, Doctor',\n",
       " \"Don't Look To The Eyes Of A Stranger\",\n",
       " 'Dream Of Mirrors',\n",
       " 'Drifter',\n",
       " 'El Dorado',\n",
       " 'Empire Of The Clouds',\n",
       " 'Face In The Sand',\n",
       " 'Fates Warning',\n",
       " 'Fear Is The Key',\n",
       " 'Fear Of The Dark',\n",
       " 'Flash Of The Blade',\n",
       " 'Flight Of Icarus',\n",
       " 'For The Greater Good Of God',\n",
       " 'Fortunes Of War',\n",
       " 'From Here For Eternity',\n",
       " 'From Here To Eternity',\n",
       " 'Futureal',\n",
       " 'Gangland',\n",
       " 'Gates Of Tomorrow',\n",
       " 'Genghis Khan',\n",
       " 'Ghost Of The Navigator',\n",
       " 'Hallowed Be Thy Name',\n",
       " 'Heaven Can Wait',\n",
       " 'Hell On Earth',\n",
       " 'Holy Smoke',\n",
       " 'Hooks In You',\n",
       " \"I Can't See My Feelings\",\n",
       " 'I Live My Way',\n",
       " 'If Eternity Should Fail',\n",
       " 'Infinite Dreams',\n",
       " 'Innocent Exile',\n",
       " 'Invaders',\n",
       " 'Invasion',\n",
       " 'Iron Maiden',\n",
       " 'Isle Of Avalon',\n",
       " 'Journeyman',\n",
       " 'Judas Be My Guide',\n",
       " 'Judgement Day',\n",
       " 'Judgement Of Heaven',\n",
       " 'Justice Of The Peace',\n",
       " 'Killers',\n",
       " 'Lightning Strikes Twice',\n",
       " 'Look For The Truth',\n",
       " 'Lord Of Light',\n",
       " 'Lord Of The Flies',\n",
       " \"Losfer Words (Big 'Orra)\",\n",
       " 'Lost In A Lost World',\n",
       " 'Man On The Edge',\n",
       " 'Massacre',\n",
       " 'MontsÃ©gur',\n",
       " 'Moonchild',\n",
       " 'More Tea Vicar',\n",
       " 'Mother Of Mercy',\n",
       " 'Mother Russia',\n",
       " 'Murders In The Rue Morgue',\n",
       " 'My Generation',\n",
       " 'New Frontier',\n",
       " 'No More Lies',\n",
       " 'No Prayer For The Dying',\n",
       " 'Nodding Donkey Blues',\n",
       " 'Only The Good Die Young',\n",
       " 'Out Of The Shadows',\n",
       " 'Out Of The Silent Planet',\n",
       " 'Paschendale',\n",
       " 'Pass The Jam',\n",
       " 'Phantom Of The Opera',\n",
       " 'Powerslave',\n",
       " 'Prodigal Son',\n",
       " 'Prowler',\n",
       " 'Public Enema Number One',\n",
       " 'Purgatory',\n",
       " 'Quest For Fire',\n",
       " 'Rainmaker',\n",
       " 'Reach Out',\n",
       " 'Remember Tomorrow',\n",
       " 'Revelations',\n",
       " 'Rime Of The Ancient Mariner',\n",
       " 'Roll Over Vic Vella',\n",
       " 'Run Silent Run Deep',\n",
       " 'Run To The Hills',\n",
       " 'Running Free',\n",
       " 'Sanctuary',\n",
       " 'Satellite .. The Final Frontier',\n",
       " 'Sea Of Madness',\n",
       " 'Senjutsu',\n",
       " 'Seventh Son Of A Seventh Son',\n",
       " 'Shadows Of The Valley',\n",
       " 'Sheriff Of Huddersfield',\n",
       " 'Sign Of The Cross',\n",
       " 'Space Station No. 5',\n",
       " 'Speed Of Light',\n",
       " 'Starblind',\n",
       " 'Still Life',\n",
       " 'Strange World',\n",
       " 'Stranger In A Strange Land',\n",
       " 'Stratego',\n",
       " 'Sun And Steel',\n",
       " 'Tailgunner',\n",
       " 'Tears Of A Clown',\n",
       " 'The Aftermath',\n",
       " 'The Alchemist',\n",
       " 'The Angel And The Gambler',\n",
       " 'The Apparition',\n",
       " 'The Assassin',\n",
       " 'The Book Of Souls',\n",
       " 'The Clairvoyant',\n",
       " 'The Clansman',\n",
       " 'The Duellists',\n",
       " 'The Edge Of Darkness',\n",
       " 'The Educated Fool',\n",
       " 'The Evil That Men Do',\n",
       " 'The Fallen Angel',\n",
       " 'The Fugitive',\n",
       " 'The Great Unknown',\n",
       " 'The Ides Of March',\n",
       " 'The Legacy',\n",
       " 'The Loneliness Of The Long Distance Runner',\n",
       " 'The Longest Day',\n",
       " 'The Man Of Sorrows',\n",
       " 'The Man Who Would Be King',\n",
       " 'The Mercenary',\n",
       " 'The Nomad',\n",
       " 'The Number Of The Beast',\n",
       " 'The Parchment',\n",
       " 'The Pilgrim',\n",
       " 'The Prisoner',\n",
       " 'The Prophecy',\n",
       " 'The Red And The Black',\n",
       " 'The Reincarnation Of Benjamin Breeg',\n",
       " 'The Talisman',\n",
       " 'The Thin Line Between Love And Hate',\n",
       " 'The Time Machine',\n",
       " 'The Trooper',\n",
       " 'The Unbeliever',\n",
       " 'The Wicker Man',\n",
       " 'The Writing On The Wall',\n",
       " \"These Colours Don't Run\",\n",
       " 'To Tame A Land',\n",
       " 'Total Eclipse',\n",
       " 'Transylvania',\n",
       " 'Twilight Zone',\n",
       " 'Virus',\n",
       " 'Wasted Years',\n",
       " 'Wasting Love',\n",
       " 'Weekend Warrior',\n",
       " 'When The River Runs Deep',\n",
       " 'When The Wild Wind Blows',\n",
       " 'When Two Worlds Collide',\n",
       " 'Where Eagles Dare',\n",
       " 'Wildest Dreams',\n",
       " 'Women In Uniform',\n",
       " 'Wrathchild'}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "songs_seen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a06a0152",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "any('\\t' in song['lyrics'] for song in songdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2444bb68",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('iron_maiden_songs.tsv', 'w', encoding='utf-8') as f:\n",
    "    f.write('\\t'.join(songdata[0].keys()) + '\\n')\n",
    "    for song in songdata:\n",
    "        if 'Ã©' in song['title']:\n",
    "            song['title'] = song['title'].replace('Ã©', 'é')\n",
    "            song['lyrics'] = song['lyrics'].replace('Ã©', 'é')\n",
    "        f.write('\\t'.join(str(x) for x in song.values()) + '\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}