surnamelist_builder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas\n",
    "import transliterate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loads(s):\n",
    "    try:\n",
    "        return json.loads(s)\n",
    "    except:\n",
    "        print (s)\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transliteration2(text):\n",
    "    cyrillic = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'\n",
    "    latin = 'a|b|v|g|d|e|e|zh|z|i|j|k|l|m|n|o|p|r|s|t|u|f|kh|ts|ch|sh|shch||y||e|yu|ya|A|B|V|G|D|E|E|Zh|Z|I|J|K|L|M|N|O|P|R|S|T|U|F|Kh|Ts|Ch|Sh|Shch||Y||E|Yu|Ya'.split('|')    \n",
    "    return text.translate({ord(k):v for k,v in zip(cyrillic,latin)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans_str(s):\n",
    "    try:\n",
    "        return transliteration2(s)\n",
    "    except:\n",
    "        return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "x = list(map(loads,open('surnames.jsonl').read().split('\\n')))\n",
    "x = list(filter(lambda x: x is not None, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "surnames_df = pandas.DataFrame.from_records(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "surnames_df = surnames_df.sort_values('count',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "surnames_df['text'] = surnames_df.text.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "surnames_df['trans'] = surnames_df.text.apply(trans_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>count</th>\n",
       "      <th>gender</th>\n",
       "      <th>ethnic</th>\n",
       "      <th>f_form</th>\n",
       "      <th>fname</th>\n",
       "      <th>text</th>\n",
       "      <th>m_form</th>\n",
       "      <th>trans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>{'$oid': '4bdaa13d9d55821a51000000'}</td>\n",
       "      <td>6926</td>\n",
       "      <td>m</td>\n",
       "      <td>[slav]</td>\n",
       "      <td>Иванова</td>\n",
       "      <td>Иван</td>\n",
       "      <td>иванов</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ivanov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>{'$oid': '4bdaa13d9d55821a51000004'}</td>\n",
       "      <td>5206</td>\n",
       "      <td>f</td>\n",
       "      <td>[slav]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Иван</td>\n",
       "      <td>иванова</td>\n",
       "      <td>Иванов</td>\n",
       "      <td>ivanova</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>{'$oid': '4bdaa13d9d55821a51000001'}</td>\n",
       "      <td>4789</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Кузнецова</td>\n",
       "      <td>NaN</td>\n",
       "      <td>кузнецов</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kuznetsov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>{'$oid': '4bdaa13d9d55821a51000002'}</td>\n",
       "      <td>4406</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Смирнова</td>\n",
       "      <td>NaN</td>\n",
       "      <td>смирнов</td>\n",
       "      <td>NaN</td>\n",
       "      <td>smirnov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>{'$oid': '4bdaa13d9d55821a51000003'}</td>\n",
       "      <td>4351</td>\n",
       "      <td>m</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Попова</td>\n",
       "      <td>NaN</td>\n",
       "      <td>попов</td>\n",
       "      <td>NaN</td>\n",
       "      <td>popov</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54512</td>\n",
       "      <td>{'$oid': '4bdaa1419d55821a5100d4f0'}</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>сутула</td>\n",
       "      <td>NaN</td>\n",
       "      <td>sutula</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320909</td>\n",
       "      <td>{'$oid': '4ca081249d5582342e2765a9'}</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>шмунк</td>\n",
       "      <td>NaN</td>\n",
       "      <td>shmunk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63578</td>\n",
       "      <td>{'$oid': '4bdaa1429d55821a5100f85a'}</td>\n",
       "      <td>5</td>\n",
       "      <td>f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>сесорова</td>\n",
       "      <td>Сесоров</td>\n",
       "      <td>sesorova</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>261137</td>\n",
       "      <td>{'$oid': '4ca068269d5582342e12caf1'}</td>\n",
       "      <td>5</td>\n",
       "      <td>f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>погирская</td>\n",
       "      <td>Погирский</td>\n",
       "      <td>pogirskaya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260687</td>\n",
       "      <td>{'$oid': '4ca068039d5582342e12ad21'}</td>\n",
       "      <td>5</td>\n",
       "      <td>f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>шунтикова</td>\n",
       "      <td>Шунтиков</td>\n",
       "      <td>shuntikova</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99387 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         _id  count gender  ethnic     f_form  \\\n",
       "0       {'$oid': '4bdaa13d9d55821a51000000'}   6926      m  [slav]    Иванова   \n",
       "4       {'$oid': '4bdaa13d9d55821a51000004'}   5206      f  [slav]        NaN   \n",
       "1       {'$oid': '4bdaa13d9d55821a51000001'}   4789      m     NaN  Кузнецова   \n",
       "2       {'$oid': '4bdaa13d9d55821a51000002'}   4406      m     NaN   Смирнова   \n",
       "3       {'$oid': '4bdaa13d9d55821a51000003'}   4351      m     NaN     Попова   \n",
       "...                                      ...    ...    ...     ...        ...   \n",
       "54512   {'$oid': '4bdaa1419d55821a5100d4f0'}      5    NaN     NaN        NaN   \n",
       "320909  {'$oid': '4ca081249d5582342e2765a9'}      5    NaN     NaN        NaN   \n",
       "63578   {'$oid': '4bdaa1429d55821a5100f85a'}      5      f     NaN        NaN   \n",
       "261137  {'$oid': '4ca068269d5582342e12caf1'}      5      f     NaN        NaN   \n",
       "260687  {'$oid': '4ca068039d5582342e12ad21'}      5      f     NaN        NaN   \n",
       "\n",
       "       fname       text     m_form       trans  \n",
       "0       Иван     иванов        NaN      ivanov  \n",
       "4       Иван    иванова     Иванов     ivanova  \n",
       "1        NaN   кузнецов        NaN   kuznetsov  \n",
       "2        NaN    смирнов        NaN     smirnov  \n",
       "3        NaN      попов        NaN       popov  \n",
       "...      ...        ...        ...         ...  \n",
       "54512    NaN     сутула        NaN      sutula  \n",
       "320909   NaN      шмунк        NaN      shmunk  \n",
       "63578    NaN   сесорова    Сесоров    sesorova  \n",
       "261137   NaN  погирская  Погирский  pogirskaya  \n",
       "260687   NaN  шунтикова   Шунтиков  shuntikova  \n",
       "\n",
       "[99387 rows x 9 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "surnames_df[surnames_df['count']>=5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "surnames_df[surnames_df['count']>=5][['text','count']].to_csv('russian_surnames.csv',sep=',',index=False,header=False)\n",
    "\n",
    "surnames_df[surnames_df['count']>=5][['trans','count']].to_csv('russian_trans_surnames.csv',sep=',',index=False,header=False)\n",
    "\n",
    "surnames_df[surnames_df['count']>=5][['text']].to_csv('russian_surnames.txt',sep=',',index=False,header=False)\n",
    "\n",
    "surnames_df[surnames_df['count']>=5][['trans']].to_csv('russian_trans_surnames.txt',sep=',',index=False,header=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}