기사전처리_한국경제_찐찐최종.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wfxuT5WPcnSN"
      },
      "source": [
        "# 패키지 설치"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "YK6YnrjOcnTn",
        "outputId": "74db0586-fecd-4b0c-bf7c-961b2a2c3c1b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting hanja\n",
            "  Downloading hanja-0.15.1-py3-none-any.whl.metadata (3.5 kB)\n",
            "Collecting pyyaml==6.0.1 (from hanja)\n",
            "  Downloading PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.1 kB)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.10/dist-packages (from hanja) (7.4.4)\n",
            "Collecting pytest-cov (from hanja)\n",
            "  Downloading pytest_cov-5.0.0-py3-none-any.whl.metadata (27 kB)\n",
            "Collecting coveralls (from hanja)\n",
            "  Downloading coveralls-4.0.1-py3-none-any.whl.metadata (5.3 kB)\n",
            "Collecting coverage!=6.0.*,!=6.1,!=6.1.1,<8.0,>=5.0 (from coverage[toml]!=6.0.*,!=6.1,!=6.1.1,<8.0,>=5.0->coveralls->hanja)\n",
            "  Downloading coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (8.3 kB)\n",
            "Collecting docopt<0.7.0,>=0.6.1 (from coveralls->hanja)\n",
            "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: requests<3.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from coveralls->hanja) (2.32.3)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.10/dist-packages (from pytest->hanja) (2.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from pytest->hanja) (24.1)\n",
            "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.10/dist-packages (from pytest->hanja) (1.5.0)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /usr/local/lib/python3.10/dist-packages (from pytest->hanja) (1.2.2)\n",
            "Requirement already satisfied: tomli>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pytest->hanja) (2.0.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=1.0.0->coveralls->hanja) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=1.0.0->coveralls->hanja) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=1.0.0->coveralls->hanja) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=1.0.0->coveralls->hanja) (2024.7.4)\n",
            "Downloading hanja-0.15.1-py3-none-any.whl (124 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.3/124.3 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (705 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m705.5/705.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading coveralls-4.0.1-py3-none-any.whl (13 kB)\n",
            "Downloading pytest_cov-5.0.0-py3-none-any.whl (21 kB)\n",
            "Downloading coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (234 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m234.7/234.7 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: docopt\n",
            "  Building wheel for docopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13704 sha256=a2f4f32344b8c2b64fcce620656432a9e85ba43d00f2013c4ff751580cae4ad5\n",
            "  Stored in directory: /root/.cache/pip/wheels/fc/ab/d4/5da2067ac95b36618c629a5f93f809425700506f72c9732fac\n",
            "Successfully built docopt\n",
            "Installing collected packages: docopt, pyyaml, coverage, pytest-cov, coveralls, hanja\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 6.0.2\n",
            "    Uninstalling PyYAML-6.0.2:\n",
            "      Successfully uninstalled PyYAML-6.0.2\n",
            "Successfully installed coverage-7.6.1 coveralls-4.0.1 docopt-0.6.2 hanja-0.15.1 pytest-cov-5.0.0 pyyaml-6.0.1\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2024.5.15)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.66.5)\n",
            "Collecting git+https://github.com/haven-jeon/PyKoSpacing.git\n",
            "  Cloning https://github.com/haven-jeon/PyKoSpacing.git to /tmp/pip-req-build-feqrqcpd\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/haven-jeon/PyKoSpacing.git /tmp/pip-req-build-feqrqcpd\n",
            "  Resolved https://github.com/haven-jeon/PyKoSpacing.git to commit b32a889cbd10b006d2f4aba118f0cd5b677e2979\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tensorflow>=2.16.2 in /usr/local/lib/python3.10/dist-packages (from pykospacing==0.5) (2.17.0)\n",
            "Requirement already satisfied: h5py>=3.10.0 in /usr/local/lib/python3.10/dist-packages (from pykospacing==0.5) (3.11.0)\n",
            "Collecting argparse>=1.1.0 (from pykospacing==0.5)\n",
            "  Downloading argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from h5py>=3.10.0->pykospacing==0.5) (1.26.4)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=24.3.25 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (24.3.25)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (0.6.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (0.2.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (18.1.1)\n",
            "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (0.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (24.1)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (3.20.3)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (2.32.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (71.0.4)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (4.12.2)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (1.16.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (1.64.1)\n",
            "Requirement already satisfied: tensorboard<2.18,>=2.17 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (2.17.0)\n",
            "Requirement already satisfied: keras>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (3.4.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.16.2->pykospacing==0.5) (0.37.1)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow>=2.16.2->pykospacing==0.5) (0.44.0)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (13.7.1)\n",
            "Requirement already satisfied: namex in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (0.0.8)\n",
            "Requirement already satisfied: optree in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (0.12.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow>=2.16.2->pykospacing==0.5) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow>=2.16.2->pykospacing==0.5) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow>=2.16.2->pykospacing==0.5) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow>=2.16.2->pykospacing==0.5) (2024.7.4)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.16.2->pykospacing==0.5) (3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.16.2->pykospacing==0.5) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.16.2->pykospacing==0.5) (3.0.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.18,>=2.17->tensorflow>=2.16.2->pykospacing==0.5) (2.1.5)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (2.16.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow>=2.16.2->pykospacing==0.5) (0.1.2)\n",
            "Downloading argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n",
            "Building wheels for collected packages: pykospacing\n",
            "  Building wheel for pykospacing (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pykospacing: filename=pykospacing-0.5-py3-none-any.whl size=2286920 sha256=2a0a44e3ea7ed32014984962a7baf64258e11863f333368c2780a6380ab54a73\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-wy8nh_47/wheels/76/b3/33/dda14886ee76b8e53eb05580a14dfcf9145e9eb9d282c53f28\n",
            "Successfully built pykospacing\n",
            "Installing collected packages: argparse, pykospacing\n",
            "Successfully installed argparse-1.4.0 pykospacing-0.5\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "id": "4f3caf8d91154bb3a80418ec541fe57a",
              "pip_warning": {
                "packages": [
                  "argparse"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/ssut/py-hanspell.git\n",
            "  Cloning https://github.com/ssut/py-hanspell.git to /tmp/pip-req-build-ohahmdir\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/ssut/py-hanspell.git /tmp/pip-req-build-ohahmdir\n",
            "  Resolved https://github.com/ssut/py-hanspell.git to commit fdc6ca50c19f1c85971437a072d89d4e5ce024b8\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from py-hanspell==1.1) (2.32.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->py-hanspell==1.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->py-hanspell==1.1) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->py-hanspell==1.1) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->py-hanspell==1.1) (2024.7.4)\n",
            "Building wheels for collected packages: py-hanspell\n",
            "  Building wheel for py-hanspell (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-hanspell: filename=py_hanspell-1.1-py3-none-any.whl size=4810 sha256=78d5b60e171f775ff82dc70a943460f9d93384afc6aaf397e3a8a4e7ce06a06b\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-vijaxudb/wheels/2e/43/cc/753c9e1d91affb9ea40e186cea5654fb9231deb454da6724e5\n",
            "Successfully built py-hanspell\n",
            "Installing collected packages: py-hanspell\n",
            "Successfully installed py-hanspell-1.1\n"
          ]
        }
      ],
      "source": [
        "# 필요한 패키지 설치\n",
        "!pip install hanja   # 한자를 한글로 변환하기 위한 패키지\n",
        "!pip install nltk   # 불용어 제거를 위한 패키지\n",
        "!pip install git+https://github.com/haven-jeon/PyKoSpacing.git   # 띄어쓰기 바로 잡기 위한 패키지\n",
        "!pip install git+https://github.com/ssut/py-hanspell.git   # 오타 및 맞춤법을 수정해주는 패키지"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OqMAXtbhcnT2"
      },
      "source": [
        "# 라이브러리 호출"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1QuUYltjcnT6"
      },
      "outputs": [],
      "source": [
        "# 필요한 라이브러리 호출\n",
        "import pandas as pd\n",
        "import re\n",
        "import hanja\n",
        "from hanspell import spell_checker\n",
        "from nltk.corpus import stopwords\n",
        "from pykospacing import Spacing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yPMY2Hu8cnT8"
      },
      "source": [
        "# 불용어 목록 파일 생성"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rrfTAhDhcnT9",
        "outputId": "e2c925b5-8c8e-4974-8353-0bc203e9bf67"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# NLTK 불용어 다운로드\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "# stopwords 폴더 안에 korean 불용어 파일 만들고 노션에 있는거 복붙해서 넣기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ToexR9wWcnT-"
      },
      "source": [
        "# 코랩 사용할 경우 파일 옮기는 방법"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "FPN8LmlrcnUE",
        "outputId": "2e3c55cf-d6b2-4f15-9942-fa32a456f7d9"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/root/nltk_data/corpora/stopwords/korean'"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# 주피터가 아니라 코랩 쓰는 경우에만 참고\n",
        "# 지연) 구글 코랩으로 작성해서 직접 생성한 korean파일을 옮기는 코드가 필요했음.. 주피터로 하면 필요없을듯!\n",
        "import shutil\n",
        "\n",
        "# 파일 경로\n",
        "src_path = '/content/korean'\n",
        "dest_path = '/root/nltk_data/corpora/stopwords'\n",
        "\n",
        "# 파일 이동\n",
        "shutil.move(src_path, dest_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k3yAiWAgxnWz"
      },
      "source": [
        "### 정규표현식 테스트 .."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oA8mqUQEODkm"
      },
      "outputs": [],
      "source": [
        "# # 정규표현식 테스트\n",
        "# patterns = [\n",
        "#     r'<.*?>|\\[.*?\\]|【.*?】|\\{.*?\\}',\n",
        "#     r'[^가-힣0-9\\s◆◇▲▼]'\n",
        "# ]\n",
        "\n",
        "# text = '안녕하세요! [안녕하세요] (안녕하세요★☆)'\n",
        "# for pattern in patterns:\n",
        "#     text = re.sub(pattern, '', text)\n",
        "# print(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jrzv324fODkn"
      },
      "outputs": [],
      "source": [
        "# text1 = '안녕하세요.이것은 테스트 문장입니다...그렇습니다.'\n",
        "# text2 = '안녕하세요.. 이것은 테스트 문장입니다. 그렇습니다.'\n",
        "\n",
        "# def split_text(article):   # split하는 함수 정의\n",
        "#     sentences = re.split(r'(?<!\\.)\\.(?!\\.)', article)  # 마침표를 기준으로 문장 나누기, 단 마침표가 2개 이상 연달아 있는 경우 제외\n",
        "#     sentences = [sentence.strip() for sentence in sentences if sentence.strip()]   # 공백 제거 및 빈 문장 필터링\n",
        "#     return sentences\n",
        "\n",
        "# print(split_text(text1))\n",
        "# print('=====')\n",
        "# print(split_text(text2))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3F46yVzccnUH"
      },
      "source": [
        "# 전처리 함수 정의"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZbHR08aYcnUJ"
      },
      "outputs": [],
      "source": [
        "# 데이터 클렌징 함수 정의\n",
        "def clean_text(text):\n",
        "\n",
        "    # 0. 한자를 한글로 변환\n",
        "    # text = re.sub(r'\\([^()]*\\)', '', text)  # 괄호 안의 한자는 삭제\n",
        "    text = hanja.translate(text, 'substitution')  # 그 외 한자는 한글로 변환\n",
        "\n",
        "    # 1. 첫 번째 문장에서 `◆...◆` 패턴 제거\n",
        "    sentences = text.split('. ')  # 마침표와 공백을 기준으로 문장을 나누기\n",
        "    if sentences:  # 문장이 존재할 경우\n",
        "        sentences[0] = re.sub(r'◆.*?◆', '', sentences[0])  # 첫 번째 문장에서 패턴 제거\n",
        "    text = '. '.join(sentences)  # 문장들을 다시 결합\n",
        "\n",
        "    # 2. 제거할 단어 및 패턴 목록\n",
        "    # 기사 전체 제거, 패턴 제거 순으로 함수 실행하기\n",
        "    patterns = [\n",
        "        r'[^.]*※[^.]*\\.',            # '※'를 포함하는 문장 전체를 제거\n",
        "        r'[^.]*\\[사설\\][^.]*\\.',      # '[사설]'을 포함하는 문장 전체를 제거\n",
        "        r'[^.]*마켓PRO[^.]*\\.',       # '마켓PRO'를 포함하는 문장 전체를 제거\n",
        "        r'[^.]*한경유레카[^.]*\\.',   # '한경유레카'를 포함하는 문장 전체를 제거\n",
        "        r'[^.]*이 기사는[^.]*\\.',    # '이 기사는'을 포함하는 문장 전체를 제거\n",
        "        r'[^.]*@[^.]*\\.',            # '@'를 포함하는 문장 전체를 제거 / 이메일을 제거하기 위함\n",
        "        r'[^.]*#장면[^.]*\\.',          # '#장면'을 포함하는 문장 전체를 제거\n",
        "        r'[^.]*\\[기자\\][^.]*\\.',      # '[기자]'를 포함하는 문장 전체를 제거\n",
        "        r'[^.]*\\[저작권\\][^.]*\\.',    # '[저작권]'을 포함하는 문장 전체를 제거\n",
        "        r'[^.]*\\[퀴즈\\][^.]*\\.',       # '[퀴즈]'를 포함하는 문장 전체를 제거\n",
        "        r'[^.]*<용 어>[^.]*\\.',\n",
        "        r'<.*?>|\\[.*?\\]|【.*?】|\\{.*?\\}', # ()제외, 온갖 종류의 괄호와 그 안의 내용 전부 제거\n",
        "\n",
        "        r'< 이 기사는 기획 PR 기사 입니다\\. >',\n",
        "        r'< 이 기사는 BizⓝCEO 기획특별판 입니다 >',\n",
        "        r'[^.?!]*?한경닷컴[^.?!]*?[.?!]',\n",
        "        r'[^.?!]*?www\\.[^.?!]*?([.?!]|$)',\n",
        "        r'▶ 삼성미술품 \\d+만\\d+천점 수조원대 이르러…\\d+억원 \\'행복한 눈물\\'은 어디에',\n",
        "        r'▶ 에버랜드 압수수색',\n",
        "        r'▶ 삼성특검, 이순동 사장 소환',\n",
        "        r'▶ 삼성 에버랜드 압수수색 이틀째…긴장 고조',\n",
        "        r'▶ 특검, \\`삼성 전략기획실\\' 임원 첫 소환조사',\n",
        "        r'무단 전재 및 재배포 금지',\n",
        "        r'한경닷컴\\[한경\\+ 구독신청\\] \\[기사구매\\] \\[모바일앱\\]',\n",
        "        r'ⓒ \\'성공을 부르는 습관\\' 한경닷컴, 무단 전재 및 재배포 금지',\n",
        "        r'\\(사진\\)',\n",
        "        r'\\(가나다순\\)',\n",
        "        r'\\(\\d{3}\\)\\d{3}-\\d{4}',   # 전화번호\n",
        "        r'☎',\n",
        "        r'\\s?\\w{3}\\s*기자',   # ooo 기자\n",
        "        r'그래픽=',\n",
        "        r'사진=',\n",
        "        r'▲대한제강=오형근에서 오치훈으로 대표이사 변경',\n",
        "        r'[^.!?]*◆특별취재팀[^.!?]*[.!?]',\n",
        "        r'\\[기사보다 빠른 주식정보 , 슈퍼개미 APP\\]',\n",
        "        r'■ SSDsolid state drive',\n",
        "        r'한경\\+는 PC·폰·태블릿에서 읽을 수 있는 한경 디지털 신문입니다구독 plus\\.hankyung\\.com 문의',\n",
        "        r'한경부동산',\n",
        "        r'뉴스룸',\n",
        "        r'한국경제TV',\n",
        "        r'최근 애널리스트 분석의견',\n",
        "        r'주체별 매매동향',\n",
        "        r'기사제보 및 보도자료',\n",
        "        r'뉴스래빗 페이스북 facebook\\.com/newslabit',\n",
        "        r'#믿고 보는 #기자 \\'한경 기자 코너\\'',\n",
        "        r'▶ 네이버에서 한국경제 뉴스를 받아보세요',\n",
        "        r'▶ 한경닷컴 바로가기',\n",
        "        r'▶ 모바일한경 구독신청',\n",
        "        r'▶ 경제지 네이버 구독 첫 400만, 한국경제 받아보세요',\n",
        "        r'▶ 한경 고품격 뉴스레터, 원클릭으로 구독하세요',\n",
        "        r'▶ 한국경제신문과 WSJ, 모바일한경으로 보세요',\n",
        "        r'▶ 국내 최고의 \\'재테크 전문가\\'들과 만나는 공간',\n",
        "        r'▶ 한국경제앱 다운받고 \\‘암호화폐\\’ 받아가세요',\n",
        "        r'▶ 한국경제 구독신청-구독료 10% 암호화폐 적립',\n",
        "        r'ⓒ 한국경제 & hankyung\\.com, 무단전재 및 재배포 금지',\n",
        "        r'THE WALL STREET JOURNAL 한경 독점제휴',\n",
        "        r'이 기사는 한국경제신문과 금융 AI 전문기업 씽크풀이 공동 개발한 기사 자동생성 알고리즘에 의해 실시간으로 작성된 것입니다\\.',\n",
        "        r'\\‘와우밴드\\’ 앱을 설치한 후 종목 상담을 신청하면 와우넷 주간수익률 베스트파트너가 자세한 종목 진단을 해 드립니다\\.',\n",
        "    ]\n",
        "\n",
        "    # 단어 및 패턴을 제거하는 함수\n",
        "    def remove_patterns(text):\n",
        "        for pattern in patterns:\n",
        "            text = re.sub(pattern, '', text)\n",
        "        return text\n",
        "\n",
        "    # 패턴 제거\n",
        "    # text = remove_article(text)\n",
        "    text = remove_patterns(text)\n",
        "\n",
        "    # 3. 문장 단위로 split하여 리스트에 저장\n",
        "    def split_text(article):   # split하는 함수 정의\n",
        "        sentences = re.split(r'(?<!\\.)\\.(?!\\.)', article)  # 마침표를 기준으로 문장 나누기, 단 마침표가 2개 이상 연달아 있는 경우 제외\n",
        "        sentences = [sentence.strip() for sentence in sentences if sentence.strip()]   # 공백 제거 및 빈 문장 필터링\n",
        "        return sentences\n",
        "\n",
        "    # 각 문장들을 요소로 갖는 리스트인 text_list\n",
        "    text_list = split_text(text)\n",
        "\n",
        "    # 4. 특수 문자 제거 (◆, ◇, ▲, ▼ 제외)\n",
        "    text_list = [re.sub(r'[^가-힣0-9\\s◆◇▲▼~,.\\'\"%]', '', s) for s in text_list]  # 한글과 숫자, 공백, 그리고 ◆, ◇, ▲, ▼, ~, ', \", . , , % 만 남김\n",
        "\n",
        "    # 5. 띄어쓰기 교정\n",
        "    spacing = Spacing()\n",
        "    text_list = [spacing(s) for s in text_list]\n",
        "\n",
        "    # # # 6. 오타 수정\n",
        "    # def correct_text(sentence):\n",
        "    #     result = spell_checker.check(sentence)\n",
        "    #     return result.checked   # 수정된 문장 반환\n",
        "    # # 직접 spell_checker 라이브러리 코드 열어서 payload의 key값에 passportKey와 _callback 추가, value는 네이버 맞춤법 검사기 개발자 도구 참고하기\n",
        "    # # data 변수도 바꿔줘야함 -> 자세한건 노션에 서술\n",
        "\n",
        "    # text_list = [correct_text(s) for s in text_list]\n",
        "\n",
        "    #아래 오류가 났을 때의 코드 -> API 응답이 뜨지 않고 오류 발생이라고 뜨면 라이브러리 코드를 수정 안했거나 잘못한 것\n",
        "    '''\n",
        "    def correct_typos(sentence):\n",
        "        if not isinstance(sentence, str):\n",
        "            print(\"입력값이 문자열이 아닙니다.\")\n",
        "            return sentence  # 문자열이 아닐 경우 원본 반환\n",
        "        try:\n",
        "            result = spell_checker.check(sentence)\n",
        "            if not result.result:\n",
        "                print(f\"수정 실패: {result.original}\")\n",
        "                return sentence  # 수정 실패 시 원본 문장 반환\n",
        "            return result.checked  # 수정된 문장 반환\n",
        "        except Exception as e:\n",
        "            print(f\"오류 발생: {e}\")\n",
        "            return sentence  # 오류 발생 시 원본 문장 반환\n",
        "\n",
        "    corrected_text_list = []\n",
        "    for sentence in text_list:\n",
        "        try:\n",
        "            result = spell_checker.check(sentence)\n",
        "            print(f\"API 응답: {result}\")  # API 응답 확인\n",
        "            corrected_text_list.append(result.checked)\n",
        "        except Exception as e:\n",
        "            print(f\"오류 발생: {e}\")\n",
        "            corrected_text_list.append(sentence)  # 오류 발생 시 원본 문장 반환\n",
        "\n",
        "    for original, corrected in zip(text_list, corrected_text_list):\n",
        "        print(f\"원본: {original}\\n수정: {corrected}\\n\")\n",
        "    '''\n",
        "\n",
        "    # 7. 불용어 제거\n",
        "    stop_words = set(stopwords.words('korean'))   # korean 불용어 파일 만들어져 있어야 함\n",
        "    def remove_stopwords(text):   # 불용어 제거 함수 정의\n",
        "        return ' '.join([word for word in text.split() if word not in stop_words])\n",
        "    text_list = [remove_stopwords(s) + '.' for s in text_list]   # 문장 마지막에 다시 온점 붙여서 반환\n",
        "\n",
        "    # 8. 분리된 문장들 다시 하나의 텍스트로 합치기\n",
        "    text = ' '.join(text_list)   # 다시 하나의 문자열로 합쳐진걸 원하면 return text 하면 됨\n",
        "\n",
        "    return text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSImGnuLcnUN"
      },
      "source": [
        "# 특정 연도 csv 파일 하나만 돌릴 때 아래 코드 사용"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "dEnAfYsbcnUO",
        "outputId": "a6ec6353-51d6-4f43-9d44-21fb8327c151"
      },
      "outputs": [
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-9cce010e2570>\u001b[0m in \u001b[0;36m<cell line: 27>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;31m# 'title'과 'body' 컬럼에 클렌징 함수 적용\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'cleaned_title'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'title'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclean_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'cleaned_body'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'body'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclean_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;31m# 결과 확인\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, by_row, **kwargs)\u001b[0m\n\u001b[1;32m   4762\u001b[0m             \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4763\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4764\u001b[0;31m         ).apply()\n\u001b[0m\u001b[1;32m   4765\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4766\u001b[0m     def _reindex_indexer(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m         \u001b[0;31m# self.func is Callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1209\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1211\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/apply.py\u001b[0m in \u001b[0;36mapply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1287\u001b[0m         \u001b[0;31m#  Categorical (GH51645).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m         \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"ignore\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCategoricalDtype\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1289\u001b[0;31m         mapped = obj._map_values(\n\u001b[0m\u001b[1;32m   1290\u001b[0m             \u001b[0mmapper\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcurried\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mna_action\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1291\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m_map_values\u001b[0;34m(self, mapper, na_action, convert)\u001b[0m\n\u001b[1;32m    919\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mna_action\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mna_action\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    920\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 921\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0malgorithms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mna_action\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mna_action\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    922\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    923\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pandas/core/algorithms.py\u001b[0m in \u001b[0;36mmap_array\u001b[0;34m(arr, mapper, na_action, convert)\u001b[0m\n\u001b[1;32m   1812\u001b[0m     \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1813\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mna_action\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1814\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1815\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1816\u001b[0m         return lib.map_infer_mask(\n",
            "\u001b[0;32mlib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<ipython-input-3-55bd40d788ed>\u001b[0m in \u001b[0;36mclean_text\u001b[0;34m(text)\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;31m# 5. 띄어쓰기 교정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0mspacing\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSpacing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m     \u001b[0mtext_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mspacing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m     \u001b[0;31m# # # 6. 오타 수정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-3-55bd40d788ed>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;31m# 5. 띄어쓰기 교정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0mspacing\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSpacing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m     \u001b[0mtext_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mspacing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m     \u001b[0;31m# # # 6. 오타 수정\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pykospacing/kospacing.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, sent, ignore, ignore_pattern)\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0;31m# if ignore == 'post', set post_process to True\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m             \u001b[0mpost_process\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mignore\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'post'\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m             \u001b[0mspaced_sent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_spaced_sent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiltered_sent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeleted_str_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeleted_idx_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_sent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpost_process\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m             \u001b[0mresult_sent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspaced_sent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m         \u001b[0mspaced_sent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_sent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pykospacing/kospacing.py\u001b[0m in \u001b[0;36mget_spaced_sent\u001b[0;34m(self, raw_sent, deleted_str_list, deleted_idx_list, orig_sent, post_process)\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmat_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0mmat_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'output_0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'1'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0.5\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m'0'\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmat_set\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_sent_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0morig_sent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0morig_sent_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"«\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0morig_sent\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"»\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pykospacing/kospacing.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmat_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0mmat_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'output_0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'1'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0.5\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m'0'\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmat_set\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_sent_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0morig_sent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m             \u001b[0morig_sent_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"«\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0morig_sent\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"»\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m__bool__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    312\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__bool__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 313\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m   \u001b[0m__nonzero__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m__bool__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    371\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mnpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mArrayLike\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 373\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    374\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import time\n",
        "\n",
        "# CSV 파일을 데이터프레임으로 불러오기\n",
        "start = time.time()\n",
        "df = pd.read_csv('2007_한국경제.csv')\n",
        "\n",
        "# 데이터 정제 ) null값과 중복 제거\n",
        "df.dropna(subset=['body'], inplace=True)\n",
        "df.drop_duplicates(inplace=True)\n",
        "\n",
        "# 패턴 목록 정의\n",
        "patterns = [\n",
        "    r'<표>',\n",
        "    r'\\[표\\]',\n",
        "    r'\\[포토\\]',\n",
        "    r'무신불립',\n",
        "]\n",
        "\n",
        "# 패턴이 포함된 행을 삭제하기 위한 조건식 작성\n",
        "condition1 = df['title'].apply(lambda text: any(re.search(pattern, text) for pattern in patterns))\n",
        "df = df[~condition1]\n",
        "condition2 = df['body'].apply(lambda text: any(re.search(pattern, text) for pattern in patterns))\n",
        "df = df[~condition2]\n",
        "\n",
        "# 'title'과 'body' 컬럼에 클렌징 함수 적용\n",
        "df['cleaned_title'] = df['title'].apply(clean_text)\n",
        "df['cleaned_body'] = df['body'].apply(clean_text)\n",
        "\n",
        "# 결과 확인\n",
        "print(df[['body', 'cleaned_body']].head())\n",
        "\n",
        "# 클렌징된 데이터프레임 저장 (원하는 파일명으로 변경)\n",
        "df.to_csv('cleaned_2007_한국경제.csv', index=False, encoding='utf-8-sig')\n",
        "\n",
        "end = time.time()\n",
        "print(end - start)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6VbLaLfqcnUQ"
      },
      "source": [
        "# 여러 연도 csv 파일 반복문으로 돌릴 때 아래 코드 사용"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u-_75UBPcnUS"
      },
      "outputs": [],
      "source": [
        "# 파일 경로 및 범위 설정\n",
        "import os\n",
        "start_year = 2007\n",
        "end_year = 2009\n",
        "\n",
        "# 모든 연도의 CSV 파일을 처리\n",
        "for year in range(start_year, end_year + 1):\n",
        "    filename = f'{year}_한국경제.csv'\n",
        "\n",
        "    if os.path.exists(filename):\n",
        "        print(f\"Processing file: {filename}\")\n",
        "\n",
        "        # CSV 파일을 데이터프레임으로 불러오기\n",
        "        df = pd.read_csv(filename)\n",
        "\n",
        "        df.dropna(subset=['body'], inplace=True)\n",
        "        df.drop_duplicates(inplace=True)\n",
        "\n",
        "        # 패턴 목록 정의\n",
        "        patterns = [\n",
        "            r'<표>',\n",
        "            r'\\[표\\]',\n",
        "            r'\\[포토\\]',\n",
        "            r'무신불립',\n",
        "        ]\n",
        "\n",
        "        # 패턴이 포함된 행을 삭제하기 위한 조건식 작성\n",
        "        condition1 = df['title'].apply(lambda text: any(re.search(pattern, text) for pattern in patterns))\n",
        "        df = df[~condition1]\n",
        "        condition2 = df['body'].apply(lambda text: any(re.search(pattern, text) for pattern in patterns))\n",
        "        df = df[~condition2]\n",
        "\n",
        "        # 'body'와 'title' 컬럼에 클렌징 함수 적용\n",
        "        df['cleaned_title'] = df['title'].apply(clean_text)\n",
        "        df['cleaned_body'] = df['body'].apply(clean_text)\n",
        "\n",
        "        # 클렌징된 데이터프레임 저장\n",
        "        output_filename = f'cleaned_{year}_한국경제.csv'\n",
        "        df.to_csv(output_filename, index=False, encoding='utf-8-sig')\n",
        "\n",
        "        print(f\"Saved cleaned file: {output_filename}\")\n",
        "\n",
        "    else:\n",
        "        print(f\"File not found: {filename}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uVBY5XTPODkw"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}