32_dgi_label_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6b94b07d-c145-44ab-8566-1d5a793bf077",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Modify the file 00_setup.py to define input/output file paths on your system\n",
    "# The information in 00_setup.py will be used across notebooks\n",
    "from importlib.machinery import SourceFileLoader\n",
    "setup = SourceFileLoader(\"setup\", \"./00_setup.py\").load_module()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a58b5dd-6863-43f9-b28c-da9378afb1d7",
   "metadata": {},
   "source": [
    "# GNN - Unsup Model Data, With Label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20b91454-640d-4055-aac4-067fe1fe58b8",
   "metadata": {},
   "source": [
    "Create Stellargraph objects for unsupervised GNN runs, including original features, plus the target as a feature."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c447a772-dc1a-4157-8d34-08e3f99ebfe6",
   "metadata": {},
   "source": [
    "For the unseen / test data, randomly set the target value in the feature"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce6fed7e-3548-4bee-8d57-9244be1222b4",
   "metadata": {},
   "source": [
    "*This script takes about 15 minutes on my MacBook Air*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "52c010c9-235e-4845-bc23-dd9b82acc741",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dd932005-3f42-434a-a7e3-34d6fdab71b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import importlib\n",
    "import pickle, re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3c0a0d65-40df-4f60-a5ba-5a19d817765c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sba_gnn.sba_gnn import sg_gnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59ba58e0-e2a1-4b3d-b125-d77a463fab1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-16 13:51:35.550796: I metal_plugin/src/device/metal_device.cc:1154] Metal device set to: Apple M1\n",
      "2024-03-16 13:51:35.550818: I metal_plugin/src/device/metal_device.cc:296] systemMemory: 16.00 GB\n",
      "2024-03-16 13:51:35.550826: I metal_plugin/src/device/metal_device.cc:313] maxCacheSize: 5.33 GB\n",
      "2024-03-16 13:51:35.550862: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:303] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2024-03-16 13:51:35.550887: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:269] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    }
   ],
   "source": [
    "import stellargraph as sg\n",
    "from stellargraph import StellarGraph"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4da4b71d-5108-49e6-80e1-521c23def729",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8b7ce228-82c7-4943-930d-72a171b48195",
   "metadata": {},
   "outputs": [],
   "source": [
    "## NN scaled input data\n",
    "comb_df = pd.read_parquet(Path(setup.temp_path).joinpath('20_DATA_combined_scaled_all.parquet'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92b9e3ec-b772-4c70-9cf8-c88186ac110f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(Path(setup.temp_path).joinpath('20_DATA_features.pkl'), 'rb') as fin:\n",
    "    imputed_features = pickle.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3ca13685-f529-45e7-8d3c-c4f7f23b972d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NoEmp', 'CreateJob', 'LowDoc', 'DisbursementGross', 'new_business', 'urban_flag', 'franchise_flag', 'missingindicator_LowDoc', 'missingindicator_new_business', 'missingindicator_urban_flag']\n"
     ]
    }
   ],
   "source": [
    "num_feat =  [c for c in imputed_features if 'NAICS' not in c]\n",
    "print(num_feat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cb738b9-df77-4397-9cd9-bced52bf6fb7",
   "metadata": {},
   "source": [
    "###### Data used to create previous graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ce454e58-663d-4dfa-af83-444f00825eab",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges_all = pd.read_parquet(Path(setup.temp_path).joinpath('30_DATA_graph_edges.parquet'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b8c4df72-df64-44cb-8f6d-17e2b8ee5907",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_features_business = pd.read_parquet(Path(setup.temp_path).joinpath('30_DATA_graph_node_features_loans.parquet'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "97e53db9-ebf6-40ec-b7ff-400c7232b324",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_features_naics = pd.read_parquet(Path(setup.temp_path).joinpath('30_DATA_graph_node_features_naics.parquet'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eb23221-aaec-4aa5-b19c-86441375b4fc",
   "metadata": {},
   "source": [
    "## Add target to node features\n",
    "Add target data to the features for the train and validation data.  For test, randomly set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9df872ce-b9ae-48d7-88f3-a0cb247336c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "targ_feat_train_val = comb_df[comb_df['dset'].isin(['train', 'val'])] \\\n",
    "    [['LoanNr_ChkDgt', 'target', 'dset']] \\\n",
    "    .set_index('LoanNr_ChkDgt') \\\n",
    "    .rename(columns={'target':'feature_target'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3ca69ca4-9be1-4647-aff4-81a3152a786e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2046546821975603"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_target_rate = targ_feat_train_val[targ_feat_train_val['dset']=='train']['feature_target'].mean()\n",
    "base_target_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b306737f-b6c1-4e58-8c0a-70bf73a8df02",
   "metadata": {},
   "outputs": [],
   "source": [
    "targ_feat_test = comb_df[comb_df['dset'] == 'test'] \\\n",
    "    [['LoanNr_ChkDgt', 'dset']] \\\n",
    "    .set_index('LoanNr_ChkDgt') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a5172b98-47e6-4c77-b69c-20b037e39f5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1234)\n",
    "targ_feat_test['feature_target'] = np.random.choice([1, 0], \n",
    "                                                    p = [base_target_rate, 1-base_target_rate],\n",
    "                                                    size = len(targ_feat_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "48b81112-d201-4382-a3d9-bc76c2cb62e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "targ_feat = pd.concat([targ_feat_train_val, targ_feat_test]).sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6931fe16-5788-4156-be7d-68bdbb7dc49f",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_features_business = pd.concat([node_features_business.sort_index(),\n",
    "                                    targ_feat[['feature_target']]], axis=1)\n",
    "node_features_business.to_parquet(Path(setup.temp_path).joinpath('32_DATA_graph_node_features_loans.parquet'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "66e2a5f4-d370-4349-b49f-627cb55845b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dset</th>\n",
       "      <th>NAICS_orig</th>\n",
       "      <th>NoEmp</th>\n",
       "      <th>CreateJob</th>\n",
       "      <th>LowDoc</th>\n",
       "      <th>DisbursementGross</th>\n",
       "      <th>new_business</th>\n",
       "      <th>urban_flag</th>\n",
       "      <th>franchise_flag</th>\n",
       "      <th>missingindicator_LowDoc</th>\n",
       "      <th>missingindicator_new_business</th>\n",
       "      <th>missingindicator_urban_flag</th>\n",
       "      <th>feature_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LoanNr_ChkDgt</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1000014003</th>\n",
       "      <td>train</td>\n",
       "      <td>451120</td>\n",
       "      <td>0.024024</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.195195</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000024006</th>\n",
       "      <td>test</td>\n",
       "      <td>722410</td>\n",
       "      <td>-0.447447</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.451451</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000034009</th>\n",
       "      <td>train</td>\n",
       "      <td>621210</td>\n",
       "      <td>0.381381</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.620502</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                dset NAICS_orig     NoEmp  CreateJob  LowDoc  \\\n",
       "LoanNr_ChkDgt                                                  \n",
       "1000014003     train     451120  0.024024       -1.0     1.0   \n",
       "1000024006      test     722410 -0.447447       -1.0     1.0   \n",
       "1000034009     train     621210  0.381381       -1.0    -1.0   \n",
       "\n",
       "               DisbursementGross  new_business  urban_flag  franchise_flag  \\\n",
       "LoanNr_ChkDgt                                                                \n",
       "1000014003             -0.195195           1.0         1.0            -1.0   \n",
       "1000024006             -0.451451           1.0         1.0            -1.0   \n",
       "1000034009              0.620502          -1.0         1.0            -1.0   \n",
       "\n",
       "               missingindicator_LowDoc  missingindicator_new_business  \\\n",
       "LoanNr_ChkDgt                                                           \n",
       "1000014003                        -1.0                           -1.0   \n",
       "1000024006                        -1.0                           -1.0   \n",
       "1000034009                        -1.0                           -1.0   \n",
       "\n",
       "               missingindicator_urban_flag  feature_target  \n",
       "LoanNr_ChkDgt                                               \n",
       "1000014003                             1.0               0  \n",
       "1000024006                             1.0               1  \n",
       "1000034009                             1.0               0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_features_business.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95cf8304-a674-49d4-b873-b56e9bf53f42",
   "metadata": {},
   "source": [
    "## Stellargraph - train and validation\n",
    "Graph without test nodes (make sure set-aside NAICS not included)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1dc0aa32-b79a-46f9-881b-7499fa3d1977",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get only relevant edges and node data\n",
    "edges_train_val, features_business_train_val, features_naics_train_val = \\\n",
    "    sg_gnn.limit_data(edges_all, node_features_business, node_features_naics.drop(columns='NAICS_orig'),\n",
    "                     ['train', 'val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f509751c-45d8-4f2e-ae64-47985af49eb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train+val graph data\n",
      "business features, start rows 688081, end 529574\n",
      "naics features, start rows 1311, end 1173\n",
      "edges, start rows 688081, end 529574\n"
     ]
    }
   ],
   "source": [
    "print(f'Train+val graph data')\n",
    "print(f'business features, start rows {node_features_business.shape[0]}, end {features_business_train_val.shape[0]}')\n",
    "print(f'naics features, start rows {node_features_naics.shape[0]}, end {features_naics_train_val.shape[0]}')\n",
    "print(f'edges, start rows {edges_all.shape[0]}, end {edges_train_val.shape[0]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8dbf7ca3-42f9-4a4b-bfdf-74d37aad9a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "sba_graph_train_val = StellarGraph({'LoanNr_ChkDgt':features_business_train_val.drop(columns='NAICS_orig'),\n",
    "                                    'NAICS': features_naics_train_val},\n",
    "                                   edges_train_val, \n",
    "                                   source_column=\"source\", target_column=\"target\",\n",
    "                                   edge_type_column=\"type\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "410b7928-83b4-428a-af5b-05ecb9c7a748",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StellarGraph: Undirected multigraph\n",
      " Nodes: 530747, Edges: 529574\n",
      "\n",
      " Node types:\n",
      "  LoanNr_ChkDgt: [529574]\n",
      "    Features: float32 vector, length 11\n",
      "    Edge types: LoanNr_ChkDgt-loan_naics->NAICS\n",
      "  NAICS: [1173]\n",
      "    Features: float32 vector, length 1\n",
      "    Edge types: NAICS-loan_naics->LoanNr_ChkDgt\n",
      "\n",
      " Edge types:\n",
      "    LoanNr_ChkDgt-loan_naics->NAICS: [529574]\n",
      "        Weights: all 1 (default)\n",
      "        Features: none\n"
     ]
    }
   ],
   "source": [
    "print(sba_graph_train_val.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "99b1b627-0cc0-492d-91d3-f9d92e2136e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save stellargraph object\n",
    "with open(Path(setup.temp_path).joinpath('32_DATA_stellargraph_train_val.pkl'), 'wb') as fout:\n",
    "      pickle.dump(sba_graph_train_val, fout)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e82b9311-7f4e-4cab-92a7-633670a0398f",
   "metadata": {},
   "source": [
    "## Stellargraph - all nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2fcf66ec-e079-4c91-a0cb-70439013577d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sba_graph_all = StellarGraph({'LoanNr_ChkDgt':node_features_business.drop(columns=['dset', 'NAICS_orig']) ,\n",
    "                              'NAICS': node_features_naics.drop(columns='NAICS_orig')},\n",
    "                             edges_all.drop(columns=['dset']).drop_duplicates(),\n",
    "                             source_column=\"source\", target_column=\"target\",\n",
    "                             edge_type_column=\"type\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "63089961-2b1f-4312-aafe-e2e7de4feede",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StellarGraph: Undirected multigraph\n",
      " Nodes: 689392, Edges: 688081\n",
      "\n",
      " Node types:\n",
      "  LoanNr_ChkDgt: [688081]\n",
      "    Features: float32 vector, length 11\n",
      "    Edge types: LoanNr_ChkDgt-loan_naics->NAICS\n",
      "  NAICS: [1311]\n",
      "    Features: float32 vector, length 1\n",
      "    Edge types: NAICS-loan_naics->LoanNr_ChkDgt\n",
      "\n",
      " Edge types:\n",
      "    LoanNr_ChkDgt-loan_naics->NAICS: [688081]\n",
      "        Weights: all 1 (default)\n",
      "        Features: none\n"
     ]
    }
   ],
   "source": [
    "print(sba_graph_all.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "18015753-3c65-4ffd-af45-00fa3e4a2427",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save stellargraph object\n",
    "with open(Path(setup.temp_path).joinpath('32_DATA_stellargraph_all.pkl'), 'wb') as fout:\n",
    "      pickle.dump(sba_graph_all, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e5ca7fb-3a51-4e31-9c6f-9b7beb016f7d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf_p39",
   "language": "python",
   "name": "tf_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}