classify_claimreview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classify Selected ClaimReview Claims"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import datetime\n",
    "import h5py\n",
    "import logging\n",
    "import argparse\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import keras\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, Activation, Input\n",
    "from tensorflow.keras.layers import BatchNormalization, LayerNormalization\n",
    "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn import metrics\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from joblib import dump, load\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import config\n",
    "from utils.file import directory_check"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prediction int to string\n",
    "def prediction_value(pred):\n",
    "    value_dict = {'0': 'dob', '1': 'education', '2': 'institution', '3':'pob', '4':'pod'}\n",
    "    index = None\n",
    "    num = max(max(pred))\n",
    "    for i,item in enumerate(pred[0]):\n",
    "        if item == num:\n",
    "            index = i\n",
    "    return value_dict[str(index)]\n",
    "\n",
    "# Adds prediction to dataframe\n",
    "def predict_df(X):\n",
    "    vec = X[\"Short_Path\"]\n",
    "    shape = vec.shape[0]\n",
    "    vec = np.reshape(vec, (shape,1)).T\n",
    "    prediction = prediction_value(MODEL.predict(vec))\n",
    "    \n",
    "    return prediction\n",
    "\n",
    "# Adds prediction evaluation to dataframe\n",
    "def predict_success(X):\n",
    "    prediction = X['Prediction']\n",
    "    actual = X['Relation']\n",
    "    \n",
    "    if prediction == actual:\n",
    "        return \"Yes\"\n",
    "    else:\n",
    "        return \"No\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load trained model and dataframe of shortest paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_f = '/sp_model-dnn_wide-dnn_wide-cr-d256-wl50-nw200-win15-p2-q3-201119.h5'\n",
    "MODEL = tf.keras.models.load_model(config.MODEL_DIR + model_f) \n",
    "\n",
    "sp_file = '/sp_df-cr-d256-wl50-nw200-win15-p2-q3-201118.pkl'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modify Dataframe for Prediction task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Object</th>\n",
       "      <th>Relation</th>\n",
       "      <th>Maj_Vote</th>\n",
       "      <th>Short_Path</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Successful?</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pob_cr1</td>\n",
       "      <td>Penny Wong</td>\n",
       "      <td>Malaysia</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.12060831, -0.004389023, 0.016019436, 0.095...</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pob_cr2</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.11935643, 0.0055734552, 0.032190174, -0.01...</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pob_cr3</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>United States</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.022086723, 0.010196335, 0.033510454, 0.000...</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pob_cr4</td>\n",
       "      <td>Fred Trump</td>\n",
       "      <td>Germany</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.08399602, -0.037151054, 0.017798765, 0.057...</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>dob_cr5</td>\n",
       "      <td>Barron Trump</td>\n",
       "      <td>2006</td>\n",
       "      <td>dob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.14999631, -0.032707285, -0.07424754, 0.104...</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        UID       Subject         Object Relation Maj_Vote  \\\n",
       "ID                                                           \n",
       "1   pob_cr1    Penny Wong       Malaysia      pob      yes   \n",
       "2   pob_cr2  Barack Obama          Kenya      pob      yes   \n",
       "3   pob_cr3  Barack Obama  United States      pob      yes   \n",
       "4   pob_cr4    Fred Trump        Germany      pob      yes   \n",
       "5   dob_cr5  Barron Trump           2006      dob      yes   \n",
       "\n",
       "                                           Short_Path Prediction Successful?  \n",
       "ID                                                                            \n",
       "1   [-0.12060831, -0.004389023, 0.016019436, 0.095...       None          No  \n",
       "2   [-0.11935643, 0.0055734552, 0.032190174, -0.01...       None          No  \n",
       "3   [-0.022086723, 0.010196335, 0.033510454, 0.000...       None          No  \n",
       "4   [-0.08399602, -0.037151054, 0.017798765, 0.057...       None          No  \n",
       "5   [-0.14999631, -0.032707285, -0.07424754, 0.104...       None          No  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle(config.SP_DIR + sp_file)\n",
    "\n",
    "df = df[df['Relation']=='cr'] # Remove all non-ClaimReview Claims\n",
    "\n",
    "# Sort entries by UID\n",
    "df['ID'] = df['UID'].apply(lambda x: x.split('_')[-1].split('r')[-1]).astype(int)\n",
    "df = df.set_index('ID')\n",
    "df = df.sort_index()\n",
    "df['Relation']=df['UID'].apply(lambda x: x.split('_')[0])\n",
    "\n",
    "# Add Two new Columns for Prediction analysis\n",
    "df['Prediction'] = 'None'\n",
    "df['Successful?'] = 'No'\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in encoder and prep data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_file = config.SP_SPLITS_DIR + '/class_encoder.npy'\n",
    "encoder = LabelEncoder()\n",
    "encoder.classes_ = np.load(encoder_file, allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/mjsumpter/miniconda3/envs/re/lib/python3.8/site-packages/sklearn/utils/validation.py:72: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  return f(**kwargs)\n"
     ]
    }
   ],
   "source": [
    "X = df[\"Short_Path\"].apply(pd.Series)\n",
    "\n",
    "y = df[['Relation']]\n",
    "y = encoder.transform(y)\n",
    "y = to_categorical(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.958\n",
      "Accuracy: 71.43%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UID</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Object</th>\n",
       "      <th>Relation</th>\n",
       "      <th>Maj_Vote</th>\n",
       "      <th>Short_Path</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Successful?</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pob_cr1</td>\n",
       "      <td>Penny Wong</td>\n",
       "      <td>Malaysia</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.12060831, -0.004389023, 0.016019436, 0.095...</td>\n",
       "      <td>dob</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pob_cr2</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.11935643, 0.0055734552, 0.032190174, -0.01...</td>\n",
       "      <td>institution</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pob_cr3</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>United States</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.022086723, 0.010196335, 0.033510454, 0.000...</td>\n",
       "      <td>pob</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pob_cr4</td>\n",
       "      <td>Fred Trump</td>\n",
       "      <td>Germany</td>\n",
       "      <td>pob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.08399602, -0.037151054, 0.017798765, 0.057...</td>\n",
       "      <td>pob</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>dob_cr5</td>\n",
       "      <td>Barron Trump</td>\n",
       "      <td>2006</td>\n",
       "      <td>dob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.14999631, -0.032707285, -0.07424754, 0.104...</td>\n",
       "      <td>pob</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>dob_cr6</td>\n",
       "      <td>Isabelle Duterte</td>\n",
       "      <td>January 26, 2002</td>\n",
       "      <td>dob</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.04258087, -0.14339234, 0.0050823386, 0.008...</td>\n",
       "      <td>dob</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education_cr7</td>\n",
       "      <td>Tej Pratap Yadav</td>\n",
       "      <td>Doctorate Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.09566146, -0.04633151, -0.0026847243, 0.02...</td>\n",
       "      <td>education</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>education_cr8</td>\n",
       "      <td>Smriti Irani</td>\n",
       "      <td>MA Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.073904075, -0.0076472485, 0.050191842, 0.0...</td>\n",
       "      <td>institution</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>education_cr9</td>\n",
       "      <td>Melania Trump</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.10019517, -0.05538463, -0.023950523, 0.043...</td>\n",
       "      <td>institution</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>education_cr10</td>\n",
       "      <td>Michelle Obama</td>\n",
       "      <td>Juris Doctor</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.07963335, 0.0015545988, 0.04472027, 0.0461...</td>\n",
       "      <td>education</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>education_cr11</td>\n",
       "      <td>Pravan Gordhan</td>\n",
       "      <td>Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.056872778, -0.025289508, 0.07884932, 0.046...</td>\n",
       "      <td>education</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>education_cr12</td>\n",
       "      <td>Alexandria Ocasio-Cortez</td>\n",
       "      <td>Economics Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.09500959, -0.001853646, 0.004757661, 0.062...</td>\n",
       "      <td>institution</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>education_cr13</td>\n",
       "      <td>Imee Marcos</td>\n",
       "      <td>Bachelor's Degree</td>\n",
       "      <td>education</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.056798395, 0.014900852, 0.089383245, 0.039...</td>\n",
       "      <td>education</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>institution_cr14</td>\n",
       "      <td>Imee Marcos</td>\n",
       "      <td>Princeton University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.0549962, 0.025464775, 0.055563904, 0.03784...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>institution_cr15</td>\n",
       "      <td>Tej Pratap Yadav</td>\n",
       "      <td>Takshsila University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.10572188, -0.050793696, 0.00036571862, 0.0...</td>\n",
       "      <td>education</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>institution_cr16</td>\n",
       "      <td>Patrick Murphy</td>\n",
       "      <td>University of Miami</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.08051448, -0.08235108, -0.12463131, 0.0750...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>institution_cr17</td>\n",
       "      <td>Vladimir Putin</td>\n",
       "      <td>Patrice Lumumba University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.1198068, -0.1268384, -0.07182188, -0.03225...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>institution_cr18</td>\n",
       "      <td>Marina Butina</td>\n",
       "      <td>American University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.015518223, -0.09944429, 0.050411142, 0.066...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>institution_cr19</td>\n",
       "      <td>Imee Marcos</td>\n",
       "      <td>University of the Philippines</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.015078805, -0.08362012, 0.0044884505, -0.0...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>institution_cr20</td>\n",
       "      <td>David Hogg</td>\n",
       "      <td>Redondo Shores Continuation High School</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.031023895, -0.06987886, 0.032975923, 0.031...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>pod_cr21</td>\n",
       "      <td>Manohar Parrikar</td>\n",
       "      <td>Goa</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.057718426, -0.05839586, -0.0493096, 0.0290...</td>\n",
       "      <td>pod</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>pod_cr22</td>\n",
       "      <td>Tucker Carlson</td>\n",
       "      <td>Washington D.C.</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.070726745, 0.010831918, 0.01879863, 0.0026...</td>\n",
       "      <td>pod</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>pod_cr23</td>\n",
       "      <td>Nasser Al Kharafi</td>\n",
       "      <td>Kuwait</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[0.011382176, 0.03712737, -0.038807902, 0.0200...</td>\n",
       "      <td>pod</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>pod_cr24</td>\n",
       "      <td>Amit Sharma</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.010342093, -0.10143781, 0.040919952, 0.022...</td>\n",
       "      <td>institution</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>pod_cr25</td>\n",
       "      <td>Jason Statham</td>\n",
       "      <td>New York</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.082907505, -0.022500908, -0.02242381, -0.0...</td>\n",
       "      <td>pod</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>pod_cr26</td>\n",
       "      <td>Robert Downey Jr.</td>\n",
       "      <td>Hollywood</td>\n",
       "      <td>pod</td>\n",
       "      <td>yes</td>\n",
       "      <td>[0.0017659222, -0.010442543, -0.04193017, 0.06...</td>\n",
       "      <td>pod</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>institution_cr27</td>\n",
       "      <td>Mahmoud Abbas</td>\n",
       "      <td>Patrice Lumumba University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.11917672, -0.12843396, -0.068598576, -0.03...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>institution_cr28</td>\n",
       "      <td>Ali Kamenei</td>\n",
       "      <td>Patrice Lumumba University</td>\n",
       "      <td>institution</td>\n",
       "      <td>yes</td>\n",
       "      <td>[-0.12323489, -0.12734397, -0.060544815, -0.03...</td>\n",
       "      <td>institution</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 UID                   Subject  \\\n",
       "ID                                               \n",
       "1            pob_cr1                Penny Wong   \n",
       "2            pob_cr2              Barack Obama   \n",
       "3            pob_cr3              Barack Obama   \n",
       "4            pob_cr4                Fred Trump   \n",
       "5            dob_cr5              Barron Trump   \n",
       "6            dob_cr6          Isabelle Duterte   \n",
       "7      education_cr7          Tej Pratap Yadav   \n",
       "8      education_cr8              Smriti Irani   \n",
       "9      education_cr9             Melania Trump   \n",
       "10    education_cr10            Michelle Obama   \n",
       "11    education_cr11            Pravan Gordhan   \n",
       "12    education_cr12  Alexandria Ocasio-Cortez   \n",
       "13    education_cr13               Imee Marcos   \n",
       "14  institution_cr14               Imee Marcos   \n",
       "15  institution_cr15          Tej Pratap Yadav   \n",
       "16  institution_cr16            Patrick Murphy   \n",
       "17  institution_cr17            Vladimir Putin   \n",
       "18  institution_cr18             Marina Butina   \n",
       "19  institution_cr19               Imee Marcos   \n",
       "20  institution_cr20                David Hogg   \n",
       "21          pod_cr21          Manohar Parrikar   \n",
       "22          pod_cr22            Tucker Carlson   \n",
       "23          pod_cr23         Nasser Al Kharafi   \n",
       "24          pod_cr24               Amit Sharma   \n",
       "25          pod_cr25             Jason Statham   \n",
       "26          pod_cr26         Robert Downey Jr.   \n",
       "27  institution_cr27             Mahmoud Abbas   \n",
       "28  institution_cr28               Ali Kamenei   \n",
       "\n",
       "                                     Object     Relation Maj_Vote  \\\n",
       "ID                                                                  \n",
       "1                                  Malaysia          pob      yes   \n",
       "2                                     Kenya          pob      yes   \n",
       "3                             United States          pob      yes   \n",
       "4                                   Germany          pob      yes   \n",
       "5                                      2006          dob      yes   \n",
       "6                          January 26, 2002          dob      yes   \n",
       "7                          Doctorate Degree    education      yes   \n",
       "8                                 MA Degree    education      yes   \n",
       "9                         Bachelor's Degree    education      yes   \n",
       "10                             Juris Doctor    education      yes   \n",
       "11                                   Degree    education      yes   \n",
       "12                         Economics Degree    education      yes   \n",
       "13                        Bachelor's Degree    education      yes   \n",
       "14                     Princeton University  institution      yes   \n",
       "15                     Takshsila University  institution      yes   \n",
       "16                      University of Miami  institution      yes   \n",
       "17               Patrice Lumumba University  institution      yes   \n",
       "18                      American University  institution      yes   \n",
       "19            University of the Philippines  institution      yes   \n",
       "20  Redondo Shores Continuation High School  institution      yes   \n",
       "21                                      Goa          pod      yes   \n",
       "22                          Washington D.C.          pod      yes   \n",
       "23                                   Kuwait          pod      yes   \n",
       "24                                    Delhi          pod      yes   \n",
       "25                                 New York          pod      yes   \n",
       "26                                Hollywood          pod      yes   \n",
       "27               Patrice Lumumba University  institution      yes   \n",
       "28               Patrice Lumumba University  institution      yes   \n",
       "\n",
       "                                           Short_Path   Prediction Successful?  \n",
       "ID                                                                              \n",
       "1   [-0.12060831, -0.004389023, 0.016019436, 0.095...          dob          No  \n",
       "2   [-0.11935643, 0.0055734552, 0.032190174, -0.01...  institution          No  \n",
       "3   [-0.022086723, 0.010196335, 0.033510454, 0.000...          pob         Yes  \n",
       "4   [-0.08399602, -0.037151054, 0.017798765, 0.057...          pob         Yes  \n",
       "5   [-0.14999631, -0.032707285, -0.07424754, 0.104...          pob          No  \n",
       "6   [-0.04258087, -0.14339234, 0.0050823386, 0.008...          dob         Yes  \n",
       "7   [-0.09566146, -0.04633151, -0.0026847243, 0.02...    education         Yes  \n",
       "8   [-0.073904075, -0.0076472485, 0.050191842, 0.0...  institution          No  \n",
       "9   [-0.10019517, -0.05538463, -0.023950523, 0.043...  institution          No  \n",
       "10  [-0.07963335, 0.0015545988, 0.04472027, 0.0461...    education         Yes  \n",
       "11  [-0.056872778, -0.025289508, 0.07884932, 0.046...    education         Yes  \n",
       "12  [-0.09500959, -0.001853646, 0.004757661, 0.062...  institution          No  \n",
       "13  [-0.056798395, 0.014900852, 0.089383245, 0.039...    education         Yes  \n",
       "14  [-0.0549962, 0.025464775, 0.055563904, 0.03784...  institution         Yes  \n",
       "15  [-0.10572188, -0.050793696, 0.00036571862, 0.0...    education          No  \n",
       "16  [-0.08051448, -0.08235108, -0.12463131, 0.0750...  institution         Yes  \n",
       "17  [-0.1198068, -0.1268384, -0.07182188, -0.03225...  institution         Yes  \n",
       "18  [-0.015518223, -0.09944429, 0.050411142, 0.066...  institution         Yes  \n",
       "19  [-0.015078805, -0.08362012, 0.0044884505, -0.0...  institution         Yes  \n",
       "20  [-0.031023895, -0.06987886, 0.032975923, 0.031...  institution         Yes  \n",
       "21  [-0.057718426, -0.05839586, -0.0493096, 0.0290...          pod         Yes  \n",
       "22  [-0.070726745, 0.010831918, 0.01879863, 0.0026...          pod         Yes  \n",
       "23  [0.011382176, 0.03712737, -0.038807902, 0.0200...          pod         Yes  \n",
       "24  [-0.010342093, -0.10143781, 0.040919952, 0.022...  institution          No  \n",
       "25  [-0.082907505, -0.022500908, -0.02242381, -0.0...          pod         Yes  \n",
       "26  [0.0017659222, -0.010442543, -0.04193017, 0.06...          pod         Yes  \n",
       "27  [-0.11917672, -0.12843396, -0.068598576, -0.03...  institution         Yes  \n",
       "28  [-0.12323489, -0.12734397, -0.060544815, -0.03...  institution         Yes  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Prediction\"] = df.apply(predict_df, axis=1)\n",
    "df[\"Successful?\"] = df.apply(predict_success, axis=1)\n",
    "predictions = MODEL.predict(X)\n",
    "print(f\"AUC: {round(metrics.roc_auc_score(y, predictions), 3)}\")\n",
    "\n",
    "acc = len(df[df['Successful?']=='Yes'])/len(df) \n",
    "print(f\"Accuracy: {round(acc*100,2)}%\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory_check(config.CLAIM_DIR)\n",
    "df.to_pickle(config.CLAIM_DF)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "re",
   "language": "python",
   "name": "re"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}