UCI Data Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cfe6e11d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07190ea9",
   "metadata": {},
   "source": [
    "# UCI Dataset Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef4818fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "RAW_DATA_FOLDER = 'data/raw'\n",
    "CLEAN_DATA_FOLDER = 'data/preprocessed'\n",
    "\n",
    "def preprocess(train_df, \n",
    "               numerical_cols, category_cols, preserved_cols):\n",
    "    to_concat = []\n",
    "    \n",
    "    # numerical cols\n",
    "    if len(numerical_cols) > 0:\n",
    "        train_df[numerical_cols] = train_df[numerical_cols].astype(float)\n",
    "\n",
    "        mu = train_df[numerical_cols].mean(axis=0)\n",
    "        std = train_df[numerical_cols].std(axis=0)\n",
    "\n",
    "        train_nums = (train_df[numerical_cols] - mu)/std\n",
    "        \n",
    "        to_concat.append(train_nums)\n",
    "\n",
    "    # categorical cols\n",
    "    if len(category_cols) > 0:\n",
    "        train_cats = pd.get_dummies(\n",
    "            train_df[category_cols], \n",
    "            dummy_na=False, columns=None, sparse=False, drop_first=False, dtype=None)\n",
    "        to_concat.append(train_cats)\n",
    "    \n",
    "    # preserved_cols\n",
    "    if len(preserved_cols) > 0:\n",
    "        train_preserved = train_df[preserved_cols]\n",
    "        to_concat.append(train_preserved)\n",
    "    \n",
    "    preprocessed_train_df = pd.concat(to_concat, axis=1)\n",
    "    preprocessed_train_df = preprocessed_train_df.fillna(0)\n",
    "    \n",
    "    return preprocessed_train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f838dad",
   "metadata": {},
   "source": [
    "## Adult dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5366fa6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "NAME = 'adult'\n",
    "\n",
    "attributes = \"\"\"\n",
    "age: continuous.\n",
    "workclass: Private, Self-emp-not-inc, Self-emp-inc, Federal-gov, Local-gov, State-gov, Without-pay, Never-worked.\n",
    "fnlwgt: continuous.\n",
    "education: Bachelors, Some-college, 11th, HS-grad, Prof-school, Assoc-acdm, Assoc-voc, 9th, 7th-8th, 12th, Masters, 1st-4th, 10th, Doctorate, 5th-6th, Preschool.\n",
    "education-num: continuous.\n",
    "marital-status: Married-civ-spouse, Divorced, Never-married, Separated, Widowed, Married-spouse-absent, Married-AF-spouse.\n",
    "occupation: Tech-support, Craft-repair, Other-service, Sales, Exec-managerial, Prof-specialty, Handlers-cleaners, Machine-op-inspct, Adm-clerical, Farming-fishing, Transport-moving, Priv-house-serv, Protective-serv, Armed-Forces.\n",
    "relationship: Wife, Own-child, Husband, Not-in-family, Other-relative, Unmarried.\n",
    "race: White, Asian-Pac-Islander, Amer-Indian-Eskimo, Other, Black.\n",
    "sex: Female, Male.\n",
    "capital-gain: continuous.\n",
    "capital-loss: continuous.\n",
    "hours-per-week: continuous.\n",
    "native-country: United-States, Cambodia, England, Puerto-Rico, Canada, Germany, Outlying-US(Guam-USVI-etc), India, Japan, Greece, South, China, Cuba, Iran, Honduras, Philippines, Italy, Poland, Jamaica, Vietnam, Mexico, Portugal, Ireland, France, Dominican-Republic, Laos, Ecuador, Taiwan, Haiti, Columbia, Hungary, Guatemala, Nicaragua, Scotland, Thailand, Yugoslavia, El-Salvador, Trinadad&Tobago, Peru, Hong, Holand-Netherlands.\n",
    "\"\"\"\n",
    "cols = [s.split(':')[0] for s in attributes.split('\\n')] + ['label']\n",
    "cols = [c for c in cols if len(c) > 0]\n",
    "\n",
    "train_df = pd.read_csv(f'{RAW_DATA_FOLDER}/adult/adult.data', names=cols, index_col=False)\n",
    "test_df = pd.read_csv(f'{RAW_DATA_FOLDER}/adult/adult.test', names=cols, index_col=False).drop(0)\n",
    "combined_df = pd.concat([train_df, test_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b873b46f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48842, 15)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5b702e1d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age               0.000000\n",
      "workclass         0.057307\n",
      "fnlwgt            0.000000\n",
      "education         0.000000\n",
      "education-num     0.000000\n",
      "marital-status    0.000000\n",
      "occupation        0.057512\n",
      "relationship      0.000000\n",
      "race              0.000000\n",
      "sex               0.000000\n",
      "capital-gain      0.000000\n",
      "capital-loss      0.000000\n",
      "hours-per-week    0.000000\n",
      "native-country    0.017546\n",
      "label             0.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# preprocessing\n",
    "numerical_cols = ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', 'hours-per-week']\n",
    "category_cols = [\n",
    "    'workclass', 'education', 'marital-status', 'occupation', \n",
    "    'relationship', 'race', 'sex', 'native-country'\n",
    "]\n",
    "preserved_cols = ['label']\n",
    "\n",
    "combined_df[category_cols] = combined_df[category_cols].replace({'\\?': np.nan}, regex=True)\n",
    "\n",
    "# original missingness\n",
    "print(combined_df.isna().sum(axis=0) / len(combined_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "00afff73",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_df = preprocess(combined_df, numerical_cols, category_cols, preserved_cols)\n",
    "combined_df['label'] = combined_df['label'].replace({' <=50K': 0, ' >50K': 1, ' <=50K.': 0, ' >50K.': 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "490803ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education-num</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>workclass_ Federal-gov</th>\n",
       "      <th>workclass_ Local-gov</th>\n",
       "      <th>workclass_ Never-worked</th>\n",
       "      <th>workclass_ Private</th>\n",
       "      <th>workclass_ Self-emp-inc</th>\n",
       "      <th>workclass_ Self-emp-not-inc</th>\n",
       "      <th>workclass_ State-gov</th>\n",
       "      <th>workclass_ Without-pay</th>\n",
       "      <th>education_ 10th</th>\n",
       "      <th>education_ 11th</th>\n",
       "      <th>education_ 12th</th>\n",
       "      <th>education_ 1st-4th</th>\n",
       "      <th>education_ 5th-6th</th>\n",
       "      <th>education_ 7th-8th</th>\n",
       "      <th>education_ 9th</th>\n",
       "      <th>education_ Assoc-acdm</th>\n",
       "      <th>education_ Assoc-voc</th>\n",
       "      <th>education_ Bachelors</th>\n",
       "      <th>education_ Doctorate</th>\n",
       "      <th>education_ HS-grad</th>\n",
       "      <th>education_ Masters</th>\n",
       "      <th>education_ Preschool</th>\n",
       "      <th>education_ Prof-school</th>\n",
       "      <th>education_ Some-college</th>\n",
       "      <th>marital-status_ Divorced</th>\n",
       "      <th>marital-status_ Married-AF-spouse</th>\n",
       "      <th>marital-status_ Married-civ-spouse</th>\n",
       "      <th>marital-status_ Married-spouse-absent</th>\n",
       "      <th>marital-status_ Never-married</th>\n",
       "      <th>marital-status_ Separated</th>\n",
       "      <th>marital-status_ Widowed</th>\n",
       "      <th>occupation_ Adm-clerical</th>\n",
       "      <th>occupation_ Armed-Forces</th>\n",
       "      <th>occupation_ Craft-repair</th>\n",
       "      <th>occupation_ Exec-managerial</th>\n",
       "      <th>occupation_ Farming-fishing</th>\n",
       "      <th>occupation_ Handlers-cleaners</th>\n",
       "      <th>occupation_ Machine-op-inspct</th>\n",
       "      <th>occupation_ Other-service</th>\n",
       "      <th>occupation_ Priv-house-serv</th>\n",
       "      <th>occupation_ Prof-specialty</th>\n",
       "      <th>occupation_ Protective-serv</th>\n",
       "      <th>occupation_ Sales</th>\n",
       "      <th>occupation_ Tech-support</th>\n",
       "      <th>occupation_ Transport-moving</th>\n",
       "      <th>relationship_ Husband</th>\n",
       "      <th>relationship_ Not-in-family</th>\n",
       "      <th>relationship_ Other-relative</th>\n",
       "      <th>relationship_ Own-child</th>\n",
       "      <th>relationship_ Unmarried</th>\n",
       "      <th>relationship_ Wife</th>\n",
       "      <th>race_ Amer-Indian-Eskimo</th>\n",
       "      <th>race_ Asian-Pac-Islander</th>\n",
       "      <th>race_ Black</th>\n",
       "      <th>race_ Other</th>\n",
       "      <th>race_ White</th>\n",
       "      <th>sex_ Female</th>\n",
       "      <th>sex_ Male</th>\n",
       "      <th>native-country_ Cambodia</th>\n",
       "      <th>native-country_ Canada</th>\n",
       "      <th>native-country_ China</th>\n",
       "      <th>native-country_ Columbia</th>\n",
       "      <th>native-country_ Cuba</th>\n",
       "      <th>native-country_ Dominican-Republic</th>\n",
       "      <th>native-country_ Ecuador</th>\n",
       "      <th>native-country_ El-Salvador</th>\n",
       "      <th>native-country_ England</th>\n",
       "      <th>native-country_ France</th>\n",
       "      <th>native-country_ Germany</th>\n",
       "      <th>native-country_ Greece</th>\n",
       "      <th>native-country_ Guatemala</th>\n",
       "      <th>native-country_ Haiti</th>\n",
       "      <th>native-country_ Holand-Netherlands</th>\n",
       "      <th>native-country_ Honduras</th>\n",
       "      <th>native-country_ Hong</th>\n",
       "      <th>native-country_ Hungary</th>\n",
       "      <th>native-country_ India</th>\n",
       "      <th>native-country_ Iran</th>\n",
       "      <th>native-country_ Ireland</th>\n",
       "      <th>native-country_ Italy</th>\n",
       "      <th>native-country_ Jamaica</th>\n",
       "      <th>native-country_ Japan</th>\n",
       "      <th>native-country_ Laos</th>\n",
       "      <th>native-country_ Mexico</th>\n",
       "      <th>native-country_ Nicaragua</th>\n",
       "      <th>native-country_ Outlying-US(Guam-USVI-etc)</th>\n",
       "      <th>native-country_ Peru</th>\n",
       "      <th>native-country_ Philippines</th>\n",
       "      <th>native-country_ Poland</th>\n",
       "      <th>native-country_ Portugal</th>\n",
       "      <th>native-country_ Puerto-Rico</th>\n",
       "      <th>native-country_ Scotland</th>\n",
       "      <th>native-country_ South</th>\n",
       "      <th>native-country_ Taiwan</th>\n",
       "      <th>native-country_ Thailand</th>\n",
       "      <th>native-country_ Trinadad&amp;Tobago</th>\n",
       "      <th>native-country_ United-States</th>\n",
       "      <th>native-country_ Vietnam</th>\n",
       "      <th>native-country_ Yugoslavia</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.025996</td>\n",
       "      <td>-1.061968</td>\n",
       "      <td>1.136500</td>\n",
       "      <td>0.146931</td>\n",
       "      <td>-0.217125</td>\n",
       "      <td>-0.034087</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.828300</td>\n",
       "      <td>-1.007094</td>\n",
       "      <td>1.136500</td>\n",
       "      <td>-0.144802</td>\n",
       "      <td>-0.217125</td>\n",
       "      <td>-2.213009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.046941</td>\n",
       "      <td>0.246031</td>\n",
       "      <td>-0.419331</td>\n",
       "      <td>-0.144802</td>\n",
       "      <td>-0.217125</td>\n",
       "      <td>-0.034087</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.047110</td>\n",
       "      <td>0.426659</td>\n",
       "      <td>-1.197247</td>\n",
       "      <td>-0.144802</td>\n",
       "      <td>-0.217125</td>\n",
       "      <td>-0.034087</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.776309</td>\n",
       "      <td>1.408515</td>\n",
       "      <td>1.136500</td>\n",
       "      <td>-0.144802</td>\n",
       "      <td>-0.217125</td>\n",
       "      <td>-0.034087</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age    fnlwgt  education-num  capital-gain  capital-loss  hours-per-week  workclass_ Federal-gov  workclass_ Local-gov  workclass_ Never-worked  workclass_ Private  workclass_ Self-emp-inc  workclass_ Self-emp-not-inc  workclass_ State-gov  workclass_ Without-pay  education_ 10th  education_ 11th  education_ 12th  education_ 1st-4th  education_ 5th-6th  education_ 7th-8th  education_ 9th  education_ Assoc-acdm  education_ Assoc-voc  education_ Bachelors  education_ Doctorate  education_ HS-grad  education_ Masters  education_ Preschool  education_ Prof-school  education_ Some-college  marital-status_ Divorced  marital-status_ Married-AF-spouse  marital-status_ Married-civ-spouse  marital-status_ Married-spouse-absent  marital-status_ Never-married  marital-status_ Separated  marital-status_ Widowed  occupation_ Adm-clerical  occupation_ Armed-Forces  occupation_ Craft-repair  occupation_ Exec-managerial  occupation_ Farming-fishing  occupation_ Handlers-cleaners  \\\n",
       "0  0.025996 -1.061968       1.136500      0.146931     -0.217125       -0.034087                       0                     0                        0                   0                        0                            0                     1                       0                0                0                0                   0                   0                   0               0                      0                     0                     1                     0                   0                   0                     0                       0                        0                         0                                  0                                   0                                      0                              1                          0                        0                         1                         0                         0                            0                            0                              0   \n",
       "1  0.828300 -1.007094       1.136500     -0.144802     -0.217125       -2.213009                       0                     0                        0                   0                        0                            1                     0                       0                0                0                0                   0                   0                   0               0                      0                     0                     1                     0                   0                   0                     0                       0                        0                         0                                  0                                   1                                      0                              0                          0                        0                         0                         0                         0                            1                            0                              0   \n",
       "2 -0.046941  0.246031      -0.419331     -0.144802     -0.217125       -0.034087                       0                     0                        0                   1                        0                            0                     0                       0                0                0                0                   0                   0                   0               0                      0                     0                     0                     0                   1                   0                     0                       0                        0                         1                                  0                                   0                                      0                              0                          0                        0                         0                         0                         0                            0                            0                              1   \n",
       "3  1.047110  0.426659      -1.197247     -0.144802     -0.217125       -0.034087                       0                     0                        0                   1                        0                            0                     0                       0                0                1                0                   0                   0                   0               0                      0                     0                     0                     0                   0                   0                     0                       0                        0                         0                                  0                                   1                                      0                              0                          0                        0                         0                         0                         0                            0                            0                              1   \n",
       "4 -0.776309  1.408515       1.136500     -0.144802     -0.217125       -0.034087                       0                     0                        0                   1                        0                            0                     0                       0                0                0                0                   0                   0                   0               0                      0                     0                     1                     0                   0                   0                     0                       0                        0                         0                                  0                                   1                                      0                              0                          0                        0                         0                         0                         0                            0                            0                              0   \n",
       "\n",
       "   occupation_ Machine-op-inspct  occupation_ Other-service  occupation_ Priv-house-serv  occupation_ Prof-specialty  occupation_ Protective-serv  occupation_ Sales  occupation_ Tech-support  occupation_ Transport-moving  relationship_ Husband  relationship_ Not-in-family  relationship_ Other-relative  relationship_ Own-child  relationship_ Unmarried  relationship_ Wife  race_ Amer-Indian-Eskimo  race_ Asian-Pac-Islander  race_ Black  race_ Other  race_ White  sex_ Female  sex_ Male  native-country_ Cambodia  native-country_ Canada  native-country_ China  native-country_ Columbia  native-country_ Cuba  native-country_ Dominican-Republic  native-country_ Ecuador  native-country_ El-Salvador  native-country_ England  native-country_ France  native-country_ Germany  native-country_ Greece  native-country_ Guatemala  native-country_ Haiti  native-country_ Holand-Netherlands  native-country_ Honduras  native-country_ Hong  native-country_ Hungary  native-country_ India  \\\n",
       "0                              0                          0                            0                           0                            0                  0                         0                             0                      0                            1                             0                        0                        0                   0                         0                         0            0            0            1            0          1                         0                       0                      0                         0                     0                                   0                        0                            0                        0                       0                        0                       0                          0                      0                                   0                         0                     0                        0                      0   \n",
       "1                              0                          0                            0                           0                            0                  0                         0                             0                      1                            0                             0                        0                        0                   0                         0                         0            0            0            1            0          1                         0                       0                      0                         0                     0                                   0                        0                            0                        0                       0                        0                       0                          0                      0                                   0                         0                     0                        0                      0   \n",
       "2                              0                          0                            0                           0                            0                  0                         0                             0                      0                            1                             0                        0                        0                   0                         0                         0            0            0            1            0          1                         0                       0                      0                         0                     0                                   0                        0                            0                        0                       0                        0                       0                          0                      0                                   0                         0                     0                        0                      0   \n",
       "3                              0                          0                            0                           0                            0                  0                         0                             0                      1                            0                             0                        0                        0                   0                         0                         0            1            0            0            0          1                         0                       0                      0                         0                     0                                   0                        0                            0                        0                       0                        0                       0                          0                      0                                   0                         0                     0                        0                      0   \n",
       "4                              0                          0                            0                           1                            0                  0                         0                             0                      0                            0                             0                        0                        0                   1                         0                         0            1            0            0            1          0                         0                       0                      0                         0                     1                                   0                        0                            0                        0                       0                        0                       0                          0                      0                                   0                         0                     0                        0                      0   \n",
       "\n",
       "   native-country_ Iran  native-country_ Ireland  native-country_ Italy  native-country_ Jamaica  native-country_ Japan  native-country_ Laos  native-country_ Mexico  native-country_ Nicaragua  native-country_ Outlying-US(Guam-USVI-etc)  native-country_ Peru  native-country_ Philippines  native-country_ Poland  native-country_ Portugal  native-country_ Puerto-Rico  native-country_ Scotland  native-country_ South  native-country_ Taiwan  native-country_ Thailand  native-country_ Trinadad&Tobago  native-country_ United-States  native-country_ Vietnam  native-country_ Yugoslavia  label  \n",
       "0                     0                        0                      0                        0                      0                     0                       0                          0                                           0                     0                            0                       0                         0                            0                         0                      0                       0                         0                                0                              1                        0                           0      0  \n",
       "1                     0                        0                      0                        0                      0                     0                       0                          0                                           0                     0                            0                       0                         0                            0                         0                      0                       0                         0                                0                              1                        0                           0      0  \n",
       "2                     0                        0                      0                        0                      0                     0                       0                          0                                           0                     0                            0                       0                         0                            0                         0                      0                       0                         0                                0                              1                        0                           0      0  \n",
       "3                     0                        0                      0                        0                      0                     0                       0                          0                                           0                     0                            0                       0                         0                            0                         0                      0                       0                         0                                0                              1                        0                           0      0  \n",
       "4                     0                        0                      0                        0                      0                     0                       0                          0                                           0                     0                            0                       0                         0                            0                         0                      0                       0                         0                                0                              0                        0                           0      0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(combined_df.head())\n",
    "combined_df.to_csv(f'{CLEAN_DATA_FOLDER}/{NAME}_clean.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c5a8f75",
   "metadata": {},
   "source": [
    "## Bank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "748c2cf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "NAME = 'bank'\n",
    "cols = [\n",
    "    'age', # (numeric)\n",
    "    'job ', # type of job (categorical', # \"admin.\",\"blue-collar\",\"entrepreneur\",\"housemaid\",\"management\",\"retired\",\"self-employed\",\"services\",\"student\",\"technician\",\"unemployed\",\"unknown\")\n",
    "    'marital', # marital status (categorical) divorced\",\"married\",\"single\",\"unknown\"; note', # \"divorced\" means divorced or widowed)\n",
    "    'education', # (categorical \"basic.4y\",\"basic.6y\",\"basic.9y\",\"high.school\",\"illiterate\",\"professional.course\",\"university.degree\",\"unknown\")\n",
    "    'default', # has credit in default? (categorical', # \"no\",\"yes\",\"unknown\")\n",
    "    'housing', # has housing loan? (categorical', # \"no\",\"yes\",\"unknown\")\n",
    "    'loan', # has personal loan? (categorical', # \"no\",\"yes\",\"unknown\")\n",
    "    # related with the last contact of the current campaign\n",
    "    'contact', # contact communication type (categorical', # \"cellular\",\"telephone\") \n",
    "    'month', # last contact month of year (categorical', # \"jan\", \"feb\", \"mar\", ..., \"nov\", \"dec\")\n",
    "    'day_of_week', # last contact day of the week (categorical', # \"mon\",\"tue\",\"wed\",\"thu\",\"fri\")\n",
    "    'duration', # last contact duration, in seconds (numeric). Important note', #  this attribute highly affects the output target (e.g., if duration=0 then y=\"no\"). Yet, the duration is not known before a call is performed. Also, after the end of the call y is obviously known. Thus, this input should only be included for benchmark purposes and should be discarded if the intention is to have a realistic predictive model.\n",
    "    # other attributes\n",
    "    'campaign', # number of contacts performed during this campaign and for this client (numeric, includes last contact)\n",
    "    'pdays', # number of days that passed by after the client was last contacted from a previous campaign (numeric; 999 means client was not previously contacted)\n",
    "    'previous', # number of contacts performed before this campaign and for this client (numeric)\n",
    "    'poutcome', # outcome of the previous marketing campaign (categorical', # \"failure\",\"nonexistent\",\"success\")\n",
    "    # social and economic context attributes\n",
    "    'emp.var.rate', # employment variation rate - quarterly indicator (numeric)\n",
    "    'cons.price.idx', # consumer price index - monthly indicator (numeric)     \n",
    "    'cons.conf.idx', # consumer confidence index - monthly indicator (numeric)     \n",
    "    'euribor3m', # euribor 3 month rate - daily indicator (numeric)\n",
    "    'nr.employed', # number of employees - quarterly indicator (numeric)\n",
    "    'label', # (desired target) has the client subscribed a term deposit? (binary', # \"yes\",\"no\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f5800a29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41188\n",
      "Original Missingness: \n",
      " age               0.000000\n",
      "job               0.008012\n",
      "marital           0.001942\n",
      "education         0.042027\n",
      "default           0.208726\n",
      "housing           0.024036\n",
      "loan              0.024036\n",
      "contact           0.000000\n",
      "month             0.000000\n",
      "day_of_week       0.000000\n",
      "duration          0.000000\n",
      "campaign          0.000000\n",
      "pdays             0.000000\n",
      "previous          0.000000\n",
      "poutcome          0.000000\n",
      "emp.var.rate      0.000000\n",
      "cons.price.idx    0.000000\n",
      "cons.conf.idx     0.000000\n",
      "euribor3m         0.000000\n",
      "nr.employed       0.000000\n",
      "y                 0.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(f'{RAW_DATA_FOLDER}/{NAME}/bank-additional-full.csv',\n",
    "                quoting=True, sep=';')\n",
    "df = df.replace({'unknown': np.nan})\n",
    "print(len(df))\n",
    "print('Original Missingness: \\n', df.isna().sum(axis=0) / len(df))\n",
    "df = df.rename({'y': 'label', 'contact': 'cellular_contact'}, axis=1)\n",
    "df['cellular_contact'] = df['cellular_contact'].replace({'telephone': 0, 'cellular': 1})\n",
    "\n",
    "binary_cols = ['default', 'housing', 'loan', 'label']\n",
    "df[binary_cols] = df[binary_cols].replace({'yes': 1, 'no': 0})\n",
    "\n",
    "numerical_cols = ['age', 'duration', 'campaign', 'pdays', 'previous', 'emp.var.rate', 'cons.price.idx', 'euribor3m', 'nr.employed']\n",
    "category_cols = ['job', 'marital', 'education', 'month', 'day_of_week', 'poutcome']\n",
    "preserved_cols = ['cellular_contact'] + binary_cols\n",
    "df = preprocess(df, numerical_cols, category_cols, preserved_cols)\n",
    "df.to_csv(f'{CLEAN_DATA_FOLDER}/{NAME}_clean.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd704f08",
   "metadata": {},
   "source": [
    "## Thyroid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "03e9f71a",
   "metadata": {},
   "outputs": [],
   "source": [
    "NAME = 'thyroid'\n",
    "tasknames = [\n",
    "    'allbp', # increased binding protein, decreased binding protein, negative\n",
    "]\n",
    "cols = [\n",
    "    'age', #continuous.\n",
    "    'sex', #M, F.\n",
    "    'on thyroxine', #f, t.\n",
    "    'query on thyroxine', #f, t.\n",
    "    'on antithyroid medication', #f, t.\n",
    "    'sick', #f, t.\n",
    "    'pregnant', #f, t.\n",
    "    'thyroid surgery', #f, t.\n",
    "    'I131 treatment', #f, t.\n",
    "    'query hypothyroid', #f, t.\n",
    "    'query hyperthyroid', #f, t.\n",
    "    'lithium', #f, t.\n",
    "    'goitre', #f, t.\n",
    "    'tumor', #f, t.\n",
    "    'hypopituitary', #f, t.\n",
    "    'psych', #f, t.\n",
    "    'TSH measured', #f, t.\n",
    "    'TSH', #continuous.\n",
    "    'T3 measured', #f, t.\n",
    "    'T3', #continuous.\n",
    "    'TT4 measured', #f, t.\n",
    "    'TT4', #continuous.\n",
    "    'T4U measured', #f, t.\n",
    "    'T4U', #continuous.\n",
    "    'FTI measured', #f, t.\n",
    "    'FTI', #continuous.\n",
    "    'TBG measured', #f, t.\n",
    "    'TBG', #continuous.\n",
    "    'referral source', #WEST, STMW, SVHC, SVI, SVHD, other.\n",
    "    'label',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "00a99be1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================ allbp ================\n",
      "2800\n",
      "Original Missingness: \n",
      " age                          0.000357\n",
      "sex                          0.039286\n",
      "on thyroxine                 0.000000\n",
      "query on thyroxine           0.000000\n",
      "on antithyroid medication    0.000000\n",
      "sick                         0.000000\n",
      "pregnant                     0.000000\n",
      "thyroid surgery              0.000000\n",
      "I131 treatment               0.000000\n",
      "query hypothyroid            0.000000\n",
      "query hyperthyroid           0.000000\n",
      "lithium                      0.000000\n",
      "goitre                       0.000000\n",
      "tumor                        0.000000\n",
      "hypopituitary                0.000000\n",
      "psych                        0.000000\n",
      "TSH measured                 0.000000\n",
      "TSH                          0.101429\n",
      "T3 measured                  0.000000\n",
      "T3                           0.208929\n",
      "TT4 measured                 0.000000\n",
      "TT4                          0.065714\n",
      "T4U measured                 0.000000\n",
      "T4U                          0.106071\n",
      "FTI measured                 0.000000\n",
      "FTI                          0.105357\n",
      "TBG measured                 0.000000\n",
      "TBG                          1.000000\n",
      "referral source              0.000000\n",
      "label                        0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>TSH</th>\n",
       "      <th>T3</th>\n",
       "      <th>TT4</th>\n",
       "      <th>T4U</th>\n",
       "      <th>FTI</th>\n",
       "      <th>TBG</th>\n",
       "      <th>referral source_STMW</th>\n",
       "      <th>referral source_SVHC</th>\n",
       "      <th>referral source_SVHD</th>\n",
       "      <th>referral source_SVI</th>\n",
       "      <th>referral source_other</th>\n",
       "      <th>sex</th>\n",
       "      <th>on thyroxine</th>\n",
       "      <th>query on thyroxine</th>\n",
       "      <th>on antithyroid medication</th>\n",
       "      <th>sick</th>\n",
       "      <th>pregnant</th>\n",
       "      <th>thyroid surgery</th>\n",
       "      <th>I131 treatment</th>\n",
       "      <th>query hypothyroid</th>\n",
       "      <th>query hyperthyroid</th>\n",
       "      <th>lithium</th>\n",
       "      <th>goitre</th>\n",
       "      <th>tumor</th>\n",
       "      <th>hypopituitary</th>\n",
       "      <th>psych</th>\n",
       "      <th>TSH measured</th>\n",
       "      <th>T3 measured</th>\n",
       "      <th>TT4 measured</th>\n",
       "      <th>T4U measured</th>\n",
       "      <th>FTI measured</th>\n",
       "      <th>TBG measured</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.529991</td>\n",
       "      <td>-0.157214</td>\n",
       "      <td>0.576078</td>\n",
       "      <td>0.450028</td>\n",
       "      <td>0.730941</td>\n",
       "      <td>-0.054372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.409706</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>-0.199828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.285626</td>\n",
       "      <td>-0.172133</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.002046</td>\n",
       "      <td>-0.452245</td>\n",
       "      <td>0.280137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.210362</td>\n",
       "      <td>-0.151548</td>\n",
       "      <td>1.862759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.184254</td>\n",
       "      <td>-1.000444</td>\n",
       "      <td>-1.358267</td>\n",
       "      <td>-0.658016</td>\n",
       "      <td>-1.240360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       TSH        T3       TT4       T4U       FTI  TBG  referral source_STMW  referral source_SVHC  referral source_SVHD  referral source_SVI  referral source_other  sex  on thyroxine  query on thyroxine  on antithyroid medication  sick  pregnant  thyroid surgery  I131 treatment  query hypothyroid  query hyperthyroid  lithium  goitre  tumor  hypopituitary  psych  TSH measured  T3 measured  TT4 measured  T4U measured  FTI measured  TBG measured  label\n",
       "0 -0.529991 -0.157214  0.576078  0.450028  0.730941 -0.054372  0.0                     0                     1                     0                    0                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0\n",
       "1 -1.409706 -0.026674 -0.030277 -0.199828  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "2 -0.285626 -0.172133  0.000000 -0.002046 -0.452245  0.280137  0.0                     0                     0                     0                    0                      1  1.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            0             1             1             1             0      0\n",
       "3  0.887328 -0.210362 -0.151548  1.862759  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             1                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "4  0.887328 -0.184254 -1.000444 -1.358267 -0.658016 -1.240360  0.0                     0                     0                     0                    1                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training AUC:  0.8984469046632327\n",
      "================ allhyper ================\n",
      "2800\n",
      "Original Missingness: \n",
      " age                          0.000357\n",
      "sex                          0.039286\n",
      "on thyroxine                 0.000000\n",
      "query on thyroxine           0.000000\n",
      "on antithyroid medication    0.000000\n",
      "sick                         0.000000\n",
      "pregnant                     0.000000\n",
      "thyroid surgery              0.000000\n",
      "I131 treatment               0.000000\n",
      "query hypothyroid            0.000000\n",
      "query hyperthyroid           0.000000\n",
      "lithium                      0.000000\n",
      "goitre                       0.000000\n",
      "tumor                        0.000000\n",
      "hypopituitary                0.000000\n",
      "psych                        0.000000\n",
      "TSH measured                 0.000000\n",
      "TSH                          0.101429\n",
      "T3 measured                  0.000000\n",
      "T3                           0.208929\n",
      "TT4 measured                 0.000000\n",
      "TT4                          0.065714\n",
      "T4U measured                 0.000000\n",
      "T4U                          0.106071\n",
      "FTI measured                 0.000000\n",
      "FTI                          0.105357\n",
      "TBG measured                 0.000000\n",
      "TBG                          1.000000\n",
      "referral source              0.000000\n",
      "label                        0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>TSH</th>\n",
       "      <th>T3</th>\n",
       "      <th>TT4</th>\n",
       "      <th>T4U</th>\n",
       "      <th>FTI</th>\n",
       "      <th>TBG</th>\n",
       "      <th>referral source_STMW</th>\n",
       "      <th>referral source_SVHC</th>\n",
       "      <th>referral source_SVHD</th>\n",
       "      <th>referral source_SVI</th>\n",
       "      <th>referral source_other</th>\n",
       "      <th>sex</th>\n",
       "      <th>on thyroxine</th>\n",
       "      <th>query on thyroxine</th>\n",
       "      <th>on antithyroid medication</th>\n",
       "      <th>sick</th>\n",
       "      <th>pregnant</th>\n",
       "      <th>thyroid surgery</th>\n",
       "      <th>I131 treatment</th>\n",
       "      <th>query hypothyroid</th>\n",
       "      <th>query hyperthyroid</th>\n",
       "      <th>lithium</th>\n",
       "      <th>goitre</th>\n",
       "      <th>tumor</th>\n",
       "      <th>hypopituitary</th>\n",
       "      <th>psych</th>\n",
       "      <th>TSH measured</th>\n",
       "      <th>T3 measured</th>\n",
       "      <th>TT4 measured</th>\n",
       "      <th>T4U measured</th>\n",
       "      <th>FTI measured</th>\n",
       "      <th>TBG measured</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.529991</td>\n",
       "      <td>-0.157214</td>\n",
       "      <td>0.576078</td>\n",
       "      <td>0.450028</td>\n",
       "      <td>0.730941</td>\n",
       "      <td>-0.054372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.409706</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>-0.199828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.285626</td>\n",
       "      <td>-0.172133</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.002046</td>\n",
       "      <td>-0.452245</td>\n",
       "      <td>0.280137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.210362</td>\n",
       "      <td>-0.151548</td>\n",
       "      <td>1.862759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.184254</td>\n",
       "      <td>-1.000444</td>\n",
       "      <td>-1.358267</td>\n",
       "      <td>-0.658016</td>\n",
       "      <td>-1.240360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       TSH        T3       TT4       T4U       FTI  TBG  referral source_STMW  referral source_SVHC  referral source_SVHD  referral source_SVI  referral source_other  sex  on thyroxine  query on thyroxine  on antithyroid medication  sick  pregnant  thyroid surgery  I131 treatment  query hypothyroid  query hyperthyroid  lithium  goitre  tumor  hypopituitary  psych  TSH measured  T3 measured  TT4 measured  T4U measured  FTI measured  TBG measured  label\n",
       "0 -0.529991 -0.157214  0.576078  0.450028  0.730941 -0.054372  0.0                     0                     1                     0                    0                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0\n",
       "1 -1.409706 -0.026674 -0.030277 -0.199828  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "2 -0.285626 -0.172133  0.000000 -0.002046 -0.452245  0.280137  0.0                     0                     0                     0                    0                      1  1.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            0             1             1             1             0      0\n",
       "3  0.887328 -0.210362 -0.151548  1.862759  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             1                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "4  0.887328 -0.184254 -1.000444 -1.358267 -0.658016 -1.240360  0.0                     0                     0                     0                    1                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training AUC:  0.9809320316114294\n",
      "================ allhypo ================\n",
      "2800\n",
      "Original Missingness: \n",
      " age                          0.000357\n",
      "sex                          0.039286\n",
      "on thyroxine                 0.000000\n",
      "query on thyroxine           0.000000\n",
      "on antithyroid medication    0.000000\n",
      "sick                         0.000000\n",
      "pregnant                     0.000000\n",
      "thyroid surgery              0.000000\n",
      "I131 treatment               0.000000\n",
      "query hypothyroid            0.000000\n",
      "query hyperthyroid           0.000000\n",
      "lithium                      0.000000\n",
      "goitre                       0.000000\n",
      "tumor                        0.000000\n",
      "hypopituitary                0.000000\n",
      "psych                        0.000000\n",
      "TSH measured                 0.000000\n",
      "TSH                          0.101429\n",
      "T3 measured                  0.000000\n",
      "T3                           0.208929\n",
      "TT4 measured                 0.000000\n",
      "TT4                          0.065714\n",
      "T4U measured                 0.000000\n",
      "T4U                          0.106071\n",
      "FTI measured                 0.000000\n",
      "FTI                          0.105357\n",
      "TBG measured                 0.000000\n",
      "TBG                          1.000000\n",
      "referral source              0.000000\n",
      "label                        0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>TSH</th>\n",
       "      <th>T3</th>\n",
       "      <th>TT4</th>\n",
       "      <th>T4U</th>\n",
       "      <th>FTI</th>\n",
       "      <th>TBG</th>\n",
       "      <th>referral source_STMW</th>\n",
       "      <th>referral source_SVHC</th>\n",
       "      <th>referral source_SVHD</th>\n",
       "      <th>referral source_SVI</th>\n",
       "      <th>referral source_other</th>\n",
       "      <th>sex</th>\n",
       "      <th>on thyroxine</th>\n",
       "      <th>query on thyroxine</th>\n",
       "      <th>on antithyroid medication</th>\n",
       "      <th>sick</th>\n",
       "      <th>pregnant</th>\n",
       "      <th>thyroid surgery</th>\n",
       "      <th>I131 treatment</th>\n",
       "      <th>query hypothyroid</th>\n",
       "      <th>query hyperthyroid</th>\n",
       "      <th>lithium</th>\n",
       "      <th>goitre</th>\n",
       "      <th>tumor</th>\n",
       "      <th>hypopituitary</th>\n",
       "      <th>psych</th>\n",
       "      <th>TSH measured</th>\n",
       "      <th>T3 measured</th>\n",
       "      <th>TT4 measured</th>\n",
       "      <th>T4U measured</th>\n",
       "      <th>FTI measured</th>\n",
       "      <th>TBG measured</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.529991</td>\n",
       "      <td>-0.157214</td>\n",
       "      <td>0.576078</td>\n",
       "      <td>0.450028</td>\n",
       "      <td>0.730941</td>\n",
       "      <td>-0.054372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.409706</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>-0.199828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.285626</td>\n",
       "      <td>-0.172133</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.002046</td>\n",
       "      <td>-0.452245</td>\n",
       "      <td>0.280137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.210362</td>\n",
       "      <td>-0.151548</td>\n",
       "      <td>1.862759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.184254</td>\n",
       "      <td>-1.000444</td>\n",
       "      <td>-1.358267</td>\n",
       "      <td>-0.658016</td>\n",
       "      <td>-1.240360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       TSH        T3       TT4       T4U       FTI  TBG  referral source_STMW  referral source_SVHC  referral source_SVHD  referral source_SVI  referral source_other  sex  on thyroxine  query on thyroxine  on antithyroid medication  sick  pregnant  thyroid surgery  I131 treatment  query hypothyroid  query hyperthyroid  lithium  goitre  tumor  hypopituitary  psych  TSH measured  T3 measured  TT4 measured  T4U measured  FTI measured  TBG measured  label\n",
       "0 -0.529991 -0.157214  0.576078  0.450028  0.730941 -0.054372  0.0                     0                     1                     0                    0                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0\n",
       "1 -1.409706 -0.026674 -0.030277 -0.199828  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "2 -0.285626 -0.172133  0.000000 -0.002046 -0.452245  0.280137  0.0                     0                     0                     0                    0                      1  1.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            0             1             1             1             0      0\n",
       "3  0.887328 -0.210362 -0.151548  1.862759  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             1                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "4  0.887328 -0.184254 -1.000444 -1.358267 -0.658016 -1.240360  0.0                     0                     0                     0                    1                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training AUC:  0.9420190274841437\n",
      "================ allrep ================\n",
      "2800\n",
      "Original Missingness: \n",
      " age                          0.000357\n",
      "sex                          0.039286\n",
      "on thyroxine                 0.000000\n",
      "query on thyroxine           0.000000\n",
      "on antithyroid medication    0.000000\n",
      "sick                         0.000000\n",
      "pregnant                     0.000000\n",
      "thyroid surgery              0.000000\n",
      "I131 treatment               0.000000\n",
      "query hypothyroid            0.000000\n",
      "query hyperthyroid           0.000000\n",
      "lithium                      0.000000\n",
      "goitre                       0.000000\n",
      "tumor                        0.000000\n",
      "hypopituitary                0.000000\n",
      "psych                        0.000000\n",
      "TSH measured                 0.000000\n",
      "TSH                          0.101429\n",
      "T3 measured                  0.000000\n",
      "T3                           0.208929\n",
      "TT4 measured                 0.000000\n",
      "TT4                          0.065714\n",
      "T4U measured                 0.000000\n",
      "T4U                          0.106071\n",
      "FTI measured                 0.000000\n",
      "FTI                          0.105357\n",
      "TBG measured                 0.000000\n",
      "TBG                          1.000000\n",
      "referral source              0.000000\n",
      "label                        0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>TSH</th>\n",
       "      <th>T3</th>\n",
       "      <th>TT4</th>\n",
       "      <th>T4U</th>\n",
       "      <th>FTI</th>\n",
       "      <th>TBG</th>\n",
       "      <th>referral source_STMW</th>\n",
       "      <th>referral source_SVHC</th>\n",
       "      <th>referral source_SVHD</th>\n",
       "      <th>referral source_SVI</th>\n",
       "      <th>referral source_other</th>\n",
       "      <th>sex</th>\n",
       "      <th>on thyroxine</th>\n",
       "      <th>query on thyroxine</th>\n",
       "      <th>on antithyroid medication</th>\n",
       "      <th>sick</th>\n",
       "      <th>pregnant</th>\n",
       "      <th>thyroid surgery</th>\n",
       "      <th>I131 treatment</th>\n",
       "      <th>query hypothyroid</th>\n",
       "      <th>query hyperthyroid</th>\n",
       "      <th>lithium</th>\n",
       "      <th>goitre</th>\n",
       "      <th>tumor</th>\n",
       "      <th>hypopituitary</th>\n",
       "      <th>psych</th>\n",
       "      <th>TSH measured</th>\n",
       "      <th>T3 measured</th>\n",
       "      <th>TT4 measured</th>\n",
       "      <th>T4U measured</th>\n",
       "      <th>FTI measured</th>\n",
       "      <th>TBG measured</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.529991</td>\n",
       "      <td>-0.157214</td>\n",
       "      <td>0.576078</td>\n",
       "      <td>0.450028</td>\n",
       "      <td>0.730941</td>\n",
       "      <td>-0.054372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.409706</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>-0.199828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.285626</td>\n",
       "      <td>-0.172133</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.002046</td>\n",
       "      <td>-0.452245</td>\n",
       "      <td>0.280137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.210362</td>\n",
       "      <td>-0.151548</td>\n",
       "      <td>1.862759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.184254</td>\n",
       "      <td>-1.000444</td>\n",
       "      <td>-1.358267</td>\n",
       "      <td>-0.658016</td>\n",
       "      <td>-1.240360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       TSH        T3       TT4       T4U       FTI  TBG  referral source_STMW  referral source_SVHC  referral source_SVHD  referral source_SVI  referral source_other  sex  on thyroxine  query on thyroxine  on antithyroid medication  sick  pregnant  thyroid surgery  I131 treatment  query hypothyroid  query hyperthyroid  lithium  goitre  tumor  hypopituitary  psych  TSH measured  T3 measured  TT4 measured  T4U measured  FTI measured  TBG measured  label\n",
       "0 -0.529991 -0.157214  0.576078  0.450028  0.730941 -0.054372  0.0                     0                     1                     0                    0                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0\n",
       "1 -1.409706 -0.026674 -0.030277 -0.199828  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "2 -0.285626 -0.172133  0.000000 -0.002046 -0.452245  0.280137  0.0                     0                     0                     0                    0                      1  1.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            0             1             1             1             0      0\n",
       "3  0.887328 -0.210362 -0.151548  1.862759  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             1                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      1\n",
       "4  0.887328 -0.184254 -1.000444 -1.358267 -0.658016 -1.240360  0.0                     0                     0                     0                    1                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training AUC:  0.9798543411670503\n",
      "================ dis ================\n",
      "2800\n",
      "Original Missingness: \n",
      " age                          0.000357\n",
      "sex                          0.039286\n",
      "on thyroxine                 0.000000\n",
      "query on thyroxine           0.000000\n",
      "on antithyroid medication    0.000000\n",
      "sick                         0.000000\n",
      "pregnant                     0.000000\n",
      "thyroid surgery              0.000000\n",
      "I131 treatment               0.000000\n",
      "query hypothyroid            0.000000\n",
      "query hyperthyroid           0.000000\n",
      "lithium                      0.000000\n",
      "goitre                       0.000000\n",
      "tumor                        0.000000\n",
      "hypopituitary                0.000000\n",
      "psych                        0.000000\n",
      "TSH measured                 0.000000\n",
      "TSH                          0.101429\n",
      "T3 measured                  0.000000\n",
      "T3                           0.208929\n",
      "TT4 measured                 0.000000\n",
      "TT4                          0.065714\n",
      "T4U measured                 0.000000\n",
      "T4U                          0.106071\n",
      "FTI measured                 0.000000\n",
      "FTI                          0.105357\n",
      "TBG measured                 0.000000\n",
      "TBG                          1.000000\n",
      "referral source              0.000000\n",
      "label                        0.000000\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>TSH</th>\n",
       "      <th>T3</th>\n",
       "      <th>TT4</th>\n",
       "      <th>T4U</th>\n",
       "      <th>FTI</th>\n",
       "      <th>TBG</th>\n",
       "      <th>referral source_STMW</th>\n",
       "      <th>referral source_SVHC</th>\n",
       "      <th>referral source_SVHD</th>\n",
       "      <th>referral source_SVI</th>\n",
       "      <th>referral source_other</th>\n",
       "      <th>sex</th>\n",
       "      <th>on thyroxine</th>\n",
       "      <th>query on thyroxine</th>\n",
       "      <th>on antithyroid medication</th>\n",
       "      <th>sick</th>\n",
       "      <th>pregnant</th>\n",
       "      <th>thyroid surgery</th>\n",
       "      <th>I131 treatment</th>\n",
       "      <th>query hypothyroid</th>\n",
       "      <th>query hyperthyroid</th>\n",
       "      <th>lithium</th>\n",
       "      <th>goitre</th>\n",
       "      <th>tumor</th>\n",
       "      <th>hypopituitary</th>\n",
       "      <th>psych</th>\n",
       "      <th>TSH measured</th>\n",
       "      <th>T3 measured</th>\n",
       "      <th>TT4 measured</th>\n",
       "      <th>T4U measured</th>\n",
       "      <th>FTI measured</th>\n",
       "      <th>TBG measured</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.529991</td>\n",
       "      <td>-0.157214</td>\n",
       "      <td>0.576078</td>\n",
       "      <td>0.450028</td>\n",
       "      <td>0.730941</td>\n",
       "      <td>-0.054372</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.409706</td>\n",
       "      <td>-0.026674</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>-0.199828</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.285626</td>\n",
       "      <td>-0.172133</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.002046</td>\n",
       "      <td>-0.452245</td>\n",
       "      <td>0.280137</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.210362</td>\n",
       "      <td>-0.151548</td>\n",
       "      <td>1.862759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.887328</td>\n",
       "      <td>-0.184254</td>\n",
       "      <td>-1.000444</td>\n",
       "      <td>-1.358267</td>\n",
       "      <td>-0.658016</td>\n",
       "      <td>-1.240360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       TSH        T3       TT4       T4U       FTI  TBG  referral source_STMW  referral source_SVHC  referral source_SVHD  referral source_SVI  referral source_other  sex  on thyroxine  query on thyroxine  on antithyroid medication  sick  pregnant  thyroid surgery  I131 treatment  query hypothyroid  query hyperthyroid  lithium  goitre  tumor  hypopituitary  psych  TSH measured  T3 measured  TT4 measured  T4U measured  FTI measured  TBG measured  label\n",
       "0 -0.529991 -0.157214  0.576078  0.450028  0.730941 -0.054372  0.0                     0                     1                     0                    0                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0\n",
       "1 -1.409706 -0.026674 -0.030277 -0.199828  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "2 -0.285626 -0.172133  0.000000 -0.002046 -0.452245  0.280137  0.0                     0                     0                     0                    0                      1  1.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            0             1             1             1             0      0\n",
       "3  0.887328 -0.210362 -0.151548  1.862759  0.000000  0.000000  0.0                     0                     0                     0                    0                      1  0.0             1                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             0             0             0      0\n",
       "4  0.887328 -0.184254 -1.000444 -1.358267 -0.658016 -1.240360  0.0                     0                     0                     0                    1                      0  0.0             0                   0                          0     0         0                0               0                  0                   0        0       0      0              0      0             1            1             1             1             1             0      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training AUC:  0.9075700746118169\n"
     ]
    }
   ],
   "source": [
    "for task in tasknames:\n",
    "    print(f'================ {task} ================')\n",
    "    df = pd.read_csv(f'{RAW_DATA_FOLDER}/{NAME}/{task}.data',\n",
    "                     names=cols, index_col=False)\n",
    "    print(len(df))\n",
    "\n",
    "    df = df.replace({'?': np.nan})\n",
    "    print('Original Missingness: \\n', df.isna().sum(axis=0) / len(df))\n",
    "    df['label_cat'] = df.apply(lambda x: x['label'].split('.|')[0], axis=1)\n",
    "    df['label'] = df.apply(lambda x: int(x['label_cat'] != 'negative'), axis=1)\n",
    "\n",
    "    binary_cols = [\n",
    "        'sex', #M, F.\n",
    "        'on thyroxine', #f, t.\n",
    "        'query on thyroxine', #f, t.\n",
    "        'on antithyroid medication', #f, t.\n",
    "        'sick', #f, t.\n",
    "        'pregnant', #f, t.\n",
    "        'thyroid surgery', #f, t.\n",
    "        'I131 treatment', #f, t.\n",
    "        'query hypothyroid', #f, t.\n",
    "        'query hyperthyroid', #f, t.\n",
    "        'lithium', #f, t.\n",
    "        'goitre', #f, t.\n",
    "        'tumor', #f, t.\n",
    "        'hypopituitary', #f, t.\n",
    "        'psych', #f, t.\n",
    "        'TSH measured', #f, t.\n",
    "        'T3 measured', #f, t.\n",
    "        'TT4 measured', #f, t.\n",
    "        'T4U measured', #f, t.\n",
    "        'FTI measured', #f, t.\n",
    "        'TBG measured', #f, t.\n",
    "    ]\n",
    "    df[binary_cols] = df[binary_cols].replace({'t': 1, 'f': 0, \n",
    "                                               'M': 1, 'F': 0})\n",
    "\n",
    "    numerical_cols = [\n",
    "        'age', #continuous.\n",
    "        'TSH', #continuous.\n",
    "        'T3', #continuous.\n",
    "        'TT4', #continuous.\n",
    "        'T4U', #continuous.\n",
    "        'FTI', #continuous.\n",
    "        'TBG', #continuous.\n",
    "    ]\n",
    "    category_cols = ['referral source']\n",
    "    preserved_cols = binary_cols + ['label']\n",
    "    # df[category_cols] = df[category_cols].astype(str)\n",
    "    df = preprocess(df, numerical_cols, category_cols, preserved_cols)\n",
    "    df.to_csv(f'{CLEAN_DATA_FOLDER}/{NAME}_{task}_clean.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0 (default, Jun 28 2018, 07:39:16) \n[Clang 4.0.1 (tags/RELEASE_401/final)]"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}