project/01_1_train_Median.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bc2c7025",
   "metadata": {},
   "source": [
    "# Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdefddcb",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.model\n",
    "import pimmslearn.models as models\n",
    "import pimmslearn.nb\n",
    "from pimmslearn.io import datasplits\n",
    "\n",
    "logger = pimmslearn.logging.setup_logger(logging.getLogger('pimmslearn'))\n",
    "logger.info(\"Median Imputation\")\n",
    "\n",
    "figures = {}  # collection of ax or figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82a53c81",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbde4d85",
   "metadata": {},
   "source": [
    "Papermill script parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b151837c",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# files and folders\n",
    "folder_experiment: str = 'runs/example'  # Datasplit folder with data for experiment\n",
    "file_format: str = 'csv'  # file format of create splits, default pickle (pkl)\n",
    "fn_rawfile_metadata: str = 'data/dev_datasets/HeLa_6070/files_selected_metadata_N50.csv'  # Metadata for samples\n",
    "# model\n",
    "sample_idx_position: int = 0  # position of index which is sample ID\n",
    "model_key: str = 'Median'  # model key (lower cased version will be used for file names)\n",
    "model: str = 'Median'  # model name\n",
    "save_pred_real_na: bool = True  # Save all predictions for real na\n",
    "# metadata -> defaults for metadata extracted from machine data\n",
    "meta_date_col: str = None  # date column in meta data\n",
    "meta_cat_col: str = None  # category column in meta data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bac9235",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "Some argument transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17c49967",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.get_params(args, globals=globals())\n",
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "071eb3aa",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.args_from_dict(args)\n",
    "args"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c50a9adc",
   "metadata": {},
   "source": [
    "Some naming conventions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79bd3e3d",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "TEMPLATE_MODEL_PARAMS = 'model_params_{}.json'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7e52cdc",
   "metadata": {},
   "source": [
    "## Load data in long format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb8a27fa",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = datasplits.DataSplits.from_folder(args.data, file_format=args.file_format)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d198001",
   "metadata": {},
   "source": [
    "data is loaded in long format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0df21944",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data.train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa7dcd09",
   "metadata": {},
   "source": [
    "Infer index names from long format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86b35447",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "index_columns = list(data.train_X.index.names)\n",
    "sample_id = index_columns.pop(args.sample_idx_position)\n",
    "if len(index_columns) == 1:\n",
    "    index_column = index_columns.pop()\n",
    "    index_columns = None\n",
    "    logger.info(f\"{sample_id = }, single feature: {index_column = }\")\n",
    "else:\n",
    "    logger.info(f\"{sample_id = }, multiple features: {index_columns = }\")\n",
    "\n",
    "if not index_columns:\n",
    "    index_columns = [sample_id, index_column]\n",
    "else:\n",
    "    raise NotImplementedError(\"More than one feature: Needs to be implemented. see above logging output.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a38f3c5b",
   "metadata": {},
   "source": [
    "load meta data for splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe75402",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.fn_rawfile_metadata:\n",
    "    df_meta = pd.read_csv(args.fn_rawfile_metadata, index_col=0)\n",
    "    display(df_meta.loc[data.train_X.index.levels[0]])\n",
    "else:\n",
    "    df_meta = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4a93cdc",
   "metadata": {},
   "source": [
    "## Initialize Comparison\n",
    "\n",
    "- replicates idea for truely missing values: Define truth as by using n=3 replicates to impute\n",
    "  each sample\n",
    "- real test data:\n",
    "    - Not used for predictions or early stopping.\n",
    "    - [x] add some additional NAs based on distribution of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75aa1ac5",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "freq_feat = pimmslearn.io.datasplits.load_freq(args.data)\n",
    "freq_feat.head()  # training data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9134efcf",
   "metadata": {},
   "source": [
    "### Produce some addional fake samples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23ac9141",
   "metadata": {},
   "source": [
    "The validation fake NA is used to by all models to evaluate training performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16f53ce5",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "val_pred_fake_na = data.val_y.to_frame(name='observed')\n",
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68ea1649",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "test_pred_fake_na = data.test_y.to_frame(name='observed')\n",
    "test_pred_fake_na.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "774136d4",
   "metadata": {},
   "source": [
    "## Data in wide format\n",
    "\n",
    "- Autoencoder need data in wide format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c679a1f9",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data.to_wide_format()\n",
    "args.M = data.train_X.shape[-1]\n",
    "data.train_X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cb9df3b",
   "metadata": {},
   "source": [
    "### Add interpolation performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d667e3",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# interpolated = pimmslearn.pandas.interpolate(wide_df = data.train_X)\n",
    "# val_pred_fake_na['interpolated'] = interpolated\n",
    "# test_pred_fake_na['interpolated'] = interpolated\n",
    "# del interpolated\n",
    "# test_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fb6cea0",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# Add median pred performance\n",
    "args.n_params = data.train_X.shape[-1]\n",
    "medians_train = data.train_X.median()\n",
    "medians_train.name = args.model\n",
    "pred = medians_train\n",
    "\n",
    "val_pred_fake_na = val_pred_fake_na.join(medians_train)\n",
    "test_pred_fake_na = test_pred_fake_na.join(medians_train)\n",
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c05ecd3a",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.save_pred_real_na:\n",
    "    mask = data.train_X.isna().stack()\n",
    "    idx_real_na = mask.index[mask]\n",
    "    idx_real_na = (idx_real_na\n",
    "                   .drop(val_pred_fake_na.index)\n",
    "                   .drop(test_pred_fake_na.index))\n",
    "    # hacky, but works:\n",
    "    pred_real_na = (pd.Series(0, index=idx_real_na, name='placeholder')\n",
    "                    .to_frame()\n",
    "                    .join(medians_train)\n",
    "                    .drop('placeholder', axis=1))\n",
    "    # pred_real_na.name = 'intensity'\n",
    "    display(pred_real_na)\n",
    "    pred_real_na.to_csv(args.out_preds / f\"pred_real_na_{args.model_key}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba2778ce",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "source": [
    "### Plots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73648586",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "feat_freq_val = val_pred_fake_na['observed'].groupby(level=-1).count()\n",
    "feat_freq_val.name = 'freq_val'\n",
    "ax = feat_freq_val.plot.box()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3662d07",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# # scatter plot between overall feature freq and split freq\n",
    "# freq_feat.to_frame('overall').join(feat_freq_val).plot.scatter(x='overall', y='freq_val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c72fe9c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "feat_freq_val.value_counts().sort_index().head()  # require more than one feat?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "786f8804",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val = val_pred_fake_na.drop('observed', axis=1).sub(val_pred_fake_na['observed'], axis=0)\n",
    "errors_val = errors_val.abs().groupby(level=-1).mean()\n",
    "errors_val = errors_val.join(freq_feat).sort_values(by='freq', ascending=True)\n",
    "\n",
    "\n",
    "errors_val_smoothed = errors_val.copy()  # .loc[feat_freq_val > 1]\n",
    "errors_val_smoothed[errors_val.columns[:-\n",
    "                                       1]] = errors_val[errors_val.columns[:-\n",
    "                                                                           1]].rolling(window=200, min_periods=1).mean()\n",
    "ax = errors_val_smoothed.plot(x='freq', figsize=(15, 10))\n",
    "# errors_val_smoothed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6da29e33",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val = val_pred_fake_na.drop('observed', axis=1).sub(val_pred_fake_na['observed'], axis=0)\n",
    "errors_val.abs().groupby(level=-1).agg(['mean', 'count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3565522f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae345647",
   "metadata": {},
   "source": [
    "## Comparisons"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b43adc40",
   "metadata": {},
   "source": [
    "### Validation data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b017353a",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# papermill_description=metrics\n",
    "d_metrics = models.Metrics()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afdefc29",
   "metadata": {},
   "source": [
    "The fake NA for the validation step are real test data (not used for training nor early stopping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47caaf3b",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(val_pred_fake_na, 'valid_fake_na')\n",
    "added_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e9cc919",
   "metadata": {},
   "source": [
    "### Test Datasplit\n",
    "\n",
    "Fake NAs : Artificially created NAs. Some data was sampled and set\n",
    "explicitly to misssing before it was fed to the model for\n",
    "reconstruction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b45f076",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(test_pred_fake_na, 'test_fake_na')\n",
    "added_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "140437f2",
   "metadata": {},
   "source": [
    "The fake NA for the validation step are real test data (not used for training nor early stopping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6a5da36",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "077a514b",
   "metadata": {},
   "source": [
    "### Save all metrics as json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26be5fa4",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pimmslearn.io.dump_json(d_metrics.metrics, args.out_metrics / f'metrics_{args.model_key}.json')\n",
    "d_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fe80e9a",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "metrics_df = models.get_df_from_nested_dict(d_metrics.metrics, column_levels=['model', 'metric_name']).T\n",
    "metrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "def60306",
   "metadata": {},
   "source": [
    "## Save predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "225dc1f0",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# val\n",
    "fname = args.out_preds / f\"pred_val_{args.model_key}.csv\"\n",
    "setattr(args, fname.stem, fname.as_posix())  # add [] assignment?\n",
    "val_pred_fake_na.to_csv(fname)\n",
    "# test\n",
    "fname = args.out_preds / f\"pred_test_{args.model_key}.csv\"\n",
    "setattr(args, fname.stem, fname.as_posix())\n",
    "test_pred_fake_na.to_csv(fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d575ba68",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64a39dc2",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "figures  # switch to fnames?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14983bf9",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args.dump(fname=args.out_models / f\"model_config_{args.model_key}.yaml\")\n",
    "args"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}