project/10_3_ald_ml_new_feat.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d5f8edbd",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "# Fit logistic regression model\n",
    "\n",
    "- based on different imputation methods\n",
    "- baseline: reference\n",
    "- model: any other selected imputation method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d8c6764",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "from pathlib import Path\n",
    "from typing import List\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import njab.sklearn\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from njab.plotting.metrics import plot_split_auc, plot_split_prc\n",
    "from njab.sklearn.types import Splits\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.analyzers\n",
    "import pimmslearn.io.datasplits\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (2.5, 2.5)\n",
    "plt.rcParams['lines.linewidth'] = 1\n",
    "plt.rcParams['lines.markersize'] = 2\n",
    "fontsize = 5\n",
    "figsize = (2.5, 2.5)\n",
    "pimmslearn.plotting.make_large_descriptors(fontsize)\n",
    "\n",
    "\n",
    "logger = pimmslearn.logging.setup_nb_logger()\n",
    "logging.getLogger('fontTools').setLevel(logging.ERROR)\n",
    "\n",
    "\n",
    "def parse_roc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:\n",
    "    ret = list()\n",
    "    for _r in res:\n",
    "        _roc = (pd.DataFrame(_r.test.roc,\n",
    "                             index='fpr tpr cutoffs'.split()\n",
    "                             )).loc[['fpr', 'tpr']]\n",
    "        _roc = _roc.T\n",
    "        _roc.columns = pd.MultiIndex.from_product([[_r.name], _roc.columns])\n",
    "        ret.append(_roc)\n",
    "    ret = pd.concat(ret, axis=1)\n",
    "    return ret\n",
    "\n",
    "\n",
    "def parse_prc(*res: List[njab.sklearn.types.Results]) -> pd.DataFrame:\n",
    "    ret = list()\n",
    "    for _r in res:\n",
    "        _prc = pd.DataFrame(_r.test.prc,\n",
    "                            index='precision recall cutoffs'.split()\n",
    "                            ).loc[['precision', 'recall']]\n",
    "        _prc = _prc.T.rename(columns={'recall': 'tpr'})\n",
    "        _prc.columns = pd.MultiIndex.from_product([[_r.name], _prc.columns])\n",
    "        ret.append(_prc)\n",
    "    ret = pd.concat(ret, axis=1)\n",
    "    return ret\n",
    "\n",
    "\n",
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1e67f6d",
   "metadata": {},
   "source": [
    "## Parameters\n",
    "Default and set parameters for the notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "443cd83d",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "folder_data: str = ''  # specify data directory if needed\n",
    "fn_clinical_data = \"data/ALD_study/processed/ald_metadata_cli.csv\"\n",
    "folder_experiment = \"runs/appl_ald_data/plasma/proteinGroups\"\n",
    "model_key = 'VAE'\n",
    "target = 'kleiner'\n",
    "sample_id_col = 'Sample ID'\n",
    "cutoff_target: int = 2  # => for binarization target >= cutoff_target\n",
    "file_format = \"csv\"\n",
    "out_folder = 'diff_analysis'\n",
    "fn_qc_samples = ''  # 'data/ALD_study/processed/qc_plasma_proteinGroups.pkl'\n",
    "\n",
    "baseline = 'RSN'  # default is RSN, as this was used in the original ALD Niu. et. al 2022\n",
    "template_pred = 'pred_real_na_{}.csv'  # fixed, do not change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13538b85",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "params = pimmslearn.nb.get_params(args, globals=globals())\n",
    "args = pimmslearn.nb.Config()\n",
    "args.folder_experiment = Path(params[\"folder_experiment\"])\n",
    "args = pimmslearn.nb.add_default_paths(args,\n",
    "                                 out_root=(args.folder_experiment\n",
    "                                           / params[\"out_folder\"]\n",
    "                                           / params[\"target\"]\n",
    "                                           / f\"{params['baseline']}_vs_{params['model_key']}\"))\n",
    "args.update_from_dict(params)\n",
    "files_out = dict()\n",
    "args"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ca8264e",
   "metadata": {},
   "source": [
    "## Load data\n",
    "\n",
    "### Load target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09ba8ea3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "target = pd.read_csv(args.fn_clinical_data,\n",
    "                     index_col=0,\n",
    "                     usecols=[args.sample_id_col, args.target])\n",
    "target = target.dropna()\n",
    "target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02bbf2a2",
   "metadata": {},
   "source": [
    "### MS proteomics or specified omics data\n",
    "Aggregated from data splits of the imputation workflow run before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4cd6005",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = pimmslearn.io.datasplits.DataSplits.from_folder(\n",
    "    args.data, file_format=args.file_format)\n",
    "data = pd.concat([data.train_X, data.val_y, data.test_y])\n",
    "data.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac6b446e",
   "metadata": {},
   "source": [
    "Get overlap between independent features and target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c79ad218",
   "metadata": {},
   "source": [
    "### Select by ALD criteria\n",
    "Use parameters as specified in [ALD study](https://github.com/RasmussenLab/pimms/tree/main/project/data/ALD_study)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3038462c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "DATA_COMPLETENESS = 0.6\n",
    "MIN_N_PROTEIN_GROUPS: int = 200\n",
    "FRAC_PROTEIN_GROUPS: int = 0.622\n",
    "CV_QC_SAMPLE: float = 0.4\n",
    "\n",
    "ald_study, cutoffs = pimmslearn.analyzers.diff_analysis.select_raw_data(data.unstack(\n",
    "), data_completeness=DATA_COMPLETENESS, frac_protein_groups=FRAC_PROTEIN_GROUPS)\n",
    "\n",
    "if args.fn_qc_samples:\n",
    "    qc_samples = pd.read_pickle(args.fn_qc_samples)\n",
    "    qc_samples = qc_samples[ald_study.columns]\n",
    "    qc_cv_feat = qc_samples.std() / qc_samples.mean()\n",
    "    qc_cv_feat = qc_cv_feat.rename(qc_samples.columns.name)\n",
    "    fig, ax = plt.subplots(figsize=(4, 7))\n",
    "    ax = qc_cv_feat.plot.box(ax=ax)\n",
    "    ax.set_ylabel('Coefficient of Variation')\n",
    "    print((qc_cv_feat < CV_QC_SAMPLE).value_counts())\n",
    "    ald_study = ald_study[pimmslearn.analyzers.diff_analysis.select_feat(qc_samples)]\n",
    "\n",
    "column_name_first_prot_to_pg = {\n",
    "    pg.split(';')[0]: pg for pg in data.unstack().columns}\n",
    "\n",
    "ald_study = ald_study.rename(columns=column_name_first_prot_to_pg)\n",
    "ald_study"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0e04598",
   "metadata": {},
   "source": [
    "Number of complete cases which can be used:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a9e70e6",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "mask_has_target = data.index.levels[0].intersection(target.index)\n",
    "assert not mask_has_target.empty, f\"No data for target: {data.index.levels[0]} and {target.index}\"\n",
    "print(\n",
    "    f\"Samples available both in proteomics data and for target: {len(mask_has_target)}\")\n",
    "target, data, ald_study = target.loc[mask_has_target], data.loc[mask_has_target], ald_study.loc[mask_has_target]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcc05bf5",
   "metadata": {},
   "source": [
    "### Load imputations from specified model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f072d5f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fname = args.out_preds / args.template_pred.format(args.model_key)\n",
    "print(f\"missing values pred. by {args.model_key}: {fname}\")\n",
    "load_single_csv_pred_file = pimmslearn.analyzers.compare_predictions.load_single_csv_pred_file\n",
    "pred_real_na = load_single_csv_pred_file(fname).loc[mask_has_target]\n",
    "pred_real_na.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eadd9ea6",
   "metadata": {},
   "source": [
    "### Load imputations from baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f2dd584",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fname = args.out_preds / args.template_pred.format(args.baseline)\n",
    "pred_real_na_baseline = load_single_csv_pred_file(fname)  # .loc[mask_has_target]\n",
    "pred_real_na_baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fa21c8b",
   "metadata": {},
   "source": [
    "## Modeling setup\n",
    "General approach:\n",
    "  - use one train, test split of the data\n",
    "  - select best 10 features from training data `X_train`, `y_train` before binarization of target\n",
    "  - dichotomize (binarize) data into to groups (zero and 1)\n",
    "  - evaluate model on the test data `X_test`, `y_test`\n",
    "\n",
    "Repeat general approach for\n",
    " 1. all original ald data: all features justed in original ALD study\n",
    " 2. all model data: all features available my using the self supervised deep learning model\n",
    " 3. newly available feat only: the subset of features available from the\n",
    "    self supervised deep learning model which were newly retained using the\n",
    "    new approach\n",
    "\n",
    "All data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f457863e",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "X = pd.concat([data, pred_real_na]).unstack()\n",
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c92c7bd",
   "metadata": {},
   "source": [
    "### Subset of data by ALD criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a387dd6f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# could be just observed, drop columns with missing values\n",
    "ald_study = pd.concat(\n",
    "    [ald_study.stack(),\n",
    "     pred_real_na_baseline.loc[\n",
    "        # only select columns in selected in ald_study\n",
    "        pd.IndexSlice[:, pred_real_na.index.levels[-1].intersection(ald_study.columns)]\n",
    "    ]\n",
    "    ]\n",
    ").unstack()\n",
    "ald_study"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8c07f73",
   "metadata": {},
   "source": [
    "Features which would not have been included using ALD criteria:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12b9d002",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "new_features = X.columns.difference(ald_study.columns)\n",
    "new_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e67247-a2a1-4a2f-b838-0bdc9f40cfa9",
   "metadata": {},
   "source": [
    "Binarize targets, but also keep groups for stratification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa1f404-427a-4e78-b98d-cb26bb1d1ec4",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "target_to_group = target.copy()\n",
    "target = target >= args.cutoff_target\n",
    "pd.crosstab(target.squeeze(), target_to_group.squeeze())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfab754f",
   "metadata": {},
   "source": [
    "## Determine best number of parameters by cross validation procedure\n",
    "\n",
    "using subset of data by ALD criteria:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90e410d1",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "cv_feat_ald = njab.sklearn.find_n_best_features(X=ald_study, y=target, name=args.target,\n",
    "                                                groups=target_to_group)\n",
    "cv_feat_ald = (cv_feat_ald\n",
    "               .drop('test_case', axis=1)\n",
    "               .groupby('n_features')\n",
    "               .agg(['mean', 'std']))\n",
    "cv_feat_ald"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b40fb391",
   "metadata": {},
   "source": [
    "Using all data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "988dea31",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "cv_feat_all = njab.sklearn.find_n_best_features(X=X, y=target, name=args.target,\n",
    "                                                groups=target_to_group)\n",
    "cv_feat_all = cv_feat_all.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])\n",
    "cv_feat_all"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0029a621",
   "metadata": {},
   "source": [
    "Using only new features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "811f75d0",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "cv_feat_new = njab.sklearn.find_n_best_features(X=X.loc[:, new_features],\n",
    "                                                y=target, name=args.target,\n",
    "                                                groups=target_to_group)\n",
    "cv_feat_new = cv_feat_new.drop('test_case', axis=1).groupby('n_features').agg(['mean', 'std'])\n",
    "cv_feat_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd57bbac",
   "metadata": {},
   "source": [
    "### Best number of features by subset of the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72655713",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "n_feat_best = pd.DataFrame(\n",
    "    {'ald': cv_feat_ald.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),\n",
    "     'all': cv_feat_all.loc[:, pd.IndexSlice[:, 'mean']].idxmax(),\n",
    "     'new': cv_feat_new.loc[:, pd.IndexSlice[:, 'mean']].idxmax()\n",
    "     }\n",
    ").droplevel(-1)\n",
    "n_feat_best"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2efdc8bf",
   "metadata": {},
   "source": [
    "## Train, test split\n",
    "Show number of cases in train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc3d3b21",
   "metadata": {
    "lines_to_next_cell": 0,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(\n",
    "    X,\n",
    "    target,\n",
    "    test_size=.2,\n",
    "    stratify=target_to_group,\n",
    "    random_state=42)\n",
    "idx_train = X_train.index\n",
    "idx_test = X_test.index\n",
    "\n",
    "njab.pandas.combine_value_counts(\n",
    "    pd.concat([y_train, y_test],\n",
    "              axis=1,\n",
    "              ignore_index=True,\n",
    "              ).rename(columns={0: 'train', 1: 'test'})\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3a33fb1",
   "metadata": {},
   "source": [
    "## Results\n",
    "\n",
    "- `run_model` returns dataclasses with the further needed results\n",
    "- add mrmr selection of data (select best number of features to use instead of fixing it)\n",
    "\n",
    "Save results for final model on entire data, new features and ALD study criteria selected data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3b4b394",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "splits = Splits(X_train=X.loc[idx_train],\n",
    "                X_test=X.loc[idx_test],\n",
    "                y_train=y_train,\n",
    "                y_test=y_test)\n",
    "results_model_full = njab.sklearn.run_model(\n",
    "    splits,\n",
    "    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'all'])\n",
    "results_model_full.name = f'{args.model_key} all'\n",
    "fname = args.out_folder / f'results_{results_model_full.name}.pkl'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.io.to_pickle(results_model_full, fname)\n",
    "\n",
    "splits = Splits(X_train=X.loc[idx_train, new_features],\n",
    "                X_test=X.loc[idx_test, new_features],\n",
    "                y_train=y_train,\n",
    "                y_test=y_test)\n",
    "results_model_new = njab.sklearn.run_model(\n",
    "    splits,\n",
    "    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'new'])\n",
    "results_model_new.name = f'{args.model_key} new'\n",
    "fname = args.out_folder / f'results_{results_model_new.name}.pkl'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.io.to_pickle(results_model_new, fname)\n",
    "\n",
    "splits_ald = Splits(\n",
    "    X_train=ald_study.loc[idx_train],\n",
    "    X_test=ald_study.loc[idx_test],\n",
    "    y_train=y_train,\n",
    "    y_test=y_test)\n",
    "results_ald_full = njab.sklearn.run_model(\n",
    "    splits_ald,\n",
    "    n_feat_to_select=n_feat_best.loc['test_roc_auc', 'ald'])\n",
    "results_ald_full.name = 'ALD study all'\n",
    "fname = args.out_folder / f'results_{results_ald_full.name}.pkl'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.io.to_pickle(results_ald_full, fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ad96ff4",
   "metadata": {},
   "source": [
    "### ROC-AUC on test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04b82583",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=figsize)\n",
    "plot_split_auc(results_ald_full.test, results_ald_full.name, ax)\n",
    "plot_split_auc(results_model_full.test, results_model_full.name, ax)\n",
    "plot_split_auc(results_model_new.test, results_model_new.name, ax)\n",
    "fname = args.out_folder / 'auc_roc_curve.pdf'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e35c686",
   "metadata": {},
   "source": [
    "Data used to plot ROC:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "322281db",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "res = [results_ald_full, results_model_full, results_model_new]\n",
    "\n",
    "auc_roc_curve = parse_roc(*res)\n",
    "auc_roc_curve.to_excel(fname.with_suffix('.xlsx'))\n",
    "auc_roc_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46e9a3f2-89aa-4bd5-a083-d8e16815020a",
   "metadata": {},
   "source": [
    "### Features selected for final models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e1bb173",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "selected_features = pd.DataFrame(\n",
    "    [results_ald_full.selected_features,\n",
    "     results_model_full.selected_features,\n",
    "     results_model_new.selected_features],\n",
    "    index=[\n",
    "        results_ald_full.name,\n",
    "        results_model_full.name,\n",
    "        results_model_new.name]\n",
    ").T\n",
    "selected_features.index.name = 'rank'\n",
    "fname = args.out_folder / 'mrmr_feat_by_model.xlsx'\n",
    "files_out[fname.name] = fname\n",
    "selected_features.to_excel(fname)\n",
    "selected_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce227174",
   "metadata": {},
   "source": [
    "### Precision-Recall plot on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56ea0d50",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=figsize)\n",
    "\n",
    "ax = plot_split_prc(results_ald_full.test, results_ald_full.name, ax)\n",
    "ax = plot_split_prc(results_model_full.test, results_model_full.name, ax)\n",
    "ax = plot_split_prc(results_model_new.test, results_model_new.name, ax)\n",
    "fname = folder = args.out_folder / 'prec_recall_curve.pdf'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc9e560d",
   "metadata": {},
   "source": [
    "Data used to plot PRC:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f5e5ce",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "prec_recall_curve = parse_prc(*res)\n",
    "prec_recall_curve.to_excel(fname.with_suffix('.xlsx'))\n",
    "prec_recall_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ddf0913",
   "metadata": {},
   "source": [
    "## Train data plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6eb3ed77",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=figsize)\n",
    "\n",
    "ax = plot_split_prc(results_ald_full.train, results_ald_full.name, ax)\n",
    "ax = plot_split_prc(results_model_full.train, results_model_full.name, ax)\n",
    "ax = plot_split_prc(results_model_new.train, results_model_new.name, ax)\n",
    "fname = folder = args.out_folder / 'prec_recall_curve_train.pdf'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64fee389",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=figsize)\n",
    "plot_split_auc(results_ald_full.train, results_ald_full.name, ax)\n",
    "plot_split_auc(results_model_full.train, results_model_full.name, ax)\n",
    "plot_split_auc(results_model_new.train, results_model_new.name, ax)\n",
    "fname = folder = args.out_folder / 'auc_roc_curve_train.pdf'\n",
    "files_out[fname.name] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "545b7a34",
   "metadata": {},
   "source": [
    "Output files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "860e0d5e",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "files_out"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}