analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48e5139d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlflow\n",
    "from mlflow.tracking import MlflowClient\n",
    "\n",
    "import os, sys\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "HOME = os.environ[\"HOME\"]\n",
    "CARDIAC_COMA_REPO = f\"{HOME}/01_repos/CardiacCOMA/\"\n",
    "import os; os.chdir(CARDIAC_COMA_REPO)\n",
    "\n",
    "from config.load_config import load_yaml_config, to_dict\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import interact\n",
    "from IPython.display import Image\n",
    "\n",
    "import pandas as pd\n",
    "import shlex\n",
    "from subprocess import check_output\n",
    "\n",
    "import pickle as pkl\n",
    "import pytorch_lightning as pl\n",
    "\n",
    "from argparse import Namespace\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython import embed\n",
    "sys.path.insert(0, '..')\n",
    "\n",
    "# import model.Model3D\n",
    "# from utils.helpers import get_coma_args, get_lightning_module, get_datamodule\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "\n",
    "from typing import List\n",
    "from tqdm import tqdm\n",
    "\n",
    "import pyvista as pv\n",
    "from ipywidgets import interact, interactive, fixed, interact_manual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e541b2e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.mlflow_queries import \\\n",
    "    list_artifacts,\\\n",
    "    get_significant_loci,\\\n",
    "    get_metrics_cols, \\\n",
    "    get_params_cols, \\\n",
    "    get_runs_df, \\\n",
    "    get_good_runs,\\\n",
    "    summarize_loci_across_runs,\\\n",
    "    get_model_pretrained_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b04fe2c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRACKING_URI = f\"file://{CARDIAC_COMA_REPO}/mlruns\"\n",
    "mlflow.set_tracking_uri(TRACKING_URI)\n",
    "client = MlflowClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45285f00",
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment_selection_widget():\n",
    "    \n",
    "    '''\n",
    "    Returns a selection widget for MLflow experiments.\n",
    "    '''\n",
    "    \n",
    "    options = [exp.name for exp in mlflow.list_experiments()]\n",
    "\n",
    "    experiment_w = widgets.Select(\n",
    "      options=options,\n",
    "      value=\"Cardiac - ED\"\n",
    "    )\n",
    "    \n",
    "    return experiment_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e154444e",
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_w = experiment_selection_widget()\n",
    "\n",
    "@interact\n",
    "def get_runs(exp_name=exp_w):\n",
    "    try:  \n",
    "        global runs_df\n",
    "        exp_id = mlflow.get_experiment_by_name(exp_name).experiment_id\n",
    "        runs_df = get_runs_df(exp_name=exp_name, only_finished=True)\n",
    "        metrics, params    = get_metrics_cols(runs_df), get_params_cols(runs_df)  \n",
    "        # display(runs_df.loc[:, [*metrics, *params]].drop(\"params.platform\", axis=1).head(10))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0b6e021",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_col_names(exper_id, run_id, df):\n",
    "    df.columns = [f\"{exper_id}_{run_id[:5]}_{col}\" for col in df.columns]\n",
    "    return df\n",
    "\n",
    "def path_to_z(row):\n",
    "    run_id = (row.experiment_id, row.run_id)\n",
    "    artifacts_dir = row.artifact_uri.replace(\"file://\", \"\") \n",
    "    z_adj_path = f'''output/z_adj_{row.experiment_id}_{row.run_id}.tsv'''    \n",
    "    z_adj_path = os.path.join(artifacts_dir, z_adj_path)    \n",
    "    return (run_id, z_adj_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14ff0f44",
   "metadata": {},
   "outputs": [],
   "source": [
    "z_paths = runs_df.reset_index().apply(path_to_z, axis=1)\n",
    "z_paths = dict(z_paths.values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd69f88a",
   "metadata": {},
   "outputs": [],
   "source": [
    "z_dfs = {}\n",
    "for run_id, z_path in tqdm(z_paths.items()):\n",
    "    try:\n",
    "        z_dfs[run_id] = pd.read_csv(z_path, sep=\"\\t\").set_index(\"ID\")\n",
    "    except FileNotFoundError:\n",
    "        # print(f\"Latent vector file does not exist for {run_id}\")\n",
    "        pass\n",
    "    \n",
    "z_dfs_renamed = [change_col_names(expid, runid, z_df) for (expid, runid), z_df in z_dfs.items()]    \n",
    "z_all_df = pd.concat(z_dfs_renamed, axis=1)\n",
    "z_all_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f10c7d64",
   "metadata": {},
   "source": [
    "# Genomic PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12b350d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "GENOMIC_PC_FILE = f'''{os.environ[\"HOME\"]}/01_repos/GWAS_pipeline/data/transforms/GenomicPCA/pcs.txt'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24ad0f1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "genomic_pca_df = pd.read_csv(GENOMIC_PC_FILE, sep=\"\\t\")\n",
    "genomic_pca_df = genomic_pca_df.set_index(\"IID\").drop(\"FID\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78f3a196",
   "metadata": {},
   "outputs": [],
   "source": [
    "genomic_pca_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abedc1b9",
   "metadata": {},
   "source": [
    "### Correlation genomic PCs vs. latent variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02a47b7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f76e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import spearmanr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb30810",
   "metadata": {},
   "outputs": [],
   "source": [
    "genomic_pca_df.loc[z_all_df.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34be88c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "spearman_coef, spearman_pvalue = spearmanr(a=z_all_df, b=genomic_pca_df.loc[z_all_df.index])\n",
    "np.log10(pd.DataFrame(spearman_pvalue[:-10,-10:])).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc1002c7",
   "metadata": {},
   "source": [
    "### Correlation genomic PCs vs. traditional cardiac indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69f6557b",
   "metadata": {},
   "outputs": [],
   "source": [
    "timeframe = \"1\".zfill(3) # 001 --> end-diastole\n",
    "datafolder = \"data/cardio/cardiac_indices\"\n",
    "\n",
    "df = pd.concat([\n",
    "    pd.read_csv(f\"{datafolder}/G{i}/LVRV_time{timeframe}.csv\", index_col=\"case_id\") \n",
    "    for i in range(1,5)\n",
    "])\n",
    "\n",
    "df.index = df.index.astype(str)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d247ac21",
   "metadata": {},
   "outputs": [],
   "source": [
    "sph_df = pd.read_csv(\"data/cardio/sphericity.csv\").set_index(\"id\")\n",
    "sph_df.index = sph_df.index.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "664cecb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "cardiac_indices_df = df.merge(sph_df, left_index=True, right_index=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}