experiments/Device_Mapping/DevMap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Copyright (c) 2021, S. VenkataKeerthy, Rohit Aggarwal\n",
    "# Department of Computer Science and Engineering, IIT Hyderabad\n",
    "#\n",
    "# This software is available under the BSD 4-Clause License. Please see LICENSE\n",
    "# file in the top-level directory for more details.\n",
    "#\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys, re\n",
    "from sklearn.model_selection import KFold\n",
    "import os\n",
    "from scipy.stats.mstats import gmean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (\n",
    "    os.path.exists(\"data/kernels_ir\")\n",
    "    and os.path.exists(\"data/cgo17-amd.csv\")\n",
    "    and os.path.exists(\"data/cgo17-nvidia.csv\")\n",
    "), \"Dataset is not present. Please download\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert os.path.exists(\"output/embeddings\"), \"Embeddings are not generated\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read data from input file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readEmd_program(filename):\n",
    "    lines = [line.strip(\"\\n\\t\") for line in open(filename)]\n",
    "    entity = []\n",
    "    rep = []\n",
    "    targetLabel = []\n",
    "    flag = 0\n",
    "    for line in lines:\n",
    "        r = line.split(\"\\t\")\n",
    "        targetLabel.append(int(r[0]))\n",
    "        res = r[1:]\n",
    "        res_double = [float(val) for val in res]\n",
    "        rep.append(res_double)\n",
    "    return rep, targetLabel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results from other works\n",
    "\n",
    "The accuracies and speedups are taken from the results quoted by NCC in their work for the purpose of comparison. For detailed analysis (discussed later), we run these models and the obtained results are stored as pickle files in ./data/prior_art_results. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "static_pred_vals = [58.823529, 56.911765]\n",
    "static_pred_mean = [57.867647]\n",
    "static_sp_vals = [1.0, 1.0]\n",
    "static_sp_mean = [1.0]\n",
    "grewe_pred_vals = [73.382353, 72.941176]\n",
    "grewe_pred_mean = [73.161765]\n",
    "grewe_sp_vals = [2.905822, 1.264801]\n",
    "grewe_sp_mean = [2.085312]\n",
    "deeptune_pred_vals = [83.676471, 80.294118]\n",
    "deeptune_pred_mean = [81.985294]\n",
    "deeptune_sp_vals = [3.335612, 1.412222]\n",
    "deeptune_sp_mean = [2.373917]\n",
    "ncc_pred_vals = [82.79, 81.76]\n",
    "ncc_pred_mean = [82.275]\n",
    "ncc_sp_vals = [3.42, 1.39]\n",
    "ncc_sp_mean = [2.405]\n",
    "\n",
    "llfiles = pd.read_csv(\"./data/all.txt\", sep=\"\\s+\")\n",
    "fileNum = llfiles[\"FileNum\"]\n",
    "filesname = llfiles[\"ProgramName\"]\n",
    "\n",
    "device_dict = {\"amd\": \"AMD Tahiti 7970\", \"nvidia\": \"NVIDIA GTX 970\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(max_depth=4, learning_rate=0.1, n_estimators=200, seed=204):\n",
    "    data = []\n",
    "    rt_label_dict = {\"amd\": \"runtime_cpu\", \"nvidia\": \"runtime_gpu\"}\n",
    "\n",
    "    for i, platform in enumerate(device_dict.keys()):\n",
    "        platform_name = device_dict[platform]\n",
    "\n",
    "        # Load runtime data\n",
    "        df = pd.read_csv(\"./data/cgo17-{}.csv\".format(platform))\n",
    "        df[\"bench_data\"] = (\n",
    "            df.loc[df[\"dataset\"] != \"default\", \"benchmark\"]\n",
    "            + str(\"_\")\n",
    "            + df.loc[df[\"dataset\"] != \"default\", \"dataset\"]\n",
    "        )\n",
    "\n",
    "        df.loc[df[\"dataset\"] == \"default\", \"bench_data\"] = df.loc[\n",
    "            df[\"dataset\"] == \"default\", \"benchmark\"\n",
    "        ]\n",
    "        df[\"bench_data_path\"] = str(\"./\") + df[\"bench_data\"] + str(\".ll\")\n",
    "\n",
    "        raw_embeddings_pd = pd.DataFrame(raw_embeddings, columns=range(1, 301))\n",
    "        efileNum = pd.DataFrame(fileIndexNum)\n",
    "        embeddings = raw_embeddings_pd\n",
    "        embeddingsData = pd.concat([efileNum, embeddings], axis=1)\n",
    "        embeddingsData = embeddingsData.merge(llfiles, left_on=0, right_on=\"FileNum\")\n",
    "\n",
    "        df = pd.merge(\n",
    "            embeddingsData, df, left_on=\"ProgramName\", right_on=\"bench_data_path\"\n",
    "        )\n",
    "        targetLabel = np.array([1 if x == \"GPU\" else 0 for x in df[\"oracle\"].values])\n",
    "\n",
    "        embeddings = df.drop(\n",
    "            columns=[\n",
    "                \"dataset\",\n",
    "                \"comp\",\n",
    "                \"rational\",\n",
    "                \"mem\",\n",
    "                \"localmem\",\n",
    "                \"coalesced\",\n",
    "                \"atomic\",\n",
    "                \"runtime_cpu\",\n",
    "                \"runtime_gpu\",\n",
    "                0,\n",
    "                \"src\",\n",
    "                \"seq\",\n",
    "                \"bench_data\",\n",
    "                \"bench_data_path\",\n",
    "                \"ProgramName\",\n",
    "                \"FileNum\",\n",
    "                \"Unnamed: 0\",\n",
    "                \"benchmark\",\n",
    "                \"oracle\",\n",
    "            ]\n",
    "        )\n",
    "        embeddings = (embeddings - embeddings.min()) / (\n",
    "            embeddings.max() - embeddings.min()\n",
    "        )\n",
    "        embeddings = np.array(embeddings)\n",
    "\n",
    "        from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "        # 10-fold cross-validation\n",
    "        n_splits = 10\n",
    "        kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n",
    "        for j, (train, test) in enumerate(kf.split(embeddings, targetLabel)):\n",
    "\n",
    "            model = xgb.XGBClassifier(\n",
    "                max_depth=max_depth,\n",
    "                learning_rate=learning_rate,\n",
    "                n_estimators=n_estimators,\n",
    "                n_jobs=10,\n",
    "            )\n",
    "            model.fit(embeddings[train], targetLabel[train])\n",
    "            predictions = model.predict(embeddings[test])\n",
    "\n",
    "            predictions = [\n",
    "                \"CPU\" if prediction == 0 else \"GPU\" for prediction in predictions\n",
    "            ]\n",
    "            test_df = df.iloc[test].reset_index()\n",
    "            assert test_df.shape[0] == len(predictions)\n",
    "            test_df = pd.concat(\n",
    "                [test_df, pd.DataFrame(predictions, columns=[\"predictions\"])], axis=1\n",
    "            )\n",
    "\n",
    "            rt_label = rt_label_dict[platform]\n",
    "            for idx, row in test_df.iterrows():\n",
    "                oracle = row[\"oracle\"]\n",
    "                pred = row[\"predictions\"]\n",
    "                rt_baseline = row[rt_label]\n",
    "                rt_oracle = (\n",
    "                    row[\"runtime_cpu\"] if oracle == \"CPU\" else row[\"runtime_gpu\"]\n",
    "                )\n",
    "                rt_pred = row[\"runtime_cpu\"] if pred == \"CPU\" else row[\"runtime_gpu\"]\n",
    "                data.append(\n",
    "                    {\n",
    "                        \"Model\": \"IR2vec\",\n",
    "                        \"Platform\": platform_name,\n",
    "                        \"Oracle Mapping\": oracle,\n",
    "                        \"Predicted Mapping\": pred,\n",
    "                        \"Correct?\": oracle == pred,\n",
    "                        \"Speedup\": rt_baseline / rt_pred,\n",
    "                        \"OracleSpeedUp\": rt_baseline / rt_oracle,\n",
    "                    }\n",
    "                )\n",
    "        ir2vec = pd.DataFrame(data, index=range(1, len(data) + 1))\n",
    "\n",
    "    print(\"Accuracy Matrix: IR2Vec Vs. others\\n\")\n",
    "    ir2vec_pred_vals = ir2vec.groupby([\"Platform\"])[\"Correct?\"].mean().values * 100\n",
    "    ir2vec_pred_mean = ir2vec_pred_vals.mean()\n",
    "    accuracy_df = pd.DataFrame(\n",
    "        {\n",
    "            \"Static Mapping\": static_pred_vals + static_pred_mean,\n",
    "            \"Grewe et al.\": grewe_pred_vals + grewe_pred_mean,\n",
    "            \"DeepTune\": deeptune_pred_vals + deeptune_pred_mean,\n",
    "            \"NCC\": ncc_pred_vals + ncc_pred_mean,\n",
    "            \"IR2Vec\": list(ir2vec_pred_vals) + [ir2vec_pred_mean],\n",
    "        },\n",
    "        index=[\"AMD Tahiti 7970\", \"NVIDIA GTX 970\", \"Average\"],\n",
    "    )\n",
    "    print(accuracy_df)\n",
    "\n",
    "    print(\"\\nSpeedup Matrix: IR2Vec Vs. others\\n\")\n",
    "    ir2vec_sp_vals = ir2vec.groupby([\"Platform\"])[\"Speedup\"].mean().values\n",
    "    ir2vec_sp_mean = ir2vec_sp_vals.mean()\n",
    "    sp_df = pd.DataFrame(\n",
    "        {\n",
    "            \"Static Mapping\": static_sp_vals + static_sp_mean,\n",
    "            \"Grewe et al.\": grewe_sp_vals + grewe_sp_mean,\n",
    "            \"DeepTune\": deeptune_sp_vals + deeptune_sp_mean,\n",
    "            \"NCC\": ncc_sp_vals + ncc_sp_mean,\n",
    "            \"IR2Vec\": list(ir2vec_sp_vals) + [ir2vec_sp_mean],\n",
    "        },\n",
    "        index=[\"AMD Tahiti 7970\", \"NVIDIA GTX 970\", \"Average\"],\n",
    "    )\n",
    "    print(sp_df)\n",
    "\n",
    "    return ir2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IR2Vec Symbolic Vs. Others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Matrix: IR2Vec Vs. others\n",
      "\n",
      "                 Static Mapping  Grewe et al.   DeepTune     NCC     IR2Vec\n",
      "AMD Tahiti 7970       58.823529     73.382353  83.676471  82.790  90.284006\n",
      "NVIDIA GTX 970        56.911765     72.941176  80.294118  81.760  87.144993\n",
      "Average               57.867647     73.161765  81.985294  82.275  88.714499\n",
      "\n",
      "Speedup Matrix: IR2Vec Vs. others\n",
      "\n",
      "                 Static Mapping  Grewe et al.  DeepTune    NCC    IR2Vec\n",
      "AMD Tahiti 7970             1.0      2.905822  3.335612  3.420  3.471963\n",
      "NVIDIA GTX 970              1.0      1.264801  1.412222  1.390  1.433372\n",
      "Average                     1.0      2.085312  2.373917  2.405  2.452667\n"
     ]
    }
   ],
   "source": [
    "raw_embeddings, fileIndexNum = readEmd_program(\n",
    "    \"./output/embeddings/Device_Mapping_Symbolic_llvm12.txt\"\n",
    ")\n",
    "ir2vec_sym = evaluate(max_depth=10, learning_rate=0.5, n_estimators=70, seed=104)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Expected Results\n",
    "Accuracy Matrix: IR2Vec Vs. others\n",
    "\n",
    "                 Static Mapping  Grewe et al.   DeepTune     NCC     IR2Vec\n",
    "AMD Tahiti 7970       58.823529     73.382353  83.676471  82.790  90.284006\n",
    "NVIDIA GTX 970        56.911765     72.941176  80.294118  81.760  87.144993\n",
    "Average               57.867647     73.161765  81.985294  82.275  88.714499\n",
    "\n",
    "Speedup Matrix: IR2Vec Vs. others\n",
    "\n",
    "                 Static Mapping  Grewe et al.  DeepTune    NCC    IR2Vec\n",
    "AMD Tahiti 7970             1.0      2.905822  3.335612  3.420  3.471963\n",
    "NVIDIA GTX 970              1.0      1.264801  1.412222  1.390  1.433372\n",
    "Average                     1.0      2.085312  2.373917  2.405  2.452667"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IR2Vec Flow-Aware Vs. Others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Matrix: IR2Vec Vs. others\n",
      "\n",
      "                 Static Mapping  Grewe et al.   DeepTune     NCC     IR2Vec\n",
      "AMD Tahiti 7970       58.823529     73.382353  83.676471  82.790  92.825112\n",
      "NVIDIA GTX 970        56.911765     72.941176  80.294118  81.760  89.686099\n",
      "Average               57.867647     73.161765  81.985294  82.275  91.255605\n",
      "\n",
      "Speedup Matrix: IR2Vec Vs. others\n",
      "\n",
      "                 Static Mapping  Grewe et al.  DeepTune    NCC    IR2Vec\n",
      "AMD Tahiti 7970             1.0      2.905822  3.335612  3.420  3.510104\n",
      "NVIDIA GTX 970              1.0      1.264801  1.412222  1.390  1.467221\n",
      "Average                     1.0      2.085312  2.373917  2.405  2.488663\n"
     ]
    }
   ],
   "source": [
    "raw_embeddings, fileIndexNum = readEmd_program(\n",
    "    \"./output/embeddings/Device_Mapping_FlowAware_llvm12.txt\"\n",
    ")\n",
    "ir2vec_fa = evaluate(max_depth=10, learning_rate=0.5, n_estimators=70, seed=104)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Expected Results\n",
    "Accuracy Matrix: IR2Vec Vs. others\n",
    "\n",
    "                 Static Mapping  Grewe et al.   DeepTune     NCC     IR2Vec\n",
    "AMD Tahiti 7970       58.823529     73.382353  83.676471  82.790  92.825112\n",
    "NVIDIA GTX 970        56.911765     72.941176  80.294118  81.760  89.686099\n",
    "Average               57.867647     73.161765  81.985294  82.275  91.255605\n",
    "\n",
    "Speedup Matrix: IR2Vec Vs. others\n",
    "\n",
    "                 Static Mapping  Grewe et al.  DeepTune    NCC    IR2Vec\n",
    "AMD Tahiti 7970             1.0      2.905822  3.335612  3.420  3.510104\n",
    "NVIDIA GTX 970              1.0      1.264801  1.412222  1.390  1.467221\n",
    "Average                     1.0      2.085312  2.373917  2.405  2.488663\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Other related observations\n",
    "For the comparison, we use the results obtained on training the earlier works  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "deeptune_res = pd.read_pickle(\"data/prior_art_results/deeptune_dm.results\")\n",
    "grewe_res = pd.read_pickle(\"data/prior_art_results/grewe_dm.results\")\n",
    "static_res = pd.read_pickle(\"data/prior_art_results/static_dm.results\")\n",
    "ncc_res = pd.read_pickle(\"data/prior_art_results/ncc_fix_DM.results\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Speedup comparisons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calcSpeedup(platform):\n",
    "    grewe_geomean = gmean(\n",
    "        grewe_res[grewe_res[\"Platform\"] == platform][\"Speedup\"].values\n",
    "    )\n",
    "    deeptune_geomean = gmean(\n",
    "        deeptune_res[deeptune_res[\"Platform\"] == platform][\"Speedup\"].values\n",
    "    )\n",
    "    ncc_geomean = gmean(ncc_res[ncc_res[\"Platform\"] == platform][\"Speedup\"].values)\n",
    "    ir2vec_sym_geomean = gmean(\n",
    "        ir2vec_sym[ir2vec_sym[\"Platform\"] == platform][\"Speedup\"].values\n",
    "    )\n",
    "    ir2vec_fa_geomean = gmean(\n",
    "        ir2vec_fa[ir2vec_fa[\"Platform\"] == platform][\"Speedup\"].values\n",
    "    )\n",
    "\n",
    "    print(f\"Geometric mean of Grewe et al. {grewe_geomean:.2f}x\")\n",
    "    print(f\"Geometric mean of DeepTune {deeptune_geomean:.2f}x\")\n",
    "    print(f\"Geometric mean of Inst2Vec {ncc_geomean:.2f}x\")\n",
    "    print(f\"Geometric mean of IR2Vec Symbolic {ir2vec_sym_geomean:.3f}x\")\n",
    "    print(f\"Geometric mean of IR2Vec Flow-Aware {ir2vec_fa_geomean:.3f}x\")\n",
    "\n",
    "    return (\n",
    "        grewe_geomean,\n",
    "        deeptune_geomean,\n",
    "        ncc_geomean,\n",
    "        ir2vec_sym_geomean,\n",
    "        ir2vec_fa_geomean,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On AMD Tahiti 7970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Geometric mean of Grewe et al. 1.22x\n",
      "Geometric mean of DeepTune 1.46x\n",
      "Geometric mean of Inst2Vec 1.40x\n",
      "Geometric mean of IR2Vec Symbolic 1.559x\n",
      "Geometric mean of IR2Vec Flow-Aware 1.581x\n"
     ]
    }
   ],
   "source": [
    "tah_grewe, tah_dt, tah_ncc, tah_ir2vSym, tah_ir2vFA = calcSpeedup(\"AMD Tahiti 7970\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On NVIDIA GTX 970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Geometric mean of Grewe et al. 1.11x\n",
      "Geometric mean of DeepTune 1.21x\n",
      "Geometric mean of Inst2Vec 1.21x\n",
      "Geometric mean of IR2Vec Symbolic 1.237x\n",
      "Geometric mean of IR2Vec Flow-Aware 1.261x\n"
     ]
    }
   ],
   "source": [
    "gtx_grewe, gtx_dt, gtx_ncc, gtx_ir2vSym, gtx_ir2vFA = calcSpeedup(\"NVIDIA GTX 970\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On both the platforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Geometric mean of Grewe et al. - 1.16x\n",
      "Geometric mean of DeepTune - 1.33x\n",
      "Geometric mean of Inst2Vec - 1.30x\n",
      "Geometric mean of IR2Vec Symbolic 1.39x\n",
      "Geometric mean of IR2Vec Flow-Aware 1.41x\n"
     ]
    }
   ],
   "source": [
    "grewe_geomean = gmean(grewe_res[\"Speedup\"].values)\n",
    "deeptune_geomean = gmean(deeptune_res[\"Speedup\"].values)\n",
    "ncc_geomean = gmean(ncc_res[\"Speedup\"].values)\n",
    "ir2vec_sym_geomean = gmean(ir2vec_sym[\"Speedup\"].values)\n",
    "ir2vec_fa_geomean = gmean(ir2vec_fa[\"Speedup\"].values)\n",
    "\n",
    "print(f\"Geometric mean of Grewe et al. - {grewe_geomean:.2f}x\")\n",
    "print(f\"Geometric mean of DeepTune - {deeptune_geomean:.2f}x\")\n",
    "print(f\"Geometric mean of Inst2Vec - {ncc_geomean:.2f}x\")\n",
    "print(f\"Geometric mean of IR2Vec Symbolic {ir2vec_sym_geomean:.2f}x\")\n",
    "print(f\"Geometric mean of IR2Vec Flow-Aware {ir2vec_fa_geomean:.2f}x\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Percentage of increase in speedup by IR2Vec Flow-Aware encodings over others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def slowDown(value1, value2):\n",
    "    return round(np.abs(((value2 - value1) / value2) * 100), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "AMD Tahiti 7970\n",
      " % Increase in SpeedUp over Grewe et al -  29.65\n",
      " % Increase in SpeedUp over DeepTune -  7.96\n",
      " % Increase in SpeedUp over Inst2Vec -  12.95\n",
      " % Increase in SpeedUp over IR2Vec Symbolic -  1.43\n",
      "\n",
      "NVIDIA GTX 970\n",
      " % Increase in SpeedUp over Grewe et al -  13.77\n",
      " % Increase in SpeedUp over DeepTune -  4.31\n",
      " % Increase in SpeedUp over Inst2Vec -  3.93\n",
      " % Increase in SpeedUp over IR2Vec Symbolic -  1.92\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nAMD Tahiti 7970\")\n",
    "print(\" % Increase in SpeedUp over Grewe et al - \", slowDown(tah_ir2vFA, tah_grewe))\n",
    "print(\" % Increase in SpeedUp over DeepTune - \", slowDown(tah_ir2vFA, tah_dt))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(tah_ir2vFA, tah_ncc))\n",
    "print(\n",
    "    \" % Increase in SpeedUp over IR2Vec Symbolic - \",\n",
    "    slowDown(tah_ir2vFA, tah_ir2vSym),\n",
    ")\n",
    "\n",
    "print(\"\\nNVIDIA GTX 970\")\n",
    "print(\" % Increase in SpeedUp over Grewe et al - \", slowDown(gtx_ir2vFA, gtx_grewe))\n",
    "print(\" % Increase in SpeedUp over DeepTune - \", slowDown(gtx_ir2vFA, gtx_dt))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(gtx_ir2vFA, gtx_ncc))\n",
    "print(\n",
    "    \" % Increase in SpeedUp over IR2Vec Symbolic - \",\n",
    "    slowDown(gtx_ir2vFA, gtx_ir2vSym),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy Comparisons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calcAccuracy(platform):\n",
    "    grewe_acc = (\n",
    "        grewe_res[grewe_res[\"Platform\"] == platform]\n",
    "        .groupby([\"Platform\"])[\"Correct?\"]\n",
    "        .mean()\n",
    "        .values\n",
    "        * 100\n",
    "    )\n",
    "    deeptune_acc = (\n",
    "        deeptune_res[deeptune_res[\"Platform\"] == platform]\n",
    "        .groupby([\"Platform\"])[\"Correct?\"]\n",
    "        .mean()\n",
    "        .values\n",
    "        * 100\n",
    "    )\n",
    "    ncc_acc = (\n",
    "        ncc_res[ncc_res[\"Platform\"] == platform]\n",
    "        .groupby([\"Platform\"])[\"Correct?\"]\n",
    "        .mean()\n",
    "        .values\n",
    "        * 100\n",
    "    )\n",
    "    ir2vec_sym_acc = (\n",
    "        ir2vec_sym[ir2vec_sym[\"Platform\"] == platform]\n",
    "        .groupby([\"Platform\"])[\"Correct?\"]\n",
    "        .mean()\n",
    "        .values\n",
    "        * 100\n",
    "    )\n",
    "    ir2vec_fa_acc = (\n",
    "        ir2vec_fa[ir2vec_fa[\"Platform\"] == platform]\n",
    "        .groupby([\"Platform\"])[\"Correct?\"]\n",
    "        .mean()\n",
    "        .values\n",
    "        * 100\n",
    "    )\n",
    "    print(f\"Accuracy of Grewe et al. {grewe_acc[0]:.2f}%\")\n",
    "    print(f\"Accuracy of DeepTune {deeptune_acc[0]:.2f}%\")\n",
    "    print(f\"Accuracy of Inst2Vec {ncc_acc[0]:.2f}%\")\n",
    "    print(f\"Accuracy of IR2Vec Symbolic {ir2vec_sym_acc[0]:.3f}%\")\n",
    "    print(f\"Accuracy of IR2Vec Flow-Aware {ir2vec_fa_acc[0]:.3f}%\")\n",
    "\n",
    "    return (\n",
    "        grewe_acc[0],\n",
    "        deeptune_acc[0],\n",
    "        ncc_acc[0],\n",
    "        ir2vec_sym_acc[0],\n",
    "        ir2vec_fa_acc[0],\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On AMD Tahiti 7970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Grewe et al. 70.29%\n",
      "Accuracy of DeepTune 83.24%\n",
      "Accuracy of Inst2Vec 82.35%\n",
      "Accuracy of IR2Vec Symbolic 90.284%\n",
      "Accuracy of IR2Vec Flow-Aware 92.825%\n"
     ]
    }
   ],
   "source": [
    "tah_grewe, tah_dt, tah_ncc, tah_ir2vSym, tah_ir2vFA = calcAccuracy(\"AMD Tahiti 7970\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On NVIDIA GTX 970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Grewe et al. 74.56%\n",
      "Accuracy of DeepTune 80.88%\n",
      "Accuracy of Inst2Vec 82.65%\n",
      "Accuracy of IR2Vec Symbolic 87.145%\n",
      "Accuracy of IR2Vec Flow-Aware 89.686%\n"
     ]
    }
   ],
   "source": [
    "gtx_grewe, gtx_dt, gtx_ncc, gtx_ir2vSym, gtx_ir2vFA = calcAccuracy(\"NVIDIA GTX 970\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Percentage of improvement in accuracy obtained by Flow Aware embeddings when compared to other methods\n",
    "\n",
    "Calculated based on the reference values taken from https://github.com/spcl/ncc/blob/master/train_task_devmap.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On AMD Tahiti 7970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "AMD Tahiti 7970\n",
      " % Increase in SpeedUp over Grewe et al -  26.5\n",
      " % Increase in SpeedUp over DeepTune -  10.93\n",
      " % Increase in SpeedUp over Inst2Vec -  12.12\n",
      " % Increase in SpeedUp over Inst2Vec-imm -  5.38\n",
      " % Increase in SpeedUp over IR2Vec Symbolic -  2.81\n"
     ]
    }
   ],
   "source": [
    "# AMD Tahiti 7970\n",
    "tah_grewe = 73.382353\n",
    "tah_dt = 83.676471\n",
    "tah_ncc = 82.790\n",
    "tah_nccimm = 88.09\n",
    "\n",
    "print(\"\\nAMD Tahiti 7970\")\n",
    "print(\" % Increase in SpeedUp over Grewe et al - \", slowDown(tah_ir2vFA, tah_grewe))\n",
    "print(\" % Increase in SpeedUp over DeepTune - \", slowDown(tah_ir2vFA, tah_dt))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(tah_ir2vFA, tah_ncc))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec-imm - \", slowDown(tah_ir2vFA, tah_nccimm))\n",
    "print(\n",
    "    \" % Increase in SpeedUp over IR2Vec Symbolic - \",\n",
    "    slowDown(tah_ir2vFA, tah_ir2vSym),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On NVIDIA GTX 970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "NVIDIA GTX 970\n",
      " % Increase in SpeedUp over Grewe et al -  22.96\n",
      " % Increase in SpeedUp over DeepTune -  11.7\n",
      " % Increase in SpeedUp over Inst2Vec -  9.69\n",
      " % Increase in SpeedUp over Inst2Vec -  3.54\n",
      " % Increase in SpeedUp over IR2Vec Symbolic -  2.92\n"
     ]
    }
   ],
   "source": [
    "# NVIDIA GTX 970\n",
    "static = 56.911765\n",
    "gtx_grewe = 72.941176\n",
    "gtx_dt = 80.294118\n",
    "gtx_ncc = 81.760\n",
    "gtx_nccimm = 86.62\n",
    "\n",
    "\n",
    "print(\"\\nNVIDIA GTX 970\")\n",
    "print(\" % Increase in SpeedUp over Grewe et al - \", slowDown(gtx_ir2vFA, gtx_grewe))\n",
    "print(\" % Increase in SpeedUp over DeepTune - \", slowDown(gtx_ir2vFA, gtx_dt))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(gtx_ir2vFA, gtx_ncc))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(gtx_ir2vFA, gtx_nccimm))\n",
    "print(\n",
    "    \" % Increase in SpeedUp over IR2Vec Symbolic - \",\n",
    "    slowDown(gtx_ir2vFA, gtx_ir2vSym),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On both the platforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " % Increase in SpeedUp over DeepTune -  11.3\n",
      " % Increase in SpeedUp over Inst2Vec -  10.92\n",
      " % Increase in SpeedUp over Inst2Vec -  4.47\n",
      " % Increase in SpeedUp over IR2Vec Symbolic -  2.86\n"
     ]
    }
   ],
   "source": [
    "dt = 81.99\n",
    "ncc = 82.275\n",
    "nccimm = (88.09 + 86.62) / 2\n",
    "ir2vSym = ir2vec_sym[\"Correct?\"].mean() * 100\n",
    "ir2vFA = ir2vec_fa[\"Correct?\"].mean() * 100\n",
    "\n",
    "print(\" % Increase in SpeedUp over DeepTune - \", slowDown(ir2vFA, dt))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(ir2vFA, ncc))\n",
    "print(\" % Increase in SpeedUp over Inst2Vec - \", slowDown(ir2vFA, nccimm))\n",
    "print(\n",
    "    \" % Increase in SpeedUp over IR2Vec Symbolic - \",\n",
    "    slowDown(ir2vFA, ir2vSym),\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}