rl_vpg_cartpole.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vanilla policy gradient in keras with Reward-To-Go\n",
    "# https://spinningup.openai.com/en/latest/spinningup/rl_intro3.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "import numpy as np\n",
    "import gym\n",
    "from tqdm import trange\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.optimizers import Adam, SGD\n",
    "from tensorflow.keras.layers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Box(4,), Discrete(2))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make(\"CartPole-v1\")\n",
    "env.observation_space, env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 4)]               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 32)                160       \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 66        \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 226\n",
      "Trainable params: 226\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "x = in1 = Input(env.observation_space.shape)\n",
    "x = Dense(32)(x)\n",
    "x = Activation('tanh')(x)\n",
    "x = Dense(env.action_space.n)(x)\n",
    "x = Lambda(lambda x: tf.nn.log_softmax(x, axis=-1))(x)\n",
    "m = Model(in1, x)\n",
    "def loss(y_true, y_pred):\n",
    "  # y_pred is the log probs of the actions\n",
    "  # y_true is the action mask weighted by sum of rewards\n",
    "  return -tf.reduce_sum(y_true*y_pred, axis=-1)\n",
    "m.compile(Adam(1e-2), loss)\n",
    "m.summary()\n",
    "lll = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ep_rew: 496.06    loss: 136.14: 100%|██████████| 40/40 [00:59<00:00,  1.56s/it]\n"
     ]
    }
   ],
   "source": [
    "# this is like 5x faster than calling m.predict and picking in numpy\n",
    "pf = K.function(m.layers[0].input, tf.random.categorical(m.layers[-1].output, 1)[0])\n",
    "\n",
    "tt = trange(40)\n",
    "for epoch in tt:\n",
    "  X,Y = [], []\n",
    "  ll = []\n",
    "  while len(X) < 8192:\n",
    "    obs = env.reset()\n",
    "    acts, rews = [], []\n",
    "    while True:\n",
    "      # pick action\n",
    "      #act_dist = np.exp(m.predict_on_batch(obs[None])[0])\n",
    "      #act = np.random.choice(range(env.action_space.n), p=act_dist)\n",
    "      \n",
    "      # pick action (fast!)\n",
    "      act = pf(obs[None])[0]\n",
    "      \n",
    "      # save this state action pair\n",
    "      X.append(np.copy(obs))\n",
    "      acts.append(act)\n",
    "      \n",
    "      # take the action\n",
    "      obs, rew, done, _ = env.step(act)\n",
    "      rews.append(rew)\n",
    "      \n",
    "      if done:\n",
    "        for i, act in enumerate(acts):\n",
    "          act_mask = np.zeros((env.action_space.n))\n",
    "          act_mask[act] = np.sum(rews[i:])\n",
    "          Y.append(act_mask)\n",
    "        ll.append(np.sum(rews))\n",
    "        break\n",
    "        \n",
    "  loss = m.train_on_batch(np.array(X), np.array(Y))\n",
    "  lll.append((np.mean(ll), loss))\n",
    "  tt.set_description(\"ep_rew:%7.2f    loss:%7.2f\" % lll[-1])\n",
    "  tt.refresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x134db16a0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VFX+//HXSW8QkhBCCpAAoYYQIHRCVURQEMWCDVd32a9ldYsF3fUrrrqrqz9ZF/eLi4u9gIooYkUUqYKhtwAJhJBQEkJ6z8z5/XFvYsAAk5DkTobP8/GYx71z52bm4zW85+bcc89RWmuEEEK4LjerCxBCCNG8JOiFEMLFSdALIYSLk6AXQggXJ0EvhBAuToJeCCFcnAS9EEK4OAl6IYRwcRL0Qgjh4jysLgCgffv2Ojo62uoyhBCiVdmyZcsprXXohfZziqCPjo4mOTnZ6jKEEKJVUUodcWQ/aboRQggXJ0EvhBAuToJeCCFcnFO00denqqqKzMxMysvLrS5FtFI+Pj5ERUXh6elpdSlCWMppgz4zM5M2bdoQHR2NUsrqckQro7UmNzeXzMxMYmJirC5HCEs51HSjlEpXSu1SSm1XSiWb24KVUiuVUgfNZZC5XSml/qWUSlVK7VRKDWxMYeXl5YSEhEjIi0ZRShESEiJ/EQpBw9rox2mtE7TWiebzOcAqrXUssMp8DnAlEGs+ZgMLGluchLy4GPL7I4ThYi7GTgPeNNffBK6ps/0tbfgRaKeUCr+IzxFCiFbpmz0nWLw5gz3HCqiy2S2rw9Gg18A3SqktSqnZ5rYwrfVxc/0EEGauRwJH6/xsprntDEqp2UqpZKVUck5OTiNKb35KKW699dba59XV1YSGhnLVVVc16+fecccdxMTEkJCQQEJCAiNGjDjv/seOHWPGjBkX/blz587lhRdecHj/N954g9DQUBISEujVqxfz5s276Boaq6G1C9HccosruPe9rcz5eBdT/rWOvk98zTX/Xs/jn+zmg+SjpJwopLqFwt/Ri7GjtNZZSqkOwEqlVErdF7XWWinVoFnGtdYLgYUAiYmJTjlDub+/P7t376asrAxfX19WrlxJZOQvvrOaxfPPP+9weEdERPDRRx81c0X1u/HGG3n55ZfJzc2lZ8+ezJgxg06dOjXrZ1ZXV+Ph4bT9CIQAYNm2LKpsmtfvGExRRTW7MvPZlVXAsm1ZvP2jcUOrj6cbT07ty42DOzdrLQ6d0Wuts8xlNrAMGAKcrGmSMZfZ5u5ZQN1/6VHmtlZp8uTJfP755wC8//77zJw5s/a1kpIS7rzzToYMGcKAAQP49NNPAUhPTycpKYmBAwcycOBANmzYAMDq1asZO3YsM2bMoFevXtxyyy1o7fh33Ny5c7ntttsYPnw4sbGxvPrqq7WfFxcXB8CePXsYMmQICQkJxMfHc/DgQQBefPFF4uLiiIuL45///Gftez7zzDP06NGDUaNGsX///trtaWlpTJo0iUGDBpGUlERKyhnf7b8QEhJC9+7dOX7c+CMvJyeH6667jsGDBzN48GDWr18PQL9+/cjPz0drTUhICG+99RYAt99+OytXrjzvsUtKSmLq1Kn06dPnvLUL1/KPr1K48T8befm7g+zMzMdud8rzwjNorVny01ESOrVjXK8OTO0fwZ+n9GHx7OHsfGIiq/40hn/emMDNQ7oQG9am2eu54GmRUsofcNNaF5nrE4G/AsuBWcCz5vJT80eWA/cppRYDQ4GCOk08jfLkZ3vYe6zwYt7iF/pEtOWJq/tecL+bbrqJv/71r1x11VXs3LmTO++8k7Vr1wJG0IwfP57XXnuN/Px8hgwZwmWXXUaHDh1YuXIlPj4+HDx4kJkzZ9aO5bNt2zb27NlDREQEI0eOZP369YwaNeoXn/vQQw/x9NNPA9C3b1/effddAHbu3MmPP/5ISUkJAwYMYMqUKWf83CuvvMIDDzzALbfcQmVlJTabjS1btvD666+zadMmtNYMHTqUMWPGYLfbWbx4Mdu3b6e6upqBAwcyaNAgAGbPns0rr7xCbGwsmzZt4p577uG7774753HKyMigvLyc+Ph4AB544AH+8Ic/MGrUKDIyMrjiiivYt29f7X9zly5d6Nq1K2vXruX2229n48aNLFiwAKXUOY/d1q1b2b17NzExMWzZsuWctQvX8cOBHP5vdRrhgT688M0BXvjmACH+XoyKbc+YHqEkxYYS2sa7yT6vstpOQVkVheVVFJZVUVheTbCfF/2iAhv0Plsz8jmYXcyz1/b7xWtubopuoQF0Cw3gmgEt00LgyN+/YcAysweDB/Ce1vorpdRPwAdKqbuAI8AN5v5fAJOBVKAU+FWTV92C4uPjSU9P5/3332fy5MlnvPbNN9+wfPny2rbh8vJyMjIyiIiI4L777mP79u24u7tz4MCB2p8ZMmQIUVFRACQkJJCenl5v0J+r6WbatGn4+vri6+vLuHHj2Lx5MwkJCbWvDx8+nGeeeYbMzEyuvfZaYmNjWbduHdOnT8ff3x+Aa6+9lrVr12K325k+fTp+fn4ATJ06FYDi4mI2bNjA9ddfX/u+FRUV9R6fJUuWsGbNGlJSUnj55Zfx8fEB4Ntvv2Xv3r21+xUWFlJcXExSUhJr1qyhS5cu3H333SxcuJCsrCyCgoLw9/enoKDgvMeupk/82rVr661duI6i8ioeXbqT7h0CWPG7URSVV7MuNYc1B06x5kAOn24/BkCf8Lb8eUpvRnZv3+DPOHiyiAc/3MHxgnIKy6sor/plm7m7m+L7P42lc4ifw+/7wU9H8fNy56r+EQ2uqTlcMOi11oeA/vVszwUm1LNdA/c2SXUmR868m9PUqVN58MEHWb16Nbm5ubXbtdYsXbqUnj17nrH/3LlzCQsLY8eOHdjt9trwA/D2/vnsw93dnerq6gbVcnaXwbOf33zzzQwdOpTPP/+cyZMn85///KdB7w9gt9tp164d27dvv+C+NW30ycnJTJw4kalTp9KxY0fsdjs//vjjGf/tAKNHj+bf//43GRkZPPPMMyxbtoyPPvqIpKQkAObNm3fOY1fzRSUuDX//MoUTheV8dPcIfDzd8fF0Z/qAKKYPiMJu1+w9XsgPB3JY/FMGf/pgB6sfGouPp3uDPuPJz/Zy+FQJV8aFE+jnSVsfD9r6ehLo60lbH0/c3RS/fiuZ/1udyrPXxTv0nsUV1Xy28xhXxYcT4O0c15JkrBsH3HnnnTzxxBP063fmn2FXXHEF8+fPr21n37ZtGwAFBQWEh4fj5ubG22+/jc1ma7JaPv30U8rLy8nNzWX16tUMHjz4jNcPHTpE165duf/++5k2bRo7d+4kKSmJTz75hNLSUkpKSli2bBlJSUmMHj2aTz75hLKyMoqKivjss88AaNu2LTExMXz44YeA8YW2Y8eO89aVmJjIbbfdxksvvQTAxIkTmT9/fu3rNV8anTp14tSpUxw8eJCuXbsyatQoXnjhBUaPHg04fuzOVbtwDRtST/HepgzuGhXDwM5Bv3jdzU0RFxnIveO68/yM/pwoLOe19Ycb9BlrDuSwLvUU90+I5bkZ8Tw2uTf3jY/l9uHRTEuIZFyvDozuEcrMwZ34aEsmmXmlDr3v5zuPUVpp48bBzdspoSEk6B0QFRXF/fff/4vtjz/+OFVVVcTHx9O3b18ef/xxAO655x7efPNN+vfvT0pKSqPORB966KHa7pUJCQlUVlYCRlPSuHHjGDZsGI8//jgREWf+afjBBx8QFxdHQkICu3fv5vbbb2fgwIHccccdDBkyhKFDh/LrX/+aAQMGMHDgQG688Ub69+/PlVdeecaXxrvvvsuiRYvo378/ffv2rb3QfD6PPPIIr7/+OkVFRfzrX/8iOTmZ+Ph4+vTpwyuvvFK739ChQ+nRowcASUlJZGVl1TZfOXrszle7aN1KKqp55OOdxLT3508Te15w/2FdQ7isdwcWfJ9GbnH9TYxns9s1z36ZQlSQL7cN73LefX87phtKwSs/pDn03ot/Okr3DgH1fkFZRmtt+WPQoEH6bHv37v3FtkvdE088oZ9//nmry2hV5Peo9Xni0906es4KvelQrsM/c/BkoY6Zs0I/8eluh/ZftjVTd3lkhV62NdOh/R/9eKeOfewLfSy/9Lz77T9RqLs8skK/uibNofe9WECydiBj5YxeCOE0Nh8+zRsb0pk1PJohMcEO/1z3Dm24cXBn3vnxCOmnSs67b0W1jRe+2U/fiLZMdfBi6d1jumHXmv/8cOi8+y356Sie7orpLdSbxlES9K3I3LlzefDBB60uQ4hmUVZp4+GPdtAp2JeHrrhwk83Z/nBZLF4ebjz/9fnvqXh74xEy88qYc2Uv3NwcGw+pU7Af1w6M5P3NGWQX1j9QXkW1jWXbsri8TxghAU3X5bMpSNALIZzCiyv3k55bynPXxuPfiN4qHdr68Jukrny+6zjbMvLq3aegrIqXv08lKbY9SbEXnFP7DPeO6061XbNwTf1n9d/uzeZ0SSU3JDrPRdgaEvRCCMttzchj0brD3Dy0MyMa0R++xm9Gd6V9gDd//yKl3rvOX/khjfzSKh6Z1KvB790lxJ9pCRG8s+kIp+q56Lsk+SgRgT4N/gJpCRL0QghLlVfZeOjDHXRs68OjVzY8gOsK8Pbg95fFsjn9NCv3njzjteMFZby27jDXJEQQF9mwO11r3DuuO5XVdl5de+ZZfWZeKWsP5jAjsRPuDjYHtSQJeiGEpf616iBpOSX87dp+tPG5+Gkfbxrcia6h/jz7VcoZo0POW3kArXGoy+a5dAsN4Kr4CN7eeITTJZW12z/akgnA9YOiGl94M5KgPw93d/cz+rI/++yzTfbedQciOx8Zfle4st1ZBfxnzSGuGxjF2J4dmuQ9PdzdmDOpF4dySlj8kzFi+oGTRXy0JZPbhnehU7DjQxnU577x3SmrsvHaOuMGLZtd82FyJqO6t7/o924uznF/rpPy9fV1aBgAIUTDVdnsPPzRToL8vHj8qt5N+t6X9wljcHQQ//z2ANcMiOS5L1Pw9/LgvnHdL/q9e4S1YXJcOG9sSOc3SV3ZkZlPVn4Zj06+uGan5iRn9I0QHR3Nww8/TL9+/RgyZAipqamAcZY+fvx44uPjmTBhAhkZGQCcPHmS6dOn079/f/r371879K7NZuM3v/kNffv2ZeLEiZSVlTlcQ33DDpeUlDBlyhT69+9PXFwcS5YsAWDOnDn06dOH+Ph46Z4pnMbCNYfYe7yQp6b1pZ2fV5O+t1KKxyb35lRxJb97byurUrL5n7HdCPJvms+5b3x3iiuqeX3DYZb8dJQgP08u7xN24R+0SOs4o/9yDpzY1bTv2bEfXHn+ppiysrIzRoZ89NFHufHGGwEIDAxk165dvPXWW/z+979nxYoV/O53v2PWrFnMmjWL1157jfvvv59PPvmE+++/nzFjxrBs2TJsNhvFxcXk5eVx8OBB3n//fV599VVuuOEGli5desaMVudyrmGHDx06RERERO34+QUFBeTm5rJs2TJSUlJQSpGfn38RB02IppGaXcxLqw5yZVxHruzXPDONDugcxJR+4Xy+6zgd2/pw58iYJnvv3uFtmdgnjEXrDlNeZeO2YdF4ezRsQLWWJGf051HTdFPzqAl5oHYCkpkzZ7Jx40YANm7cyM033wzAbbfdxrp16wD47rvvuPvuuwGj3T8w0LjiXzNdIMCgQYNIT093qK66ww4HBATUDjvcr18/Vq5cySOPPMLatWsJDAwkMDAQHx8f7rrrLj7++OPaYX2FsIrdrpmzdCc+Hm48Oa15R6Z96IqedGzrw2NTeuPr1bRBfP+EWIrKq6myaacawKw+reOM/gJn3laoOzzw2UMFO+rsIYsb0nRTnx49erB161a++OIL/vKXvzBhwgT+93//l82bN7Nq1So++ugjXn755fNOICJEQ9ntmg+3HGVEN8cuRr6z6QjJR/J4fkY8Hdr4XHD/ixHd3p8Nc8Y7fAdsQ8RFBjKlXzj5ZZX07Nj8s0RdDDmjb6Sa9u8lS5YwfPhwAEaMGMHixYsBY/THmjHWJ0yYwIIFCwCjXb6goOCiPvtcww4fO3YMPz8/br31Vh566CG2bt1KcXExBQUFTJ48mXnz5l1wuGEhGmrNwRweWbqLyS+t5ZNt5581NDOvlOe+TCEptj0zWqgrYnOEfI35Mwfwzl1Dm+39m0rrOKO3yNlt9JMmTartYpmXl0d8fDze3t68//77AMyfP59f/epXPP/884SGhvL6668D8NJLLzF79mwWLVqEu7s7CxYsIDzc8XbJp59++ox5XjMzM2uHHQZqhx3++uuveeihh3Bzc8PT05MFCxZQVFTEtGnTKC8vR2vNiy++eNHHRYi6PtmWRVsfD3p2bMPvl2znu5RsnromjkDfM/vEa615bNluNPC36f0a/ZewM2nOL5GmpOq7TbilJSYm6pp5QWvs27eP3r2btstVU4mOjiY5OZn27Rt/q7ZoGc78e+QKSiqqSXz6W6YPjOSpaXEsWJ3KvG8P0rGtD//vhv4M6xpSu+/SLZn86cMdPDm1L7NGRFtXtAtRSm3RWideaD9puhFCNNo3e09QVmVj+oBI3N0U942PZendI/B0V8x89Uee+yqFymo7OUUV/HXFXhK7BHHbsPNP9CGanjTdNIKjvWOEcHXLth0jKsiXQXVmU0ro1I7P70/iqRV7WbA6jbUHcwj296asysaz18W3muYOV+LUZ/TO0KwkWi/5/Wle2UXlrDuYwzUJkb8Ib39vD569Lp5Xbh1EVl4Zaw7k8MCEWLp3CLCo2kub057R+/j4kJubS0hIiEtctBEtS2tNbm4uPj7N233vUvbZjuPYNVwz4NyzNE2K68iAzu34PiWb65x0wK9LgdMGfVRUFJmZmeTk5FhdimilfHx8iIqScGkun2zLol9kIN07nL8PeVhbH24a0rmFqhL1cdqg9/T0JCam6W5ZFkI0ndTsInZlFfD4VX2sLkU4wKnb6IUQzumTbcdwU3B1/+YZp0Y0LQl6IUSD2O2aT7ZnMSo2tNmHMBBNQ4JeCNEgWzLyyMwrY/p5LsIK5yJBL4RokGXbsvD1dGdin45WlyIcJEEvhHBYRbWNz3ce54q+Yfh7O21fDnEWCXohhMNW78+hoKyKawZEWl2KaAAJeiGEwz7ZlkX7AC9GdZcB/VoTh4NeKeWulNqmlFphPo9RSm1SSqUqpZYopbzM7d7m81Tz9ejmKV0I0ZIKyqpYtS+bq/tH4OEu54itSUP+bz0A7Kvz/Dlgnta6O5AH3GVuvwvIM7fPM/cTQrRyX+46TqXNznRptml1HAp6pVQUMAX4r/lcAeOBj8xd3gSuMdenmc8xX5+gZLAaIVq9Zduy6BrqT7/IQKtLEQ3k6Bn9P4GHAbv5PATI11pXm88zgZqv+UjgKID5eoG5/xmUUrOVUslKqWQZz0aI5mW3a2z2xo/mmZVfxqbDp5meECmDDLZCFwx6pdRVQLbWektTfrDWeqHWOlFrnRgaGtqUby2EOMsra9IY9PRK1h081aif/3S7MRfstARptmmNHDmjHwlMVUqlA4sxmmxeAtoppWo60kYBNbMCZwGdAMzXA4HcJqxZCNFAX+w6Tn5pFbNe38x7mzIa9LNHT5fy/uYMErsE0TnEr5kqFM3pgkGvtX5Uax2ltY4GbgK+01rfAnwPzDB3mwV8aq4vN59jvv6dlhkghLBMfmkle44VMnt0V0Z1b89jy3bx1Iq9F2zK0Vrz3qYMJv1zDXklVfz+sh4tVLFoahdza9sjwGKl1NPANmCRuX0R8LZSKhU4jfHlIISwyMa0XLSGK/qG8fAVPXn6830sWneYI7klvHTTgHrvcD1RUM7DS3ey5kAOI7uH8Nx18UQFydl8a9WgoNdarwZWm+uHgCH17FMOXN8EtQkhmsD6tFP4e7kTH9UOD3c35k7tS9dQf578bC8zXtnIolmJRLTzBYyz+GXbspi7fA9VNs1T0/pyy9AuMs9rKyeDVQjh4jak5jK0awiedW5yun14NF1C/Lnv3a1M+/d6/nu7EfZ/XraLb/aeJLFLEC9c35/o9v4WVi6aigS9EC7sWH4Zh06VcPPQX07lN6ZHKEvvGcGdb/zEDf/ZiJ+XOyWVNv48uTd3jorBXc7iXYYEvRAubH2q0Z1y5DnGpukR1oZP7x3Jfe9to6LaxnPXxRMbdv45YEXrI0EvhAvbkJZLiL8XPc8T3iEB3rw/e1gLViVamoxMJISL0lqzPvUUw7uFyMXUS5wEvRAuKi2nmOyiinM224hLhwS9EC5qfapxQ/rIbhL0lzoJeiFc1PrUU0QF+cqwBUKCXghXZLNrfjyUK2fzApCgF8Il7c4qoLC8mhHdfzFCuLgESdAL4YLWpxn950fIGb1Agl4Il7QhNZeeYW0IbeNtdSnCCUjQC+Fiyqts/JR+WpptRC0JeiFczNaMPCqq7XIhVtSSoBfCxWxIzcXdTTG0a7DVpQgnIUEvhItZn3aK+KhA2vh4Wl2KcBIS9EK4kMLyKnZmFkizjTiDBL0QLmTTodPY7FouxIozSNAL4ULWp57C28ONgZ2DrC5FOBEJeiFcyIa0UwyODsbH093qUoQTkaAXwkVkF5Vz4GSxNNuIX5CgF8JFbEyTYYlF/STohXAR61NP0dbHg7jIQKtLEU5Ggl4IF2BMG5jL8G4huMu0geIsEvRCuICM06Vk5ZfJtIGiXhL0QrRy6adKuOfdrbgpGB0banU5wgl5WF2AEKLxPttxjEc/3oWHu+LV2xOJbu9vdUnCCUnQC9EKlVfZePrzvbzzYwYDO7dj/s0DiWzna3VZwklJ0AvRyhw+VcK9725l7/FCfjumKw9O7Imnu7TCinOToBeiFVm+4xiPLt2Jp4cbr92RyPheYVaXJFoBCXohWoFqm50nlu/h3U0ZDOoSxPyZA4iQphrhoAv+vaeU8lFKbVZK7VBK7VFKPWluj1FKbVJKpSqlliilvMzt3ubzVPP16Ob9TxDC9f39yxTe3ZTBb0d3ZfHsYRLyokEcadirAMZrrfsDCcAkpdQw4Dlgnta6O5AH3GXufxeQZ26fZ+4nhGikpVsyWbTuMHeMiObRyb2lPV402AV/Y7Sh2HzqaT40MB74yNz+JnCNuT7NfI75+gSllNyqJ0Qj7Diaz6PLdjG8awh/ntLb6nJEK+XQqYFSyl0ptR3IBlYCaUC+1rra3CUTiDTXI4GjAObrBYAMpydEA2UXlfPbt7fQoY03/75loJzJi0Zz6DdHa23TWicAUcAQoNfFfrBSarZSKlkplZyTk3OxbyeES6mstnPPO1spKKti4W2JBPt7WV2SaMUadIqgtc4HvgeGA+2UUjW9dqKALHM9C+gEYL4eCOTW814LtdaJWuvE0FC5bVuIup5YvofkI3k8f308fSLaWl2OaOUc6XUTqpRqZ677ApcD+zACf4a52yzgU3N9ufkc8/XvtNa6KYsWwpW98+MR3t+cwT1ju3FVfITV5QgX4Eg/+nDgTaWUO8YXwwda6xVKqb3AYqXU08A2YJG5/yLgbaVUKnAauKkZ6hbCJW06lMvc5XsY1zOUP03saXU5wkVcMOi11juBAfVsP4TRXn/29nLg+iapTohLSFZ+Gfe8u5XOwX68NHOAjCsvmoxcxhfCCWituffdrVRW21l4eyJtfTytLkm4EAl6IZxAWk4J24/m86eJPejeIcDqcoSLkaAXwgn8cMDoYjyhtwxSJpqeBL0QTmD1/my6hfrTKdjP6lKEC5KgF8JiZZU2Nh0+zZgeHawuRbgoCXohLPbj4Vwqq+2M7Sk3DormIUEvhMV+2J+Dj6cbQ2KCrS5FuCgJeiEs9sOBHIZ1DcHH093qUoSLkqAXwkJHcks4fKqEsT2k2UY0Hwl6ISy0xuxWOaanXIgVzUeCXggLrd6fQ+dgP6JDpFulaD4S9EJYpKLaxoa0XMb2DEUmYRPNSYJeCIskp+dRVmVjjLTPi2YmQS+ERVbvz8bL3Y1hXWWmTdG8JOiFsMgPB3IYHBOEv7cj00II0XgS9EJY4Fh+GQdOFjNWhj0QLUCCXggL/FDbrVLa50Xzk6AXwgI/7M8hPNCHWBl7XrQACXohWliVzc761FPSrVK0GAl6IVrY1iN5FFVUS7dK0WIk6IVoYT8cyMHDTTGie3urSxGXCAl6IVrYDwdyGNglSCYAFy1Ggl6IFpRdVM6eY4XSbCNalAS9EC1ozYFTABL0okVJ0AvRgn44kEP7AG/6hLe1uhRxCZGgF6KF2OyatQdzGNMjFDc36VYpWo4EvRAtZEdmPvmlVXI3rGhxEvRCtJAf9ufgpiBJulWKFiZBL0QLKKmoZtm2LBI6tSPI38vqcsQlRoJeiBbw9Of7OJpXysOTelldirgESdAL0cxW7TvJ+5szmJ3UVSYZEZa4YNArpToppb5XSu1VSu1RSj1gbg9WSq1USh00l0HmdqWU+pdSKlUptVMpNbC5/yOEcFaniit4ZOlOenVswx8n9rC6HHGJcuSMvhr4k9a6DzAMuFcp1QeYA6zSWscCq8znAFcCseZjNrCgyasWohXQWjNn6S4Ky6r5500JeHu4W12SuERdMOi11se11lvN9SJgHxAJTAPeNHd7E7jGXJ8GvKUNPwLtlFLhTV65EE5uyU9H+XbfSR6e1JNeHeUGKWGdBrXRK6WigQHAJiBMa33cfOkEEGauRwJH6/xYprlNiEvGkdwS/rpiLyO6hXDnyBiryxGXOIeDXikVACwFfq+1Lqz7mtZaA7ohH6yUmq2USlZKJefk5DTkR4VwatU2O39Ysh13N8UL1/eXu2CF5RwKeqWUJ0bIv6u1/tjcfLKmScZcZpvbs4BOdX48ytx2Bq31Qq11otY6MTRU7hQUrmPB6jS2ZuTz9DVxRLTztbocIRzqdaOARcA+rfWLdV5aDswy12cBn9bZfrvZ+2YYUFCniUcIl7YzM5+XVh3k6v4RTEuQFkvhHDwc2GckcBuwSym13dz2GPAs8IFS6i7gCHCD+doXwGQgFSgFftWkFQvhpMoqbfx+yXZC23jz9LQ4q8sRotYFg15rvQ44VyPjhHr218C9F1mXEK1Klc3Ogx/S/LAHAAAXi0lEQVTu4FBOCe/+eiiBfjJ7lHAejpzRCyHOo6Laxu/e28Y3e0/y2ORejJRBy4STkaAX4iKUV9n47dtb+OFADnOv7sMd0pVSOCEJeiEaqaSiml+/mcyPh3P5+7X9mDmks9UlCVEvCXohGqGwvIo7X/+JrRl5vHhDf6YPiLK6JCHOSYJeiAbKL61k1mub2XOskPkzBzIlXkb4EM5Ngl6IBjhVXMGt/93EoZwSXrl1EJf1CbvwDwlhMQl6IRx0qriCmxb+SGZeKf+dlcjoHnJHt2gdJOiFcNC/Vh3kSG4Jb905lOHdZAIR0XrIDFNCOCC7qJzFPx3l2gFREvKi1ZGgF8IBi9Ydptpm53/GdrO6FCEaTIJeiAvIL63knY1HmBIfQUx7f6vLEaLBJOiFuIA3NqRTUmnjHjmbF62UBL0Q51FcUc3r69O5rHcHeofLdICidZKgF+I83tt0hIKyKu4d193qUoRoNAl6Ic6hvMrGq2sPM7J7CAM6B1ldjhCNJkEvxDl8uCWTnKIK7h0rZ/OidZOgF6IeVTY7r6xOY0DndtJvXrR6EvRC1GP59mNk5Zdx79juGNMmC9F6SdALcRa7XfN/q1Pp1bENE3p3sLocIS6ajHUjxFm+3nOCtJwS5s8cIGfzovHsdjixA0pOAcqceVuBUmcu28dC24hmLUWCXog6tNa8/H0qMe39mdxPxpm3TN4RSF/386Po2Pn3jxgAA26FvteCj4P3OxSdhD3L4ND3UFkCtkrjUV0Jtoqf1z19octI6DoGopOg7Xl+L6rK4PAa2P8F7P8Kik9cuI4pL8LguxyruZEk6IWo44cDOew5Vsg/rovH3U3O5ltMfsbPoX54LRRkGNt9gyF6FITMMM+A62GrgoPfwGcPwJdzoM80I/S7jAS3s1qny/Jg73LY/ZHxWdoOIbHgFwIe3uDd1li6e4K7N3h4QelpSFkB298x3qN9D4gZAzGjjdq0HQ58bYR72ndQVQpebaD7BOg5GYK7Ahq0rn8Z3Px3XCutdbN/yIUkJibq5ORkq8sQgutf2UBWXhmrHxqHl4dcwmpyWkPhMTi+HY7vgGPbjfXik8brvsEQPdI4c45OgtBevwzrc71v1lbY9jbsXgoVhRAUDQm3QN/pxmft+ghSvwV7lRG+cTMg7jro0OvC72+3wYldxtn64TVwZANUlVDbBKPt0DYSel5pPKKTjC+MZqaU2qK1TrzgfhL0QsChnGKe/nwf36Vk8+TUvswaEW11Sc5Ja+MM9/Qh45F32FjmZ4ByM5o5PH3B0+/Mpd0GJ3cbgVuSY7yXcoP2PSG8P0QONM6OQ3s7FuznU1lqnIFve9sI5RptIiDuWiPcIwac+y8ER9iqIGuL8f5aQ48rjP+OFr6mI0EvhAMKy6uYv+ogr69Px9fTnd9N6M5do7pees02paeNsC7Lg/J8Y1mWf+bz/KNw+jBUFNT5QQWBUdCus7FeVWq0U1eXGcuqMqP9G4yz84gECE8wQrFjHHg182igeelGW3nHOOg84uK/RJyMo0EvbfTikmSzaz5MPsrzX+/ndGklNyZ24k8TexLapvn/3Lac1sZZ+NFNkLERMjbBqf317+vuDb5B4NvOaJqIGmw0e9Q82nUGT58Lf6bdbk3IBkXDsP9p+c91MhL04pLzU/ppnvxsD7uzChkcHcSbVw8hLjLQ6rKaV0EW7PkYMn40Ar6m+cQnEKKGQPwNxhm3b1CdRzuj2aUpuNiZdGsjQS8uGccLyvjbFyl8tuMY4YE+/GvmAK6OD3ftvvJVZbBhPqybZzSrBEVDtwnQeSh0Gub4xU7RqknQC5dXUW1j0brDzF+Vil1r7p8Qy91juuHr5W51ac1Ha9i3HL75i9H23nsqXDYXQmTylEuRBL1wad/vz+avn+3l8KkSrugbxl+m9KFTsJ/VZTWvk3vgy0cgfS106AOzPjP6fItLlgS9cEkZuaX8dcVevt13kq7t/XnzziGM6RFqdVnNq/Q0fP83SF5ktL1PfgEG/Qrc5Z/5pe6CvwFKqdeAq4BsrXWcuS0YWAJEA+nADVrrPGU0dr4ETAZKgTu01lubp3Qhfqms0saC1am8suYQHm6KOVf24s6RMa5785Ot2ug5k7ICdi6B8gJIvAvGPQZ+wVZXJ5yEI1/1bwAvA2/V2TYHWKW1flYpNcd8/ghwJRBrPoYCC8ylEM0uLaeY37yZzKFTJUxLiODRK3vTMdCBrn+tTXUFHPrBaIPf/wWU5oKHD8ReDmMfhbC+VlconMwFg15rvUYpFX3W5mnAWHP9TWA1RtBPA97Sxl1YPyql2imlwrXWx5uqYCHq8/3+bO5/bxteHm689+uhjOje3uqSmo7WUHAUMn+ClC+McVUqi4xxWXpcAb2vhu6XNf/NR6LVamzjXVid8D4BhJnrkcDROvtlmtsk6EWz0FqzcM0hnv0qhd4d27Lw9kFEBbXii61aG3dznjEWzA4oO2287hcCcdONXjQxo1tkPBXR+l30VRqttVZKNXgcBaXUbGA2QOfOnS+2DHEJKq+y8ejHu1i2LYsp/cJ5/vp4/Lxa6YXH6gr49D44+LXRzg7g5mH0muk1xRw6YIAxdIBcXBUN1NjfmJM1TTJKqXAg29yeBXSqs1+Uue0XtNYLgYVgjHXTyDrEJepEQTm/fTuZHZkFPDixB/eOa8VT/mkNy++HXR8Yoy1GJRrjwYT1lTN20SQaG/TLgVnAs+by0zrb71NKLca4CFsg7fOiqW3LyOO3b2+hpKKahbcNYmLfjlaXdHHWzYOdi2HsYzD2EaurES7Ike6V72NceG2vlMoEnsAI+A+UUncBR4AbzN2/wOhamYrRvfJXzVCzuERV2ey8tfEIz32VQse2Prx911B6dmxjdVkXZ99nsOpJY+jcMQ9bXY1wUY70upl5jpcm1LOvBu692KKEqEtrzcq9J/n7lykcPlXC2J6hzLshgSB/L6tLuzjHd8DHsyFyEEz7d4uPZS4uHXJVRzi13VkFPLViL5sOn6ZbqD+v3ZHIuJ4dWm97fI2iE/D+TGOUyJvea7pRIoWohwS9cErHC8p4/uv9LNuWRZCfF09N68tNQzrj6e4Cd7hWlcHim43JPO78Gtq08msMwulJ0AunUV5lIy2nmK92n+DVtYew2+G3o7txz7hutPXxtLq8pqE1fHqvMQ3dje9CeLzVFYlLgAS9aHHlVTZSs4s5mF3EwZPFHDhZTGp2ERmnS7GbHW2v7h/Bw1f0dL2RJn/4hzF59YQnoPdVVlcjLhES9KLZFJVXkZZTwsGTRaRmF5vhXszRvFJqpir2cFPEtPenb0Qg0xIiiQ0LoG9EIDHtXfB2/h1LYPXfoP9MGPUHq6sRlxAJenFRtNbkFFeQll1Cak4xadnFpOUYoX68oLx2Py93N7qG+hMfFci1AyPpEdaGHmEBdAnxd4129/MpOQVfzYFdH0Ln4XD1S9LDRrQoCXrxCzuO5rMhLRe71mitsWuwm0vjueZkYQVpZrAXllfX/qyflzvdQgMY3jWE7mEBdA8NIDasDZ2CfPFw9UA/m9ZGuH/5CFQUwZg5kPRHudtVtDgJelHrWH4Z//gqhU+2HzvnPm4KlFIE+XnRvYM/UxMi6BYaQPcOAXQLDSA80Kf1d31sCvkZsOKPkLoSogbD1PnQobfVVYlLlAS9oLSymv/8cIj/rEnDruG+cd35dVIMvl7uuCllPpAAd4TdBj/9F7590ng+6TkY8htwc+H5aYXTk6C/hNntmuU7jvHslymcKCxnSnw4cyb1cr2eLi3Bbjdmevp2LmRuNsaHv2oetJORWYX1JOhdUHmVjd1ZxlC33h7u+Hi64e3hjrenGz7mcs+xQp5asZftR/PpFxnI/JsHMDhapp5rEK3h2Daju+Tuj6HoGPgGw/SFEH+DXHAVTkOC3kUcyy/ju5Rsvk/JZn3aKcqr7Bf8mQ5tvHnh+v5cOyASNzcJJYdl7zPDfSmcPgRunsY0fnFPQY9J4B1gdYVCnEGCvpWqttnZdjS/NtxTThQB0CnYlxsTOzEqNhRvDzcqqu2UV9moqLZTUW2jvMpY+nt5MGNQFP7e8ivgELsNdn4AG+ZD9h5QbsYMT6P+aNz45BtkdYVCnJP8K3di1TY7xwvKOZJbypHTJWTklprrpRzJLaG00oa7m2JwdBCPTe7F+F4d6BYaIBdNm5LWsP9LWPVXyNkHHfvBlc9D32sgoIPV1QnhEAl6J5JTVMHmw6fZdDiXzYdPk5pdTLX958m3vNzdiAr2JTrEn6ExwSRGB5EUG0qgr4uMA+Ns0tf/fHE1uBtc/wb0ngZul9j9AKLVk6C30PGCMjYdMoJ90+HTHMopAYybjgZ1CWJcrw50Cfajc4gfXUL86djWB3dpS29+x3caZ/CpK6FNuHEna8It4C5fqKJ1kqBvJkdyS9iQlsvpkkpOl1SSV1LJ6dK6yyqKK4w7Stt4ezA4JpgbEzsxtGsIfSPauv6wAM6m6ASkr4OUFbBnGfgEwmVPwpDZ4CXdTUXrJkHfhApKq1ix6xjLtmaRfCSvdruflztBfl4E+xuPrqEBBPl5ERXky5CYYHqHt5Uz9ZZWkGk0zRxZZyxPpxnbvdsaF1hH3i8XWIXLkKC/SJXVdlbvz2bZtixW7cum0mYntkMAD0/qyZVx4YQH+uDjKXdFWkZrKM42esqc3Asndxs3NuWlG697B0KX4ZD4K+gyEjrGg7v8sxCuRX6jG8Fm1ySnn+bzXcf5bMcx8kqraB/gxS3DOnPtgCjiIttKzxcraA3ZeyFrK5zc83O4l576eZ+AMIhMhCG/heiREBYnwxMIlydB76Bqm53Nh0/zxe7jfLX7JKeKK/D2cOPyPmFcOzCSpNhQaVe3QtEJOLQa0r6HQ99D8Ulju4evMYhYz0lGmHfoA2F9wb+9peUKYQUJ+vOostnZkJbLl7uO883ek5wuqcTX053xvTowKa4j43p1IEBuOGpZFUVwdJMR7GnfG2ftAH4h0HUsdBsPnYZBcIycqQthkpQ6S024f77zGF/vOUlBWRX+Xu5M6B3G5H4dGdOjA75eEiAtouQUHN9hPE7sNLo91lw0dfeGzsPgsrlGuIf1k/7tQpyDBD1Gs8zGQ7l8vvM4X+05QX5pFQHeHlzeJ4zJ/cJJim0vF1QvpKrMGIM9Lx3yjoC2gbsXePgYE214eBvr7l7Go6rEODuveZQXmuuFxsXTE7uMQcJqtOtiTKTdfyZEDjRmapJuj0I45JIM+mqbnYzTpRzMLmb1/hy+2n2cvFLjzP3yPmFMiY+49MK9sgSObDCbRL4zuh/6tjMfwUZXw5qHX7Cxf176z4+i4xdfg1cAeLcxPiN6lBHs4f2NYQekq6MQjebSQV9cUc0hc/5SY9q7EtJyiknPLaHKZgwt4OflzmW9w5gSH86YHqGtK9ztdqNHScFRI5hrH0eNs2L/UAjsBO061Vl2NoJaa6M55JAZ7Bk/gq3SaBLpMsJo7y4vgLI8KDtt9GYpyzMe9mpAQdtICIqGbhOMZXCMsWzX2biLtLoCqsuhutJcVoCtwljWhHrdh7SpC9EsWn3Qa63JKaqoDXNjWUJqdjEnCn+enNrdTdElxI9uoQFM6B1mTn3nT+/wtk0f7rZqIyQri43Ac/eqs/T6OdDsNqPXSGGWEdCFWVCQBYWZxrKy2AhkNGi7sa7txnO7HUpyjOCsy9PfCHT/UMhNNc7Qq0rO2sfPqKM833geFmfcAdptvBHynr7n/m/T2mhiqWmOEUI4vVYd9G9tTOf5r/dTVGdy6gBvD7qF+jOiewjdQgNq5zPtHOyHl0cTXKwrLzD6aJ/YbYxmWJprnunmG4/yfKOd+XyUuxG0tkqjLbsurwDjTDkw0jgzVm7GBBbKDVBnPvdvb5ypB0b9/PBpd+aEF1ob9eVnGGf6+UeNZWWxcYNQ17HQpqPj//1KgU9bx/cXQliuVQd9THt/pg+IPGNy6rC23k1zs5Kt2gjE7L3GhcGaR/6Rn/fxCTRuwPENgrYRRl/tum3ZXn5GM4etygh1W6XRjFGz7u5lBHrbKHMZabxnU95spZTRVOMXDBEJTfe+QohWo1UHfVJsKEmxoY37Ya2NM++CTONi4unDkHf452V+htkWDaAgpBtEDICBtxsXBzv2M0Y2lDtghRBOrlUH/Rmqyo0LkyU5Rv/r4mxjvfQUlOb9fFGx9LSxrL2oWId3IARHG+Od9JkGQTHG3ZUd+sj0cEKIVqt1B/3Wt2DdPCPYz9Uu7u5tNFv4ms0XoT3PfN4m3OwtEmM8F0IIF9MsQa+UmgS8BLgD/9VaP9scn4Nfe4gYaPQw8W9vLkPPfO7lL80rQohLWpMHvVLKHfg3cDmQCfyklFqutd7b1J9Fr8nGQwghxDk1x+AgQ4BUrfUhrXUlsBiY1gyfI4QQwgHNEfSRwNE6zzPNbWdQSs1WSiUrpZJzcnKaoQwhhBDQPEHvEK31Qq11otY6MTS0kV0khRBCXFBzBH0W0KnO8yhzmxBCCAs0R9D/BMQqpWKUUl7ATcDyZvgcIYQQDmjyXjda62ql1H3A1xjdK1/TWu9p6s8RQgjhmGbpR6+1/gL4ojneWwghRMPI3GtCCOHilNba6hpQSuUARy64Y/3aA6easJymJLU1jtTWOFJb47Tm2rporS/YbdEpgv5iKKWStdaJVtdRH6mtcaS2xpHaGudSqE2aboQQwsVJ0AshhItzhaBfaHUB5yG1NY7U1jhSW+O4fG2tvo1eCCHE+bnCGb0QQojzaNVBr5SapJTar5RKVUrNsbqeupRS6UqpXUqp7UqpZItreU0pla2U2l1nW7BSaqVS6qC5DHKi2uYqpbLMY7ddKWXJpANKqU5Kqe+VUnuVUnuUUg+Y2y0/duepzfJjp5TyUUptVkrtMGt70tweo5TaZP57XWIOkeIstb2hlDpc57gltHRtdWp0V0ptU0qtMJ9f/HHTWrfKB8bwCmlAV8AL2AH0sbquOvWlA+2trsOsZTQwENhdZ9s/gDnm+hzgOSeqbS7woBMct3BgoLneBjgA9HGGY3ee2iw/doACAsx1T2ATMAz4ALjJ3P4KcLcT1fYGMMPq3zmzrj8C7wErzOcXfdxa8xm9THDiIK31GuD0WZunAW+a628C17RoUaZz1OYUtNbHtdZbzfUiYB/G3AqWH7vz1GY5bSg2n3qaDw2MBz4yt1t13M5Vm1NQSkUBU4D/ms8VTXDcWnPQOzTBiYU08I1SaotSarbVxdQjTGt93Fw/AYRZWUw97lNK7TSbdixpVqpLKRUNDMA4A3SqY3dWbeAEx85sftgOZAMrMf76ztdaV5u7WPbv9ezatNY1x+0Z87jNU0p5W1Eb8E/gYcBuPg+hCY5baw56ZzdKaz0QuBK4Vyk12uqCzkUbfxM6zVkNsADoBiQAx4H/Z2UxSqkAYCnwe611Yd3XrD529dTmFMdOa23TWidgzEcxBOhlRR31Obs2pVQc8ChGjYOBYOCRlq5LKXUVkK213tLU792ag96pJzjRWmeZy2xgGcYvuzM5qZQKBzCX2RbXU0trfdL8x2gHXsXCY6eU8sQI0ne11h+bm53i2NVXmzMdO7OefOB7YDjQTilVM2Ku5f9e69Q2yWwK01rrCuB1rDluI4GpSql0jKbo8cBLNMFxa81B77QTnCil/JVSbWrWgYnA7vP/VItbDswy12cBn1pYyxlqQtQ0HYuOndk+ugjYp7V+sc5Llh+7c9XmDMdOKRWqlGpnrvsCl2NcQ/gemGHuZtVxq6+2lDpf3AqjDbzFj5vW+lGtdZTWOhojz77TWt9CUxw3q68wX+TV6ckYvQ3SgD9bXU+durpi9ALaAeyxujbgfYw/46sw2vjuwmj7WwUcBL4Fgp2otreBXcBOjFANt6i2URjNMjuB7eZjsjMcu/PUZvmxA+KBbWYNu4H/Nbd3BTYDqcCHgLcT1fadedx2A+9g9syx6gGM5edeNxd93OTOWCGEcHGtuelGCCGEAyTohRDCxUnQCyGEi5OgF0IIFydBL4QQLk6CXgghXJwEvRBCuDgJeiGEcHH/H3aOooa9o5mVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot([x[0] for x in lll], label=\"Mean Episode Reward\")\n",
    "plot([x[1] for x in lll], label=\"Epoch Loss\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}