predict_SP500_Attn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment of Attention based Network in predicting the price movements of SP500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "from pandas_datareader import data as pdr\n",
    "import fix_yahoo_finance as yf\n",
    "from tqdm import tqdm_notebook\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# import quandl\n",
    "import itertools\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "import collections\n",
    "from sklearn.manifold import Isomap,MDS\n",
    "from matplotlib import pyplot\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import sklearn\n",
    "import statsmodels\n",
    "yf.pdr_override()\n",
    "import scipy.stats as scs\n",
    "import statsmodels.api as sm\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import classification,confusion_matrix,matthews_corrcoef,accuracy_score,classification_report\n",
    "from models.ffn_classifier import FFNClassifier\n",
    "import scipy.stats as stats\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Price Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lmap=lambda fun,it:list(map(lambda x:fun(x),it))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  1 of 1 downloaded\n"
     ]
    }
   ],
   "source": [
    "securities = pdr.get_data_yahoo([\"SPY\"], start=\"2005-01-01\", end=\"2018-05-30\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_number=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=securities['Adj Close'].fillna(method='ffill')\n",
    "diff=(data.shift(-1)/data-1).fillna(0)\n",
    "real_diff=data.shift(-1)-data.fillna(0)\n",
    "labels=pd.qcut(diff,q=class_number,labels=range(class_number)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/mnt/miniconda3/envs/py36/lib/python3.6/site-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f0180e3bf60>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEKCAYAAAACS67iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XuUpHV95/H3t+63vs1Mz4W5MDOKIBABGVEimAgqghpIggrJuhPDhkM2JnqSk10Tk91sYs5qNhvj2dwOahTX4CVegKgLIogSwcEZuc8MMPdrT9+7bt3VXVW//aOeGnqGvlR3170/r3PmdNVTTz3P96G7P/z6V8/zfcw5h4iItD5fowsQEZHqUKCLiLQJBbqISJtQoIuItAkFuohIm1Cgi4i0CQW6iEibUKCLiLQJBbqISJsI1HNnq1atcps3b67nLkVEWt6uXbsGnXO9861X10DfvHkzO3furOcuRURanpkdrmQ9TbmIiLQJBbqISJtQoIuItAkFuohIm1Cgi4i0CQW6iEibUKCLiLQJBbqISJtQoEtT+ubPjvGOT/0Q3fNWpHIKdGlKzxwb48VTaZLj+UaXItIyFOjSlIYzkwAMpHMNrkSkdSjQpSmNZEuBPqRAF6mYAl2a0lDaC3RvpC4i81OgS1PSCF1k4RTo0nScc6fn0AfTGqGLVEqBLk0nO1kgly8CMJTRCF2kUgp0aTrD0+bNhzRCF6mYAl2azpefOHL68Z6TSe7ecYS7dxyZ4x0iAgp0aUKZXAGArmiQtPdYROZXUaCbWbeZfd3M9prZHjO70sxWmNmDZvaS97Wn1sXK8pCdLF0durojTDo31eBqRFpHpSP0TwP3O+cuAC4B9gAfBR5yzp0HPOQ9F1myzGRpVL66I8zEVJF8sdjgikRaw7yBbmZdwFuAzwE45yadc6PAjcBd3mp3ATfVqkhZXrK5PD6DVR1h4OUpGBGZWyUj9C3AAPB5M3vSzD5rZnFgjXPupLdOH7CmVkXK8pKZzBMLBUiEA6XnOTXoEqlEJYEeAF4P/KNz7jIgw1nTK67U43TGPqdmdruZ7TSznQMDA0utV5aBTK5AZHKExz7/cZhIkVagi1SkkkA/Bhxzzu3wnn+dUsCfMrN1AN7X/pne7Jy70zm3zTm3rbe3txo1S5vLTubxnXyevpeewX/yWY3QRSo0b6A75/qAo2Z2vrfoWmA3cB+w3Vu2Hbi3JhXKspPJFXBDpfPO/af2aIQuUqFAhev9LvAvZhYCDgAfpPQ/g6+Z2W3AYeB9tSlRlpvMZJ7gwCEAfIP7SGWyjS1IpEVUFOjOuaeAbTO8dG11y5HlrlB0jKeTuLF+zr3gdRze+wz9B3bDpec2ujSRpqcrRaWpjI1PYSNHAfj5G94H/iAj+59qcFUirUGBLk1lODOJb+QImLHxvIsIr97MRL/6uIhUQoEuTWUkO4lv9AhdqzcQjsYId6wgnx1rdFkiLUGBLk1lKD2JpQdZec5mAKKdPbiJFEVd/i8yLwW6NJWR7CSWS9HRXer1Fu/qwQqTpDOZBlcm0vwU6NJU+kdSWD5Hd88KADq7S18HBgYbWZZIS1CgS1Pp6y8Fd0dXaYTe1bMSgKEhBbrIfBTo0lT6vZF4vLMbgBUrS4E+MjzUsJpEWoUCXZrKoBfcZwd6amS4YTWJtAoFujSVsZERAGJeoHd3duPMRyY50siyRFqCAl2aSjo5Crw8Qo+EAxDuYDylQBeZjwJdmkouPYYvGCYUjgDgM4NIBxOp0QZXJtL8FOjSVPLZJIFY5xnL/LFOJjO6WlRkPgp0aRrOOQrjKUKvCPQu8gp0kXkp0KVpTEwVsVyacOLMQA/FuyiMpygUdLNokbko0KVpZCbzXqB3n7E8nOgGV2R0VPPoInNRoEvTyExMwWSaaKLrjOWRjtJVo7rJuMjcFOjSNE4NjWCuePoc9LKo93xoSFeLisxFgS5N41T5sv+OMwM9nkgAMDamD0ZF5qJAl6ZR7uOS8BpzlcW9D0kHhjWHLjIXBbo0jcHB0pRKouvMOfRERynQ+wfVz0VkLgp0aRrlKZXOzjMDPR6L4nwBhkc15SIyFwW6NI2xZBKAeEfHGcsjQR8EY4zotEWROQUqWcnMDgEpoADknXPbzGwF8FVgM3AIeJ9zTh2UZNHSqRTO/MSj0TOWRwJ+XCjK2FiyQZWJtIaFjNDf6py71Dm3zXv+UeAh59x5wEPec5FFS6dTEIwQCvjPWB4N+nHBKMmkAl1kLkuZcrkRuMt7fBdw09LLkeUsm05DKIbfZ2csj4T8EIySSSvQReZSaaA74HtmtsvMbveWrXHOnfQe9wFrql6dLCvj2RS+cOwVyyMBHy4YKwW+iMyqojl04Crn3HEzWw08aGZ7p7/onHNm5mZ6o/c/gNsBNm3atKRipb3lshn8oegrlgf8PiwcY2JQgS4yl4pG6M65497XfuBbwBXAKTNbB+B97Z/lvXc657Y557b19vZWp2ppS5PjGYLR+IyvBSJxpiay5PP5Olcl0jrmDXQzi5tZR/kx8A7gOeA+YLu32nbg3loVKcvDXIEejJSWp1KpepYk0lIqmXJZA3zLzMrr3+2cu9/Mfgp8zcxuAw4D76tdmdLunHMUclnCswR6KBZnktLFRz09PTOuI7LczRvozrkDwCUzLB8Crq1FUbL85HI5XCFPNJ6Y8fVwNEEadOqiyBx0pag0hXJQzxbokXjHGeuJyCsp0KUplOfG44mOGV8vB71a6IrMToEuTaE88k6cdT/RsnJ/F43QRWanQJemUA7qzs6ZR+jlnujDatAlMisFujSFkdFy69yZAz0WjeD8QQZ1kwuRWSnQpSkMjZQCvfusm1uURYM+CEYZUqCLzEqBLk2hPJXS0zXzHHokUOq4OKoPRUVmpUCXpjAylsT5g3QmXtmcCyAS9OOCsdM3wRCRV1KgS1MYG0tCIEos6J/x9UjQD6EoaQW6yKwU6NIUkskkLhghFp4t0H24YJRMRh0XRWajQJemkE6ncKEYsdDM3SgiwdJNLiYU6CKzUqBLU0inSrefi4VmHqGHAqWzXHITWYrFYp2rE2kNCnRpCtl0GheMzhroPjPCsQQ4pxa6IrNQoEtTGM+mSx+KzjLlAhCJqSe6yFwU6NJwzjkmsnOP0OHlxl3q5yIyMwW6NFw2m8UVi1goSjgw+49kR0fpoiON0EVmpkCXhiuPuEPRON6dsWbU7V1Fqha6IjNToEvDlUfcoVluP1fW0911xvoiciYFujTcfHcrKlvZo0AXmYsCXRru5UCfuXVu2aruThzG8KimXERmokCXhisHenyeEXp3PAzBCIPDI/UoS6TlKNCl4cpTKInOmVvnlnVFg7hglJFRnbYoMhMFujRceYTekZh7hN4VDUIwyuiYAl1kJhUHupn5zexJM/u293yLme0ws31m9lUzC9WuTGlnqVQKC0aIR+f+EeqOlUboyZQCXWQmCxmhfxjYM+35J4FPOedeDYwAt1WzMFk+kskkzHOVKLw8Qk/rLBeRGVUU6Ga2AXgX8FnvuQHXAF/3VrkLuKkWBUr7SyaTFIMR4nP0cYGX59CzaQW6yEwqHaH/LfBfgHLf0pXAqHMu7z0/Bqyvcm2yTCSTKYqBCNF5RugdkSAuGGEim6lTZSKtZd5AN7N3A/3OuV2L2YGZ3W5mO81s58DAwGI2IW1uzJtymW+E7vcZ4WiC/OQEU1NTdapOpHVUMkJ/M/BLZnYI+AqlqZZPA91mVv4N3AAcn+nNzrk7nXPbnHPbent7q1CytJtkcgwXjM47QgeIxtVCV2Q28wa6c+6PnHMbnHObgVuAh51zvw78ALjZW207cG/NqpS2lkqlKvpQFNRCV2QuSzkP/b8Cv29m+yjNqX+uOiXJclIsFslmMvP2Qi/r6CgFukboIq8096TlWZxzjwCPeI8PAFdUvyRZTtLpNM45XDBKIhycd/3OrlKDLo3QRV5pQYEuUm1f+tHe0oNglMf2D3JkODvn+j1eT3QFusgr6dJ/aajxbBoAF4wSDsw/5bKiuxtQoIvMRIEuDZXzAp1ghHBw/h/H3pWlQB9Wgy6RV1CgS0OVLxJywSiRCkboKzsTOPOrha7IDBTo0lATp0foUYL+2e8nWtYdC0EwoptciMxAgS4NNZEpBXo4NvcNostO90RXC12RV1CgS0ONZ1JgRjgaq2j97lip4+LYmEboImdToEtDTWTS+EIxosH5z0GHl0foaqEr8koKdGmo8WwKC8cIByr7Uez0Aj2TTte4MpHWo0CXhprIpCEYreiURYCOcABC0Zc/TBWR0xTo0lATmTSEYhVdVATg8xnhaJxcttQyQERepkCXhhrPpikGokQqHKEDRGIJioU8uVyuhpWJtB4FujTURCZFIRCp6KKislgiAejyf5GzKdClYZxzjGdSFAKVz6GDWuiKzEbdFqVhstksrlgsfShawQj97h1HABgnDMDXH3+RjYNBfu2Nm2pap0ir0AhdGqZ8cZALxRY0hx6Ll6ZcdLNokTMp0KVhynPglbbOLYt5t6Ertw0QkRIFujTM6Q81g1EiwcoDvTyHnklrDl1kOgW6NMz0KZdKrxQF6Ojw7lqU0lkuItMp0KVhpo/QF3KWS2c8gvMH1c9F5CwKdGmYl+fQYws6Dz0eCkAgSlZz6CJnUKBLw4yNjYH5IBBe0Ag9Hg7gQtFS610ROU2BLg0zNjZGIBLDzAj5FxLoflwwqtMWRc6iQJeGSSaT+MIxwkFfRXcrKgv5fVgwyuS4Al1kunkD3cwiZvaEmT1tZs+b2f/wlm8xsx1mts/MvmpmodqXK+0kmUziC8UXNH8OYGYEIjGmFOgiZ6hkhJ4DrnHOXQJcCrzTzN4EfBL4lHPu1cAIcFvtypR2lEwmsdDCznApC8Y6yE/oQ1GR6eb9TXIl5d+coPfPAdcAX/eW3wXcVJMKpW2NjY0tqBf6dOFYB25ynEI+X4PKRFpTRUMjM/Ob2VNAP/AgsB8Ydc6Vf5uOAetnee/tZrbTzHYODAxUo2ZpE8lkkmJwYb3Qy6KJ0sVFOtNF5GUV/SY55wrOuUuBDcAVwAWV7sA5d6dzbptzbltvb+8iy5R2UywWGRsboxhc3Ag95gV6Nj1W7dJEWtaChkbOuVHgB8CVQLeZldvvbgCOV7k2aWPJZJJisUghuLBOi2WJzi4AUmMKdJGySs5y6TWzbu9xFHg7sIdSsN/srbYduLdWRUr7GRkZAWDKv7BOi2WdXqCXtyMild3gYh1wl5n5Kf0P4GvOuW+b2W7gK2b2ceBJ4HM1rFPazOjoKAD5YHxRZ7l09fQAkNQIXeS0eQPdOfcMcNkMyw9Qmk8XWbDyyNot4jx0gJ7u0gg9mRytal0irUxXikpDnJ4qWWDr3LKuRLzUcTGpEbpImQJdGmL6CD28gJtblJUadMUZT6snukiZAl0aYmRkhGAoBP7Qos5yCQd8EIozrrsWiZymQJeGGB0dJd7RBWaLmkM3M/yROJNZBbpImQJdGmJ4eJhYR+mDzcXMoQMEoh1MjSvQRcoU6NIQIyMjhOOlmz0vZg4dIBRLUJhQx0WRMgW6NMTIyAihWCnQI4scoYfjnbhclkKhUM3SRFqWAl0aYnR0FH8kgQHBRQZ6qUGXe/lm0yLLnAJd6m5qaqrUyyUUJxry41vA3Yqmi3tz8KcGBqtZnkjLUqBL3Y15l+tnidATW/yNrhKdpY6LR/uGqlKXSKtToEvdlS8qShVDdMeCi95OV2c3AMdOqc++CCjQpQHKgT6cD9IdXXygd3sNuvr6NeUiAgp0aYByp8WcP0ZPfPFTLitWrgA0hy5SpkCXupvex6U7uvhAX9kRxwUi9GnKRQRQoEsDDA8Plx6E4kuaQw8H/Vi0k8EhjdBFQIEuDTAwMEA4lgB/YElnuQAEYl0kR4arVJlIa1OgS90NDg4STnTREQ4QDS3usv+ySEcX2ZRuQycCCnRpgIGBASzaxfqe6JK3FevsIZ9JUiy6KlQm0toU6FJ3AwMD5IMJNlQh0Du7V2D5CY4N6s5FIgp0qSvnHIODg2R9cdZ3Lz3Qe1asBOD5g8eXvC2RVqdAl7pKpVLkcjkmg/GqTLmsWrUKgJcOn1jytkRanQJd6mpgoHTOuIt0sr47tuTtrVndC8DBY31L3pZIq1OgS10NDpbOGXfhjqrMoXf3lKZcjvedWvK2RFrdvIFuZhvN7AdmttvMnjezD3vLV5jZg2b2kve1p/blSqs7Y4RejbNcOroAY0CX/4tUNELPA3/gnLsQeBPwO2Z2IfBR4CHn3HnAQ95zkTmVAz2c6GblEvq4lPkDAYKxDkZH1EJXZN5Ad86ddM79zHucAvYA64Ebgbu81e4CbqpVkdI+BgcHMX+Qc3p7sEXe2OJsia5usslR8oViVbYn0qoWNIduZpuBy4AdwBrn3EnvpT5gzSzvud3MdprZzvLoTJavgYEB/LFONqyIV22bPStWwkSKvuRE1bYp0ooqDnQzSwDfAD7inDvjJo7OOQfMeKmec+5O59w259y23t7eJRUrra+/v59CqIONVZg/L1vd2wsTSY4Oj1dtmyKtqKJAN7MgpTD/F+fcN73Fp8xsnff6OqC/NiVKO+nrHyAfSnDBus6qbXPT+nVYLsnhoXTVtinSiio5y8WAzwF7nHN/M+2l+4Dt3uPtwL3VL0/azcDAAC7cyWvXdlRtm6/evBErFth3VOeiy/JWyQj9zcAHgGvM7Cnv3w3AJ4C3m9lLwNu85yKzmpiYIJtO4SKdnF/FQN+w/hwADhw6WrVtirSiwHwrOOf+HZjtdIRrq1uOtLMTJ0qX5/esXkNHZPE3tjjbOeeUAv3YCV3+L8ubrhSVuikH+tZNG6u63XXr1gEw0K8pF1ne5h2hi1TD3TuO8NNHnwEg0LGCu3ccqdq2Ozo6CIajpIcHyeULhANLu2mGSKvSCF3q5sSJEzjzsdEbUVeLmdHTuwYbH+HkqM5Fl+VLgS51M3CqDxftYn0VLyoqW7N2LZYd5ehIturbFmkVCnSpm+RQPxZbQU8Veric7dwN67HxEV1cJMuaAl3qZnxskEjXKnxV6uEy3dZzN2CTGQ6d0g2jZflSoEtd5KemKGZG6Vw5Y8ufJVvvnbq474huRSfLlwJd6uJEXx/g6F2ztibbL5+6ePSYAl2WL522KHVx4FDpNMXySLqa7t5xhNGBUm+448dPnD4l8tfeuKnq+xJpZhqhS10cPV4aOW89t7oXFZV1rujFfH4mx/qZzKsvuixPCnSpi/6+PsBY0bu6Jtv3+f3EVqzBlx5gJDtZk32INDsFutTcxFSB1OAJQp0rCASrf8piWc/qc7DMIKMKdFmmFOhSc88dH8NSA/Ss2VDT/aw+ZyOWHmAok6vpfkSalQJdam7noWEs3c85GzfXdD9rN2zCinlO9alJlyxPCnSpuceeO4AVJlm3obZnnaxaW/oLYPCk+qLL8qRAl5pyzvHMnhcBWLmuNme4lK1Yux6AsX71RZflSYEuNXVwMENqoHTK4qoaB3qiawW+UITxoZM13Y9Is1KgS03tOjyCL91PKBon3tVT032ZGbEV6ygk+xmfLNR0XyLNSIEuNfX4gSGC2UFWrduA1aAp19lWrl2PpQc4Pqqui7L8KNClZvKFIg/v7SeUHaz5dEvZ+k2b8Y2PcLhvqC77E2kmCnSpmZ8eGmF0LEkuNVzzD0TLNm49D4BDB16qy/5EmokCXWrmgef7CKdLH1Cu3fSquuxz7bmvBuDUkQN12Z9IM1GgS00453hw9ym2+ks3nFi3+by67LejeyXBeCe5gcP0p3R/UVle5g10M/tnM+s3s+emLVthZg+a2Uve19qeviAt5/kTSY6PjpMYP8m6deuId3bXZb9mRu+GrfjGTvDM0bG67FOkWVQyQv8C8M6zln0UeMg5dx7wkPdc5LTv7T6Fz2D0xAEuuuiiuu5709bXYKk+njw0UNf9ijTavIHunPsRMHzW4huBu7zHdwE3VbkuaXHfe76Py9ZGOHbkCBdffHFd971+y3mYK/KTp56v635FGm2xdyxa45wrX47XB9TmRpHScu7ecYR0Ls/evhRXxEunDvYHVtNRxxrWnlv6APbFvXtwztXl/HeRZrDkD0Wdcw5ws71uZreb2U4z2zkwoD+Bl4MjQ1kAgmOlS/7r9YFoWU/vOgKROBOnDnJ0WBcYyfKx2EA/ZWbrALyv/bOt6Jy70zm3zTm3rbe3d5G7k1ZyaCiD32ekTrxET+86YonOuu7ffD7OOe9ifIP7eerYaF33LdJIiw30+4Dt3uPtwL3VKUfaweGhDOu7whx94Vk2v/aShtTwmosuxZcd4qFdLzRk/yKNUMlpi18GHgfON7NjZnYb8Ang7Wb2EvA277kIk/kix0fH6S0MMpFNs/nCSxtSxxbvfyQ/fnwHpVlBkfY374eizrlbZ3np2irXIm3g2EiWogP/4D4ANl/QmBH6mo1biMQTjB3Zw4HBDK/qTTSkDpF60pWiUlWHhjIYkDq8m1XnbCLRvaIhdZjPx2Wvvxzf0H4efVEfxsvyoECXqjo8lGV1PMCxfc+z5cLLGlrLW69+M77MEA88ofPRZXlQoEvV5AtFDg9nWZE5RH4yx9aLX9/Qeq655hoAnnr8R+TyuuGFtD8FulTNnpMpJvNFCkefIhyNsfWixgb6D48V6Fm/leKxZ/ir+1/g7h1HGlqPSK0p0KVqvvPsSXyuQP/enbzm0isJBEONLomL33A1vpHDPL9PYS7tT4EuVVEoOu558jgbp46Ry6Z57RVXN7okAC6+4ioAdu/8sU5flLanQJeqeHz/EH3JCcJ9zxCKxHjVRZc3uiQAVp2ziY41m8jt+wn7BzKNLkekphToUhXffPIYCZvgxLOPcfGbfpFAqPHTLWVXvu09+MaO8cMduxpdikhNKdBlyTK5PPc/18f547sp5Kd4w9tubHRJZ7j0qmvxBcMc/en3OTmmZl3SvhTosmQPPN9HdmKSU08+xObXXsLqDZsbXdIZwtE4F1zxi/iOP8nnH9Y56dK+FOiyJBNTBf7u4X2sHdvNUP8p3vC25rzXyVvedTNWzPOVL32ByXyx0eWI1IQCXZbkbx58kQOnRvHvuZ+LLrqI8y97U6NLmlHvOZvYfNnVTL7wIz7zvacaXY5ITSjQZVHu3nGET/y/vXzmRwfYMvYkwwOnuOxdH8B8zfsjdcP7fwNzRf7xH/6e1MRUo8sRqbrm/e2TpjaUzvG1nUfpKIwx9MR9vPp1b2h475b5rFyznnf/yvso7H+MP/usWvhL+1Ggy4I9vPcUf//IPsZzU6zY8018Ph83bP+9RpdVkT//4z8k2rOG73zhbzl48ux7n4u0NgW6VMQ5x+P7h/itL+7kN7+wk55YiMvTP+HU/ud5x6130LVydaNLrEg0GuUvPv5xyAyz/bc/TLGoD0ilfSjQZV6f//FBrv/0o9z6mZ/w432DvPX8Xt4cOMCuB/6Vy37hei65+h2NLnFB3nXNVVx9820MvLCT3/vYX6olgLSNee9YJMtbf2qCzzx6gL6xCa6/eC1v2rqS3Y8/xL99/m/Z/NpLuf4DH8LMGl1mxcodF6++/ld54pk9PHTP3Xxwssjn//q/tdRxiMxEI3SZ1UAqx6/+42MMpib5wJvO5apXreTxb3+Z+z7712y+8BLe/5H/gT/QmmOCgN/HB377Dyic+0Z2fPcr/PEff4xcLtfoskSWRIEuM8rlC9zxpV0MpHLcdtUW1gYn+PKn/pQffuuLXHzlNdzy4T8nFI40uswlOacnztv+w4eYOv/t3HPPt7jlllvYu3dvo8sSWbTWHF5JTTnn+NN7nmPX4RH+969cwPfv/Vce++7XcMUi1//HD3H5W9/dNtMTV523Gnfzdh74/gb2P/dN3vve93LLLbdwxx13sHLlykaXJ7IgVs8PhLZt2+Z27txZt/3JwkxMFfjusyf54uOHeXr3i2wr7uXokz9kbGyM11x2Jdf92h10965tdJk1MZkv8Gff+CmbTz7C4LM/wu/3c8MNN3Drrbfyute9rtHlyTJnZrucc9vmW08j9GWuWHR859mTfPvpE/zgiacpntxNeGAvkYED7Pb7ueDyN/Mr7/hlNrz6wkaXWlO/8eYtBPw+/uSeOFdcfh3hg//Ov333fu655x5CKzcQPfcS3nPdtXzkfdcSjzRPa2CR6ZY0QjezdwKfBvzAZ51zn5hrfY3Qm0OhUODEiRM8/vQe/uGbj3Di4IsExo7ixpMArNm4lQvf+AtcevV1JLp6Glxtfe08NMy3njyOA6JMkTj1FFOHf0aubz+4IgSjhFZvJrpmC6vPPY/fec+VvPXy1xJqov7v0n4qHaEvOtDNzA+8CLwdOAb8FLjVObd7tvco0KsnXyjy4/1D3PPkcXYcGCKdyzMxlWd1BDbEYUPCsT6ax59LsfvgMQ4e62Nk4BSZoT6K6UEoFk5va+XaDVy57VLoPY9XvW4bnT2rGnhkjTeQymHAykTo9GcF4+kkjz/+Y57+2S4mTh0kP3Icyr87ZkS7ewl2raEQ6WbcH8einWzZsI6Ltq7nkq3ncMX569m0ZiXBYLBxByYtqx6BfiXwZ86567znfwTgnPufs71nsYFeLBYpFos4505fBOKcY2Iyz3BmksHUBIVikXjYTzzspyMUIOi30+s558gXikzmCwymcwylJwn6jETETzzkJxb04ffZGdsuFotkJwuMZHIkx6fwmxHwl6YoclMFCs55pwg5fAYGGM77HS/iCkUm81OcHM3SN5LBzNEV8tMZ9tER8RENGK5YJJ/PM5UvkB7PMZye4ORollNjWSYmckzmcmTGx0lnxpmczBGyIgGXJzsxTjY7gStM4StOEXGTTE5kKE5OzP4fMRglkOgh0rOGaM9aQt1rSKxay1vfeCm9K5bXKLwaJnMTHDu0n8ee3suBgwfxZ4awzABkRymMp2Z9nwXCBKMx4okOOuIxErEonYkYsWiEcDhCIBjC+YPgD5GIR+mMRQgEAhTwMZ539KenGMrmCYeCdMbCdMcidCdK/3riEXoSETA/U0XHZAFy+SK5vGOy4JgqOjpjIVZ3RuiOhggG/Ph8Pnw+H2aGmc36HHjFa6ePyXt89tcEwo/sAAAIJ0lEQVTZli31fUUHuakC4/kiE5MFcoUisVCAnliIWDgw7/uroVB0pCfyJCemSE3kGZ8q0BML0tsRJhEOVH2/9ZhDXw8cnfb8GPDGJWxvVnfccQePPvpoLTbd/PxBfIEgFggx6Qvi/AH8gRAd4TCxWILuRIxILEEkFj/jayASY9wfYzKQ4FUb17G6O9E2Z6Y0g1A4wtbzL2Lr+Re94rVCPk8mOUJ6dJjRkWFODo7QPzRCKpUkm0kznk4zOJ5mMDkFI2NYYRAKU1CYwgpTUJyCwiQ27a8oqaYZfg9m+N2Yeag7z++QTX9opweWAPfeey9btmypqMLFqvmHomZ2O3C79zRtZi9UeRergMEqb7OZLafj1bG2p2V5rFu3bl3Kds6tZKWlBPpxYOO05xu8ZWdwzt0J3LmE/czJzHZW8qdIu1hOx6tjbU861tpZypWiPwXOM7MtZhYCbgHuq05ZIiKyUIseoTvn8mb2IeABSqct/rNzTnfgFRFpkCXNoTvnvgt8t0q1LFbNpnOa1HI6Xh1re9Kx1khdL/0XEZHaUbdFEZE20RKBbmYrzOxBM3vJ+zrjlTBmdr+ZjZrZt89avsXMdpjZPjP7qvchblNawLFu99Z5ycy2T1v+iJm9YGZPef+a7t5wZvZOr8Z9ZvbRGV4Pe9+nfd73bfO01/7IW/6CmV1Xz7oXY7HHamabzWx82vfxn+pd+0JVcKxvMbOfmVnezG4+67UZf56b1RKPtTDt+1rdE0nKV1I28z/gr4CPeo8/CnxylvWuBd4DfPus5V8DbvEe/xPw240+pqUcK7ACOOB97fEe93ivPQJsa/RxzHF8fmA/sBUIAU8DF561zn8G/sl7fAvwVe/xhd76YWCLtx1/o4+pRse6GXiu0cdQ5WPdDLwO+CJw87Tls/48N+O/pRyr91q6VrW1xAgduBG4y3t8F3DTTCs55x4Czrju2kqXR14DfH2+9zeJSo71OuBB59ywc24EeBB4Z53qW6orgH3OuQPOuUngK5SOebrp/w2+DlzrfR9vBL7inMs55w4C+7ztNaulHGurmfdYnXOHnHPPAGffmbvVfp6Xcqw11SqBvsY5d9J73AesWcB7VwKjzrm89/wYpbYFzaqSY52p7cL0Y/q89+fcnzZhOMxX+xnreN+3MUrfx0re20yWcqwAW8zsSTP7oZldXetil2gp35t2/L7OJWJmO83sJ2ZW1cFl0/RDN7PvAzPdPeFj058455yZtfSpOTU+1l93zh03sw7gG8AHKP3ZJ63lJLDJOTdkZpcD95jZRc65ZKMLkyU71/sd3Qo8bGbPOuf2V2PDTRPozrm3zfaamZ0ys3XOuZNmtg7oX8Cmh4BuMwt4I6AZWxTUUxWO9Tjwi9Oeb6A0d45z7rj3NWVmd1P687CZAr2SlhHldY6ZWQDoovR9rKjdRBNZ9LG60mRrDsA5t8vM9gOvAZq1//RSvjez/jw3qSX9HE77HT1gZo8Al1Gak1+yVplyuQ8of/K9Hbi30jd6vxg/AMqfNC/o/Q1QybE+ALzDzHq8s2DeATxgZgEzWwVgZkHg3cBzdah5ISppGTH9v8HNwMPe9/E+4BbvzJAtwHnAE3WqezEWfaxm1mulew7gjeTOo/RhYbNaSiuQGX+ea1RnNSz6WL1jDHuPVwFvBma9h8SCNfoT4wo/VV4JPAS8BHwfWOEt30bpTknl9R4FBoBxSvNa13nLt1L6xd8H/CsQbvQxVeFYf9M7nn3AB71lcWAX8AzwPN7dpBp9TDMc4w2Ubo6yH/iYt+zPgV/yHke879M+7/u2ddp7P+a97wXg+kYfS62OFfhV73v4FPAz4D2NPpYqHOsbvN/LDKW/uJ6f6+e5mf8t9liBnweepXRmzLPAbdWsS1eKioi0iVaZchERkXko0EVE2oQCXUSkTSjQRUTahAJdRKRNKNCl5ZjZTWbmzOyCOdb5QrnLnZl91swunGGdoJl9wuvw9zMze9zMrvdeO1Q+p1+kVSjQpRXdCvy793Vezrn/5Jyb6eKNvwDWARc7515PqRFaR9WqFKkzBbq0FDNLAFcBt1G6Qq+83Mzs77we1d8HVk977REz23bWdmLAbwG/65wrX2J/yjn3tRn2+ftm9pz37yPesriZfcfMnvaWv99bfrnXTGuXmT3gtW8QqYum6eUiUqEbgfudcy+a2ZCZXe6c2wX8MnA+pZ7payhdTv3Pc2zn1cARN0+zK68x1geBNwIG7DCzH1K6+viEc+5d3npdXruF/wPc6Jwb8EL+LyldBSlScxqhS6u5lVL/abyv5WmXtwBfds4VnHMngIertL+rgG855zLOuTTwTeBqSpdtv93MPmlmVzvnxij9D+Vi4EEzewr4E0qNm0TqQiN0aRlmtoLSzUp+zmsr7Aecmf3hIja3D9hkZp3zjdJn4v2F8HpKPT0+bmYPAd+i1LPjykXUI7JkGqFLK7kZ+L/OuXOdc5udcxuBg5RGzD8C3m9mfm/e+q1zbcg5lwU+B3za65iH1+HwvWet+ihwk5nFzCxOaWrnUTM7B8g6574E/C/g9ZQahvWa2ZXe9oJmdlGVjl1kXgp0aSW3UhoFT/eNactfojR3/kXg8bPWm6kL3Z9Q6s6528yeA74NnDFad879DPgCpU6IOyh1vHwS+DngCW9q5b8DH3el25HdDHzSzJ6m1Cnx5xd1pCKLoG6L0vbM7FlKbU0PNroWkVrSCF3ampk9CDyrMJflQCN0EZE2oRG6iEibUKCLiLQJBbqISJtQoIuItAkFuohIm1Cgi4i0if8PKup1eE8UQPEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(diff,fit=stats.t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f0180614b38>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAD8CAYAAABkbJM/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XPV56P/PM9pHu6zF1mbJ4AWbxRhjSMhCNgJJA+l2CwkNSZPStEl7b3ubX8nNfUEu6e/XpOmre9qU5FKStCUJZCMJCQUCIQ0YMGAWG29ItiVZ1r6P1pnn98ecI4+FZI2kOXNmed6v1+CZs4weZs6Z53zXI6qKMcYYs5yA3wEYY4xJD5YwjDHGxMUShjHGmLhYwjDGGBMXSxjGGGPiYgnDGGNMXCxhGGOMiYslDGOMMXGxhGGMMSYuuX4HkEjV1dXa0tLidxjGGJM2nnvuuX5VrYln24xKGC0tLezbt8/vMIwxJm2IyIl4t7UqKWOMMXGxhGGMMSYuljCMMcbExRKGMcaYuFjCMMYYExdLGMYYY+JiCcMYY0xcPEsYItIkIo+JyEEROSAi/32RbURE/l5EjonISyKyK2bdLSJy1Hnc4lWcZmVGQrN87cnjfO+FTmbDEb/DMRlmZi7Cffs6+MbeE4xNzfodjlnAy4F7c8D/VNXnRaQUeE5EHlbVgzHbXAdsdh5XAP8MXCEiVcAdwG5AnX0fUNUhD+M1y+gfn+bX/ulJTg6GAPjhi9185UO7yQmIz5GZTDA9F+aWu59hb9sgAF9/8jj3ffwNVATzfY7MuDwrYahqt6o+7zwfA14FGhZsdgPwdY3aC1SIyAbg3cDDqjroJImHgWu9itXE544fHKBndIp7f/dKbv+V7fzsUC//+st2v8MyGeKrv2hnb9sgn/+1i7jnI5fT3j/B5370qt9hmRhJacMQkRbgUuDpBasagI6Y153OsqWWG5+80jXCj1/u5vfeeh5vOG8dH7mqhbduqeEfHzvG+PSc3+GZNDc4McM/P/4a79pex417mrl6ay0ffVMr332hk/b+Cb/DMw7PE4aIlADfAf6Hqo568P63isg+EdnX19eX6Lc3jq/8oo3Swlw+9uZWAESEP3rHZoZDszyw/5TP0Zl0953nOhmfnuNPr9k6v+yjb24lLyfAV37R5mNkJpanCUNE8ogmi39X1e8uskkX0BTzutFZttTy11HVu1R1t6rurqmJa8JFs0JjU7M8dOA0N+ysp6wwb375ruYKttaV8q19HefY25hzU1W++exJLttYydb1pfPLa0sLed/F9fxw/ymmZsM+RmhcXvaSEuD/Aq+q6l8vsdkDwIec3lJXAiOq2g08BFwjIpUiUglc4ywzPvjJK6eZmo3w67saz1ouIvzm7kZe7Bi2agOzagdOjfJa3wS/eVnj69Zdv7Oesek5njhitQepwMsSxlXAbwNvF5H9zuM9IvJxEfm4s82DQBtwDPgK8AcAqjoIfA541nnc6SwzPnjkYA8NFUXsbKp43bp371gPwKOv9iQ7LJMhHj7Ygwi8a3vd69a98bx1VBXn8+OXu32IzCzkWbdaVf0v4Jz9LVVVgU8sse5u4G4PQjMrMD0X5r+O9fOrlzYQLTSerakqyNa6Uh55tYePvXmTDxGadPfooR52NVeyrqTgdevycgJcvbWGxw71EokoAevC7Ssb6W3O6dn2IUIzYd6+rXbJbd5+QS3PHh9iwnpLmRXqH5/mla7Rcx5fb9lcw1BolgOnEt5nxqyQJQxzTk+19ZMTEK7ctG7Jbd6waR3hiPL8SRtXaVbm2fZoTfMbzlv6+Lrq/GoAnjhq7Rh+s4RhzunZ40NcWF9GccHStZe7NlaSExCebrNmJrMyT7cPUpSXw4X15UtuU1NawNa6Up5ut+PLb5YwzJKm58Ls7xhmd0vVObcrKcjlwvoynrET2qzQ0+2D7NpYQX7uuX+Kdm2s5IWTQ0QimqTIzGIsYZglvdI1wsxchMuXSRgAe1qr2N8xbP3lTdxGQrMcOj3Knpalq6Ncl22sZGxqjmN940mIzCzFEoZZ0jPt0TaJ3S2Vy257eUsVM+EIr3SNeB2WyRDPdwyhCpe3Ln987WqOdul+/oS1k/nJEoZZ0osdw2xcF6R6ke6OC13ijNF4qdMShonPAefi4sKGpdsvXK3VxVQG83jOEoavLGGYJR3oHjlnY2SsurJCaksLrIRh4vZK1yit1cVnTTezFBFhV3MlL3QMJyEysxRLGGZRI6FZOgYn2dFQFvc+FzWU85IlDBOnl7tG2FEf//G1o6Gctr5xJmesncwvljDMog50R3/4d8RZwgC4qLGc1/rGbQCfWdbQxAxdw5NcFEd1lGtHfRkRhUOnbQCfXyxhmEUddEbVruQK8KKGclSxEblmWe4xEk/7hWv7huixeLDbji+/WMIwi3qla4T1ZYVxNXi73KtFa8cwy3m5yy3Bxn9B0lhZRFlhrl2Q+MgShlnUgVOjKzqZIToitzKYx9HeMY+iMpnilVMjNFUVreh+3SLC9vqy+dKvST5LGOZ1JmfCvNY3zo4VVBdA9ITeUlfK4dOWMMy5Heoena9iWokd9eUcOj1K2EZ8+8IShnmdY73jRBQuiLn7Wby2ri/laM840ZnrjXm96bkwxwdCbKlb+fG1bX0pU7MRTg6GPIjMLMfLO+7dLSK9IvLKEus/FXNjpVdEJCwiVc664yLysrNun1cxmsW5VUqbV3FCb6krZWx6ju6RqUSHZTJEe/8E4Yhyfm3Jivd1j8mjPVaK9YOXJYx7gGuXWqmqX1TVnaq6E/g08PMFd9V7m7N+t4cxmkUc7R0nL0fYuC644n3dq8bDdkKbJRztic4HtZoShptkjvbanFJ+8CxhqOoTQLzTl94E3OtVLGZljvaM01pdTF7Oyg+PLXXRE/qItWOYJRztGSMgsKmmeMX7lhTkUl9eyDFLGL7wvQ1DRIJESyLfiVmswH+KyHMicqs/kWWvY71jq6ouAKgI5lNXVmAlDLOko73jtKwrpiA3Z1X7n19Xaj3xfOJ7wgDeB/xyQXXUm1R1F3Ad8AkRectSO4vIrSKyT0T29fXZHbnWamo2zMnBEOfXrry6wLWlrnS+2sGYhY70jLG5bnUXJACba0uiHTOsp1TSpULCuJEF1VGq2uX82wt8D9iz1M6qepeq7lbV3TU1NZ4Gmg3a+iaIaPSkXK0tzhWgndBmIbeH1OY1XJBsri1hajZC1/BkAiMz8fA1YYhIOfBW4Acxy4pFpNR9DlwDLNrTyiTemR5Sq08Ym2qKmZqNcHrUekqZsx3vDxGO6JqOrzMN31YtlWxedqu9F3gK2CoinSLyURH5uIh8PGazXwX+U1UnYpbVAf8lIi8CzwA/VtWfehWnOdux3nECEr3/wGq5+7b3Tyyzpck2bmP1atvIYve1as/ky/XqjVX1pji2uYdo99vYZW3AJd5EZZZzrHecjWtokATYVB09odv6J7jq/OpEhWYywPGB6EXEWi5IKoL5VJcUWE8pH6RCG4ZJIccHQrSsYvxFrLqyAoL5ObTZ/ZfNAu39E87xsbZr1U3VxfPJxySPJQwzT1U5MTBByxqu/iA6p1RrdbFVSZnXae+foGXd2o4vgJbqIO39Nj1IslnCMPN6x6YJzYTXVF3gsoRhFnO8fyIhx1dLdTH949OMTc0mICoTL0sYZp77A5+IK8BN1cV0DIaYmYus+b1MZhidmmVgYmbNJViAVucYPTFgpYxksoRh5p1IQIOkq7WmmIjCyUErZZio4wm8IGmxnni+sIRh5rX3h8jLETaUF675veZ7SvXZCW2i3B/3hFRJOUnnuCWMpLKEYeYd75+gqSpI7iomHVzIrgDNQsedRurVzIK8UFF+DuvLCmm3nlJJZQnDzDs+MDFfN7xW5UV5VJfkW8Iw89r7x2moKKIwb/VjfGK1VhdbCSPJLGEYINql9ngCutTGaq0utiopM699IERL9dpLF66W6mKOW6N3UlnCMAD0jE4zNRtJaMLYuK7YbqVp5h1P0BgMV2t1kMGJGUYmrWttsljCMEBsl9rEXQE2VwU5PTrF1Gw4Ye9p0tOQ88OeiAZvlzV8J58lDAOcmeMnkVeAzVXR5NM5ZNNQZ7t2D44vm+Qy+SxhGCCaMPJzAtRXFCXsPZuchGFjMcz8GIwEljDOHF9W7ZksljAMED2hm9cFyQlIwt7TLWGctIbJrHdiIIQINFUl7oKkMC+HurICSxhJZAnDANETemNV4tovAKpL8gnm53By0Kqksl3HUIj1ZYVrmjZ/Mc1VQTosYSSNJQyDqtI5NDlfxE8UEaG5KmhXgIaOwRBNlYk9viBaLWUJI3m8vOPe3SLSKyKL3l5VRK4WkRER2e88bo9Zd62IHBaRYyJym1cxmqjh0Czj03MJTxhgJ7SJ6hhM/AUJREsY3aNTTM9ZT7xk8LKEcQ9w7TLb/EJVdzqPOwFEJAf4EnAdsB24SUS2exhn1usYiv6gN1Umrn7Z5ZYwVDXh723Sw9RsmJ6xqYS2X7iaKoOoQpf1xEsKzxKGqj4BDK5i1z3AMVVtU9UZ4JvADQkNzpylw2lj8OoKcHI2TN/4dMLf26SHruFJVPGkSqrZGTfUYQkjKfxuw3iDiLwoIj8RkR3OsgagI2abTmeZ8YjbxuBVwgCsWiqLdSTh+LJ2suTwM2E8D2xU1UuAfwC+v5o3EZFbRWSfiOzr6+tLaIDZomMoRGUwj5KCtd1neTHuFaCd0NnLvfpv9iBh1JQUkJ8bsAuSJPEtYajqqKqOO88fBPJEpBroAppiNm10li31Pnep6m5V3V1TU+NpzJmqYzDkydUfQENFESJ2Z7Rs1jkYIj83QG1pQcLfOxAQmiqLbKxPkviWMERkvYiI83yPE8sA8CywWURaRSQfuBF4wK84s0Hn0KQn9csQHVy1vqzQShhZ7ORgiMaKIgIJHBQaq7kqON9xw3gr8XUQDhG5F7gaqBaRTuAOIA9AVb8M/Abw+yIyB0wCN2q0K82ciHwSeAjIAe5W1QNexZntIhGla2iSa3bUefY3rGttdusY8q4EC9GEse/4EKqKcw1qPOJZwlDVm5ZZ/4/APy6x7kHgQS/iMmfrGZtiJhzxrIQB0RP6F0etfSlbdQxOsrOpwrP3b6oKMjY9x8jkLBXBfM/+jvG/l5TxmZddal0bq4LO/TZscFW2GZmcZWRy1tMLEpuEMHksYWS5+S6PHgzac833lbcTOuu437kXPaRcZ7pu21gMr1nCyHIdQ9FZRBs8TBiNle7gKksY2aZzyLsxGC4rYSSPJYws1zE4SV1p4mcRjeVOCWFdH7PPfJWnh1VSJQW5VBXnW8JIAksYWS7ag8W70gVEB1cV5gVs+oYsdHIwRFlhLuXBPE//TlNVcL40Y7xjCSPLdXo07XQsEaGx0rrWZiOvu9S6miqLrISRBJYwstjMXITu0Skak3BCRwdXWQkj23h1H4yFmqqCnBqeJByxWZG9ZAkji52an0XU2yopiP6NTpvmPKtEIu6NuZJxfAWZDSunR6c8/1vZzBJGFutIQg8WV+zgKpMd+sanmZ6LeNql1uUmJav29JYljCyWjEF7rvmutdZXPmu4P97JqPJsqrSxPslgCSOLdQyFyMsR1pcVev635q8ArSdL1jhzJ0fvE0a9MyuytZN5yxJGFusYDFFfUUSOR7OIxrLBVdnHLU02JqGNLD83wIayQjrt+PKUJYws1uHhtOYLlRXmUV6UZ1UGWaRjMERtaQGFed4NCo3VaNOce84SRhaL3jjJ+6s/V1NVkVUZZJHOocmklC5cTZVBayPzmCWMLDUxPcfgxMx8Y3QyNFcFrcogi3QOJ2fQnqupqoiesSmm52xWZK94ljBE5G4R6RWRV5ZY/0EReUlEXhaRJ0Xkkph1x53l+0Vkn1cxZrPOoeT1kHI1VQbpHJokYoOrMt5cOMKp4amklzBUoctKsZ7xsoRxD3DtOda3A29V1YuAzwF3LVj/NlXdqaq7PYovq83PIprEE7qxKshMOELv2HTS/qbxx+nRKcIRTWoJ1r34sWpP73iWMFT1CWDwHOufVNUh5+VeoNGrWMzrzfeRT+YJXWlda7PFfAk2qQnDBu95LVXaMD4K/CTmtQL/KSLPicitPsWU0TqHJinMC1BdkrxbWs53rbVpzjPemQuS5JVg60oLyc8J2AWJhzy7p3e8RORtRBPGm2IWv0lVu0SkFnhYRA45JZbF9r8VuBWgubnZ83gzRcdQiMbKICLej8FwNVRYCSNbdA5NIgIbKrwfFOoKBISGyiI6raeUZ3wtYYjIxcBXgRtUdcBdrqpdzr+9wPeAPUu9h6repaq7VXV3TU2N1yFnjM6hyaS2XwAU5uVQV1ZgXR+zQOfQJOvLvL0x12IaK4vsgsRDviUMEWkGvgv8tqoeiVleLCKl7nPgGmDRnlZm9aJ95JNXv+xqqrTBVdkgWoJN7gUJRKs9rQ3DO55VSYnIvcDVQLWIdAJ3AHkAqvpl4HZgHfBPTrXInNMjqg74nrMsF/gPVf2pV3Fmo9GpWUYmZ305oZurguxtG1h+Q5PWuoYm2dNalfS/21QZZCg0y/j0HCUFvte4ZxzPPlFVvWmZ9R8DPrbI8jbgktfvYRKlM4mz1C7UWBWke38XM3MR8nNTpc+FSaTZcITukeRXecLZPaUu2FCW9L+f6eyMzUJulZAvVQaVRahGb95kMlP38BQRTW6XbZdNc+6tuBKGiHxXRN4rIpZgMoAffeRdZwZX2QmdqTr9vCCxwXueijcB/BPwAeCoiHxeRLZ6GJPxWOdQiOL8HCqCeUn/2zbNeebzY9oZV2Uwj2B+jpUwPBJXwlDVR1T1g8Au4DjwiDP/00dEJPm/OmZNOgYnkz4Gw7W+rJC8HLGutRmsYyhEQGB9efLGYLhExJmzzBKGF+KuYhKRdcCHiTZUvwD8HdEE8rAnkRnPdA4ld1rzWDkBob7C+spnss6hSTaUF5GX408NdlNV0XwpxyRWvG0Y3wN+AQSB96nq9ar6LVX9Q6DEywBNYqmqb2MwXE2VNs15Juv0aQyGq7EyOhZD1WZFTrR4LwG+oqrbVfUvVLUbQEQKAGw22fQyMhnto+7nCd1UFbRGyQzmVnn6pakqyMRMmKHQrG8xZKp4E8afL7LsqUQGYpLDLar7e0IXMTgxw8T0nG8xGG9Mz4XpGUvufTAWmp8V2UqxCXfOgXsish5oAIpE5FLAbSUtI1o9ZdKMexL51YYBMX3lh0JsW2+DqzJJ9/AUqv70kHLFdt2+pKnCtzgy0XIjvd9NtKG7EfjrmOVjwP/yKCbjodQoYbiDqyYtYWQYPweFumKPL5NY50wYqvo14Gsi8uuq+p0kxWQ81DkUorQwl/Ii/3pDu1UGNhYj85y5IPEvYZQU5FIZzLOeeB5YrkrqZlX9N6BFRP5k4XpV/etFdjMprGNo0pcR3rGqivNtcFWG6hwKkRsQ1pclfwxGLJu11hvLVUkVO/9a19kM0TkUomVd8fIbesgGV2WujsFJNlQUkuvTGAxXU2WQg92jvsaQiZarkvoX59//k5xwjJdUlY7BSd682f8bTTVVFVkdcwbqHArRWOF/f5jGqiIePthDJKIEAsmf0SBTxTtw7y9FpExE8kTkURHpE5GbvQ7OJNbgxAyTs2Ff65dd0bEYNrgq03QOTfraA8/VVBlkJhyhZ2zK71AySrzlxmtUdRT4FaJzSZ0PfMqroIw3UqGHlKupMkhoJszgxIzfoZgEmZoN0zs2nRrHl/WU8kS8CcOtunovcJ+qjsSzk4jcLSK9IrLoLVYl6u9F5JiIvCQiu2LW3SIiR53HLXHGac7B7TWSEleANg11xuka9r+HlMsG73kj3oTxIxE5BFwGPCoiNUA8Zb17gGvPsf46YLPzuBX4ZwARqSJ6S9crgD3AHSJSGWesZgkpVcKosq61mcbPac0XaqgsQsTuu5Jo8U5vfhvwRmC3qs4CE8ANcez3BDB4jk1uAL6uUXuBChHZQHTA4MOqOqiqQ0RnxD1X4jFx6BgMURnMS4l7Hdud0TKP+12mQgmjIDeHutJCq5JKsJX8cmwjOh4jdp+vr/HvNwAdMa87nWVLLX8dEbmVaOmE5ubmNYaT2fyepTZWcUEuVcX51rU2g3QOTZKXI9SW+jsGw9VUZdPoJ1q8vaS+AfwV8CbgcueRErPUqupdqrpbVXfX1PjfXTSV+XkfjMU0VVrX2kzSORSioaKInBTpxmrT6CdevCWM3cB2TXwfyC6gKeZ1o7OsC7h6wfLHE/y3s4p7H4x3XFDndyjzGquCvNIVV/8JkwY6BkMpU4KF6PHVvb+LmbkI+bn+DiTMFPF+iq8A6z34+w8AH3J6S10JjDj323gIuEZEKp3G7mucZWaV+sammZ6LpET9squ5Ksip4UnCERuLkQlODoZoXpc6CaOpsghVODVspdhEibeEUQ0cFJFngGl3oapef66dROReoiWFahHpJNrzKc/Z98vAg8B7gGNACPiIs25QRD4HPOu81Z2qeq7Gc7OME07RvDkFerC4miqDzIaV06NTNFSkTiIzKzc6NctQaDa1jq+Yac5bqv2dDidTxJswPruaN1fVm5ZZr8Anllh3N3D3av6ueb2TA9GEsdHneaRiue0pHYMhSxhpbv74SsWEYe1kCRNvt9qfEx3hnec8fxZ43sO4TIKdGAwREFLqh9ntWmtjMdLfyfkbc6VOwlhfVkhejlhPqQSKt5fU7wL3A//iLGoAvu9VUCbxTg5MsKG8KKUa/+orooOrrCdL+nMTxsYUasPICQj1FUU21ieB4v31+ARwFTAKoKpHgVqvgjKJd2IwlFInM0B+boANZYU2PUgGODEQoqo4n9JC/27MtZimyqAdXwkUb8KYVtX5WeKcwXvWtSWNdKRgwoBo10e7Akx/HYOhlKqOcjVVFVkJNoHiTRg/F5H/BRSJyLuA+4AfeheWSaTx6Tn6x2dS84SuDFodcwY4MTiRUg3ersbKIAMTM0xMz/kdSkaIN2HcBvQBLwO/R7Q77P/2KiiTWGd6sKRODylXc1WQntFppmbDfodiVmk2HOHU8FRKdal1uRdJXTYWIyHi6larqhER+T7wfVXt8zgmk2AnByeA1GqQdLlda7uGJzmvxu4EnI7cwZepNGjPFTvN+Za6Up+jSX/nLGE4I7A/KyL9wGHgsHO3vduTE55JBLcHS0qe0FU2a226O5mCg0Jddnwl1nJVUn9MtHfU5apapapVRO9RcZWI/LHn0ZmEODEQoiKYR1mK9WABm+Y8E5wYSL0uta51xfkU5eVYT6kEWS5h/DZwk6q2uwtUtQ24GfiQl4GZxDk5GErJBkmA2tIC8nMDdkKnsY7BEPm5AepSZFrzWCJCY6WNxUiU5RJGnqr2L1zotGOk3uWqWdSJgRDNKTQlSKxAQGi0wVVp7cRAiKbKIgIpMq35Qk1VNhYjUZZLGDOrXGdSxFw4QtfwJM0pdB+MhRqrrGttOjs5GErJ9gtXU2V0LEbi786QfZbrJXWJiIwuslyA1Ct/mtc5NTxFOKIp2aXW1VRZxIsdw36HYVZBVTk5GGJPa5XfoSypqSrI2PQcI5OzVATz/Q4nrZ0zYahqTrICMd444XSpTcUeUq7mqiAjk7OMTs2mZMO8WdpQaJbx6bmULmG4sR0fCLHTEsaapM5MdMYTqdyDxWVdH9PX8QHngiSFE0arcy+M4/0TPkeS/jxNGCJyrYgcFpFjInLbIuv/RkT2O48jIjIcsy4cs+4BL+PMZCdTuAeLy7rWpq/2vuiP8Kaa1K3ybF4XRATaLWGsWbw3UFoxEckBvgS8C+gEnhWRB1T1oLuNqv5xzPZ/CFwa8xaTqrrTq/iyRVvfBK3rilO2BwucuTp1S0MmfbT3T5ATkJScp8xVkJtDY2WRJYwE8LKEsQc4pqptzky33wRuOMf2NwH3ehhPVmrrH58vkqeq8mAeVcX589UbJn2090/QXBUkLye1a7db1hVbwkgAL7/lBqAj5nWns+x1RGQj0Ar8LGZxoYjsE5G9IvJ+78LMXHPhCCcHQildXeBqrS7mtT47odNNW/9Eyl+QAGyqjiYM61q7NqlyWXAjcL+qxk5ZulFVdwMfAP5WRM5bbEcRudVJLPv6+mxexFidQ5PMRTStTmiTPiIR5XiaJIzW6uL5af7N6nmZMLqAppjXjc6yxdzIguooVe1y/m0DHufs9o3Y7e5S1d2qurumpmatMWcU9wc4HUoYm2pK6BubZmxq1u9QTJx6xqaYnA2nR8JwZkK2i5K18TJhPAtsFpFWEcknmhRe19tJRLYBlcBTMcsqRaTAeV5NdALEgwv3Nef2Wt84AK3VqT9tuPujYyd0+pjvIZUOCWOde3yN+xxJevMsYajqHPBJ4CHgVeDbqnpARO4UketjNr0R+KaeXbl4AbBPRF4EHgM+H9u7ysSnvX+CCqdBOdWd55SC2qwdI220Ocm9NQ1KsA2VReTlyHzMZnU861YLoKoPEr07X+yy2xe8/uwi+z0JXORlbNmgPU3qlyHaVz4g2AmdRtr7JyjKy0npMT6unICwcV2xDd5bo1Rp9DYeSKeEEe0rH6Stz6oM0kV7/wQt1ak9xieWda1dO0sYGSo0M0f3yFRa1C+7NtUUW5VUGmnvn0i74+v4QIhwxLrWrpYljAx1podU6jd4u1qtr3zamA1HODkYSpsSLESPr5m5CKeG7d4Yq2UJI0O5CSOdTuhNNSVMzoY5PTrldyhmGR2D0Sv1dDq+WtZZT7y1soSRodwujy0peqe9xbjVG+1WLZXy3KrDdOgh5Tq/NlraPtZr7WSrZQkjQ7X1T1BfXkhRfvrc0sQdYPiaXQGmvKPOj+55aVTlWV2ST0Uwbz52s3KWMDLUkZ4xzq8r9TuMFVlfVkgwP4fX7IROeUd7x1hfVkh5Ufrc8EpE2FxbwrHeMb9DSVuWMDJQOKIc6x1nS236XP1B9IQ+v7bEqgzSwLHecTbXpdfxBbC5rpQjPePWsWKVLGFkoI7BENNzEbakWQkDYHNtKUd67AowlUUiytGecTbXpuPxVcLI5KxNQrhKljAykPuDm45XgFvqSugdm2Y4ZCd0quoanmRyNpwEQpYeAAAZhUlEQVSWx5eb5I5atdSqWMLIQG6j3vlpViUFsGV99IQ+0mPVUqnK/bHdnIbHl5vkrNpzdSxhZKCjPWPUlxdSWpg+DZIutxrNqqVS11EnmadjlVRtaQGlhbnz/w9mZSxhZKAjPeNsTsP2C4D68kJKCnI5agkjZR3tHae2tIDyYPpdkLg9paxKanUsYWSYcER5rW+cLWlYvwxnekpZlVTqOtozlpbtF67NtaVWJbVKljAyzEmnh1S6ljAg2vBtVVKpSVU52puePaRcm+tK6B+fYXDCOlaslKcJQ0SuFZHDInJMRG5bZP2HRaRPRPY7j4/FrLtFRI46j1u8jDOTuD+06dil1rWlrpSBiRkGxqf9DsUs0DU8SWgmPXtIubY6HSsOdY/6HEn68SxhiEgO8CXgOmA7cJOIbF9k02+p6k7n8VVn3yrgDuAKYA9wh4hUehVrJjlyOpow0rGHlOtMw7dVG6SaV7ujx9cFG8p8jmT13NgPWsJYMS9LGHuAY6rapqozwDeBG+Lc993Aw6o6qKpDwMPAtR7FmVEOdo/Ssi5ISYGnN1P0lHsFePi0ndCp5uCpUURg2/r0LcFWlxRQW1pgCWMVvEwYDUBHzOtOZ9lCvy4iL4nI/SLStMJ9zQIHu0fZXp++V38Q7fq4rjifA6fshE41B7tHaK0uJpifvhckANvry+ZLSyZ+fjd6/xBoUdWLiZYivrbSNxCRW0Vkn4js6+vrS3iA6WR0apYTAyF21Jf7HcqaiAjb68ssYaSgg92jbE/j6ijXBRvKONY7xsxcxO9Q0oqXCaMLaIp53egsm6eqA6rqtmx+Fbgs3n1j3uMuVd2tqrtramoSEni6etX5gc2EE3pHfTlH7YROKSOTs3QMTqZ9CRai58hsWK177Qp5mTCeBTaLSKuI5AM3Ag/EbiAiG2JeXg+86jx/CLhGRCqdxu5rnGXmHNw62R0ZcELvqI+e0Na9NnW4vYoy4YLEGr5Xx7OKSFWdE5FPEv2hzwHuVtUDInInsE9VHwD+SESuB+aAQeDDzr6DIvI5okkH4E5VHfQq1kxx4NQo1SX51JQW+B3Kml3YEK1WO3hqdP658Zf745oJJYzW6mIK8wK8agljRTxtuVLVB4EHFyy7Peb5p4FPL7Hv3cDdXsaXaQ6eGmV7fTki4ncoa7axKtrT65VTI/y3s2onjV8Onhp1ehgV+h3KmuUEhK3ryzho7WQr4nejt0mQmbkIR3vHMqI6CiAQEC7YUGoN3ykkE3rgxdq+oYyD3aN2M6UVsISRIY72jjEb1oyoX3btqC/n1e5RwhE7of02PRfmSM9YRh1fFzeWMzIZ7Vlo4mMJI0O83DkCkFH1/dvrywjNhDk+MOF3KFnv4KlRZsPKzqbMOb52NlUAsL9j2OdI0ocljAzxwslhKoJ5tKwL+h1KwlzS6JzQJ+2E9pv7o7qzKXNm6NlSV0owP8cSxgpYwsgQ+zuG2dlUkREN3q7NtSWUFuTy/Mkhv0PJevs7hllfVsj68vRv8HblBIQLG8otYayAJYwMMDY1y5HesfkidqYIBISdzRU8byUM373YMcwlGVQd5bq0qYKDp0aZngv7HUpasISRAV7uHEEVLm3OnOoC16XNlRw+Pcr49JzfoWStoYkZjg+EMqo6yrWzqYKZcMTmlYqTJYwM8IJbv9yYWSUMgF3NFUQUXrJqA9/s73TbLzLv+NrZ7LaTWbVnPCxhZIAXTg6zqaY4Le+xvJxLnataa8fwzwsnhwkIXNSYeVVS68sKqS0tsHaMOFnCSHORiPL8yaH5H9ZMUx7M4/zaEmvH8NHTbQPsqC9P63usLEVE2NVcyb4TdkESD0sYae5o7ziDEzNcuanK71A8s6u5ghdODtmIXB9Mz4V5oWOYPa2Ze3xdsamKzqFJuoYn/Q4l5VnCSHNPtw8AcOWmdT5H4p3dG6sYCs3aVNQ+eLFjhJm5CFdkcsJojZ47T7cN+BxJ6rOEkeb2tg3QUFFEY2WR36F45g3nRU/oXx7r9zmS7OP+iF7ekrkJY9v6UsqL8thrCWNZljDSmKryTPsgV7RWZdSAvYWaqoI0VRXx5Gt2QifbM8cH2ba+lMrifL9D8UwgIOxpreLpdruDwnIsYaSx1/rG6R+f4YoMbr9wvXFTNXvbBmwiwiSamYvw3ImhjG6/cF3RWsWJgRDdI9aOcS6WMNLYfx2NVtFkcvuF643nr2N0as7uX5BE+04MEpoJc9X51X6H4rkz1Z5Wij0XTxOGiFwrIodF5JiI3LbI+j8RkYMi8pKIPCoiG2PWhUVkv/N4YOG+Bh4/0kdrdTEb1xX7HYrn3uAkxV++Zu0YyfLzI33k5UhWJIztG8qoKS3g8cO9foeS0jxLGCKSA3wJuA7YDtwkItsXbPYCsFtVLwbuB/4yZt2kqu50Htd7FWe6mpoN89RrA1y9tcbvUJKitqyQbetLeeyQndDJ8vPDfezeWJWR4y8WEhGu3lLDE0f6mAtH/A4nZXlZwtgDHFPVNlWdAb4J3BC7gao+pqru3Uv2Ao0expNRnmobYHouwtVba/0OJWnecUEt+04MMRKa9TuUjHd6ZIpDp8d4a5ZckAC8bVsto1NzNur7HLxMGA1AR8zrTmfZUj4K/CTmdaGI7BORvSLyfi8CTGePH+qlMC+Q0f3jF3rHBXWEI8rjR6yU4bUnjvQBZE0JFuCq86vJCQiPWbXUklKi0VtEbgZ2A1+MWbxRVXcDHwD+VkTOW2LfW53Esq+vry8J0fpPVXn0UC9vPK+awrwcv8NJmp2NFVSX5PPIq3ZCe+0/D56mvryQrXWlfoeSNOVFeVy2sZJH7fhakpcJowtoinnd6Cw7i4i8E/gMcL2qTrvLVbXL+bcNeBy4dLE/oqp3qepuVd1dU5MdV0Mvd43QOTTJtTvW+x1KUgUCwtu31fL44V5mrZ7ZM6NTszxxpJ/rLtqQ0eN7FnPtjvUcOj3Ga302q8BivEwYzwKbRaRVRPKBG4GzejuJyKXAvxBNFr0xyytFpMB5Xg1cBRz0MNa08qOXuskNCNfsqPM7lKS7Zvt6xqbm+MXR7ChN+uGRgz3MhCO856INfoeSdO+5aAMi8OOXuv0OJSV5ljBUdQ74JPAQ8CrwbVU9ICJ3iojb6+mLQAlw34LusxcA+0TkReAx4POqagmDaHXUj1/q5s2bq6kIZu7o26W8ZUsNFcE8vv/CKb9DyVgPvtzNhvJCLs3A+18sZ315IZdvrLKEsQRP+8up6oPAgwuW3R7z/J1L7PckcJGXsaWrFzqG6Rqe5I/ftcXvUHyRnxvgPRdt4HvPdzExPUdxFnT5TKaRyWh11M1XbiQQyK7qKNd7L97AHQ8c4GjPGJuzqA0nHinR6G3id/9znRTkBnjX9uyrjnK9f2cDk7NhHj7Y43coGef7L3QxE47wa7vO1aExs73nog3kBoT7n+v0O5SUYwkjjYxPz/GDF7r4lYvrKS/KvLvrxWv3xkoaKor41rMdy29s4qaq3PvMSS5sKOPChsy7u168akoLeOcFddz3XCfTc2G/w0kpljDSyAP7TzExE+YDVzT7HYqvAgHhg1c281TbAEd6xvwOJ2O81DnCodNj3Hh5dh9fAB+4opnBiRkeOmCl2FiWMNKEqvLvT59g2/pSdjVnX2PkQjde3kx+boCvPXnc71Ayxjf2nqAoL4cbdtb7HYrv3nR+NU1VRfzb3hN+h5JSLGGkif861s+BU6Pc8saWrOsbv5iq4nyuv6Se7z7fZVOFJEDX8CTff6GL37q8idLC7K3udAUCwoeubOGZ9kGes/t9z7OEkSb+4WfH2FBemNWNkQt97M2tTM6G+cov2vwOJe195YnoZ3jrWzb5HEnq+MAVzVQG8/jSY8f8DiVlWMJIA0+3DfBM+yC3vmUTBbnZMxXIcratL+O9F2/gX3/ZzuDEjN/hpK3e0SnufeYkv3ppA/UVmXur35UqLsjld65q5WeHenmla8TvcFKCJYwUF4ko/9+Dr1JXVmCNkYv443duZnI2bFeBa/DFhw6jCp98+/l+h5JyPvTGFiqCeXzuRwdRtbs9WsJIcd95vpMXO0e47bptFOVb6WKh82tL+W+7m7jnyeN2N75VeKlzmPuf7+TDV7VkxY24Vqq8KI8/vWYrT7cP8iMb/W0JI5X1jk7x5z9+lV3NFdxwibVdLOW267ZRUZTHp7/3st3zewWm58J86r6XqCkp4BNvs9LFUm7a08yO+jL+/McHGcryqk9LGCkqHFE+df9LTM+F+eJvXpK10zTEoyKYz+3v286LHcP83aNH/Q4nbfzVQ4c53DPGX/zaRVk9EHQ5OQHhC79+MUMTs3zq/peyumrKEkaK+suHDvHzI3387/du57yaEr/DSXnXX1LPb1zWyN8/epSfHbLBVsv5/gtdfOUX7dx8ZTPvuCB7p5mJ14UN5fzZddt45NUevvzz7O2VZwkjBX39qeP8y8/buPnKZm6+cqPf4aQFEeHP338hO+rL+MS/v8Az7YN+h5SynjjSx5995yX2tFZx+6/s8DuctPE7V7Xwvkvq+cJPD2XtPFOWMFKIqvLPj7/G7T84wDsvqOOO99nJvBKFeTnc85E91FcU8pF/fYbH7Vabr/PQgdN87Ov7OK+mhC/ffBn5ufYTEC8R4a9+82LedH41n7r/Rf71l+1+h5R0drSkiJHQLJ/4j+f5wk8P8b5L6vnnm3eRl2Nfz0rVlBbwH797Jc3rivnIPc/yd48ctbvzEW3g/sJPD/F733iOCzaU8R+/ewVVxdl3P5W1KsjN4Ssf2s012+v4Pz88yJ/e9yKjU9kz04B42YAjItcCfwfkAF9V1c8vWF8AfB24DBgAfktVjzvrPg18FAgDf6SqDy3393bv3q379u1L6P+D16Zmw9z3XCd/8/ARRiZn+X/evZXfffMma+Reo9DMHJ/+7sv8YP8pttaV8qfv3so7ttVm3ecajigPHzzNF356mPb+CW7a08Qd79uRVfeC90I4ovztI0f40mPHqCkt4I/esZnfvKwpLUtsIvKcqu6Oa1uvEoaI5ABHgHcBnURv2XpT7J3zROQPgItV9eMiciPwq6r6WyKyHbgX2APUA48AW1T1nHMNp0vCCEeU508O8fDBHu5/rpPBiRn2tFRxx/Xb2VGfvdNKe+Hhgz3c+aMDdAxOsrm2hPdf2sB1F66ntbo4Y+fkUlWO9Izz0IHTfPf5To4PhNhUXcwd1+/grVuy4773ybK/Y5g7f3iA508OU1dWwPsvbeA9F27gwoZyctLk4iRVEsYbgM+q6rud158GUNW/iNnmIWebp0QkFzgN1AC3xW4bu925/mYqJAxVZXouQmgmzMT0HCOTs5wanqR7ZIqTgyEOnBrhQNcoY9Nz5OUIV2+t5XeuauXKTVUZ+wPmt7lwhB+91M3XnjrOCyeHAaguKWBXcwWbakpoWRekobKIymA+FcE8KoP5FOXlpGxpZHouzMR09Pgam5qjZ2yKU8OTdA5N8mr3KC93jjAwMYNI9N4hH7mqlXfvWJ82P2DpRlV5/Egf//bUCR4/0kc4opQW5rKzqYLNtaWcV1tMXWkh1aUFVJfkU1qYR2FegPycQEqc8ytJGF7e37IBiL3DTSdwxVLbqOqciIwA65zlexfs69nItV/5h18wORNGFRSIqKJ65l9VXbB8sWXRf6dmw8wtMXisIDfABRvKuH5nPVduWsdbt9ZQZjODei43J8D7L23g/Zc20DkU4udH+th3fIgXO4d57HAvs+HFv6/cgJCfGyA/N0BeToC8gMyf4CLOg5jXRBtGBcB5rUT/o0SPmYiC4h5Xrz+O3O0WHovudlOz4SXjzQkIm2tLePu2WnZtrOQd22qpLStM4CdpFiMivG1rLW/bWkv/+DS/PNbPU68N8MqpEf7jmRNMzS7ehhaQaEeNwrwccgJCwDmeAhJ9z0AAAs7xFHAOsIXpxT0eq4L5fPvjb/D2fxSP7+mdDCJyK3ArQHPz6uZaOr+mhNmIzn8xIme+KJl/7XyZgTM/CrHbRmOBorwcigtyKSnIpbggl9LCXDaUF7K+vJDq4oKUvWrNFo2VQT54xUY+eEW0u/JcOMKp4SlOj04xFJphODTDcGiWydkwM3MRZsMRZuYizIQjzIbdH3KdTwJw5sf8rB98iL6ISSSB2KTiHE+xx5ZI7DpedxwCFOXnUFKQSzD/zHFWV1ZAfUURNSUF5FpHCV9VlxRww84GbtgZvb6NRJTu0Sn6xqbpH5umf3ya8ek5pmbDTM1Gov/OhQlH3AuKsy9KI/MXGdGLh7PEvCwtTM5PuZd/pQtoinnd6CxbbJtOp0qqnGjjdzz7AqCqdwF3QbRKajWB/u2Nl65mN5MBcnMCNK8L0rwu6HcoJgMFAkJDRRENGTILsJeXI88Cm0WkVUTygRuBBxZs8wBwi/P8N4CfabRR5QHgRhEpEJFWYDPwjIexGmOMWYZnJQynTeKTwENEu9XeraoHROROYJ+qPgD8X+AbInIMGCSaVHC2+zZwEJgDPrFcDyljjDHe8nQcRrKlQi8pY4xJJyvpJWUtZMYYY+JiCcMYY0xcLGEYY4yJiyUMY4wxcbGEYYwxJi4Z1UtKRPqAE6vcvRroT2A4iWJxrYzFtTIW18pkYlwbVTWuWSkzKmGshYjsi7drWTJZXCtjca2MxbUy2R6XVUkZY4yJiyUMY4wxcbGEccZdfgewBItrZSyulbG4Viar47I2DGOMMXGxEoYxxpi4ZFXCEJEqEXlYRI46/1YusV1YRPY7jwdilreKyNMickxEvuVM256UuERkp4g8JSIHROQlEfmtmHX3iEh7TMw71xjPtSJy2Pn/vG2R9QXO//8x5/NoiVn3aWf5YRF591riWGFMfyIiB53P5lER2RizbtHvM4mxfVhE+mJi+FjMuluc7/2oiNyycF+P4/qbmJiOiMhwzDpPPjMRuVtEekXklSXWi4j8vRPzSyKyK2adl5/VcnF90InnZRF5UkQuiVl33Fm+X0QSOvtpHHFdLSIjMd/V7THrzvn9r0r0dpDZ8QD+ErjNeX4b8IUlthtfYvm3gRud518Gfj9ZcQFbgM3O83qgG6hwXt8D/EaCYskBXgM2AfnAi8D2Bdv8AfBl5/mNwLec59ud7QuAVud9cpIU09uAoPP8992YzvV9JvHz+jDwj4vsWwW0Of9WOs8rkxXXgu3/kOgtCDz9zIC3ALuAV5ZY/x7gJ0RvOngl8LTXn1Wccb3R/XvAdW5czuvjQLVPn9fVwI/W+v3H+8iqEgZwA/A15/nXgPfHu6OICPB24P7V7L/WuFT1iKoedZ6fAnqBuAbbrNAe4JiqtqnqDPBNJ76l4r0feIfz+dwAfFNVp1W1HTjmvJ/nManqY6oacl7uJXqXxmSI5/NayruBh1V1UFWHgIeBa32K6ybg3gT97SWp6hNE732zlBuAr2vUXqBCRDbg7We1bFyq+qTzdyGJx1ccn9dS1nJcLinbEkadqnY7z08DdUtsVygi+0Rkr4i4P97rgGFVnXNedwINSY4LABHZQ/Sq4bWYxf+vU2T+GxEpWEMsDUBHzOvF/j/nt3E+jxGin088+3oVU6yPEr1KdS32fSZKvLH9uvP93C8i7u2Hvfq8VvTeTvVdK/CzmMVefmbnslTcXn5WK7Xw+FLgP0XkORG51Yd43iAiL4rIT0Rkh7PMk88rOXcOTyIReQRYv8iqz8S+UFUVkaW6iG1U1S4R2QT8TEReJvqj6HdcOFdb3wBuUdWIs/jTRBNNPtHudX8G3LmWeNOViNwM7AbeGrP4dd+nqr62+Dt44ofAvao6LSK/R7R09vYk/v3l3Ajcr2ff1dLvzywlicjbiCaMN8UsfpPzWdUCD4vIIadkkAzPE/2uxkXkPcD3id7S2hMZV8JQ1Xeq6oWLPH4A9Dg/uO4Pb+8S79Hl/NsGPA5cCgwQLR67SbYR6EpmXCJSBvwY+IxTXHffu9spwk8D/8raqoG6gKaY14v9f85v43we5UQ/n3j29SomROSdRBPw9c5nASz5fSbKsrGp6kBMPF8FLot3Xy/jinEjC6qjPP7MzmWpuL38rOIiIhcT/f5uUNUBd3nMZ9ULfI/EVMPGRVVHVXXcef4gkCci1Xj1ea21ESSdHsAXObtx+S8X2aYSKHCeVwNHcRqLgPs4u9H7D5IYVz7wKPA/Flm3wflXgL8FPr+GWHKJNii2cqaxbMeCbT7B2Y3e33ae7+DsRu82EtPoHU9MlxKtotsc7/eZoO8untg2xDz/VWCv87wKaHdirHSeVyUrLme7bUQbbSWJn1kLSzfivpezG72f8fqzijOuZqJtcm9csLwYKI15/iRwbRLjWu9+d0QT1Unns4vr+19xLIn8H0v1B9F69kedE+AR94AjWoXxVef5G4GXnQ/4ZeCjMftvAp5xDpz73JMqSXHdDMwC+2MeO511P3NifQX4N6BkjfG8BzhC9Af4M86yO4leuQMUOv//x5zPY1PMvp9x9jsMXJfA7265mB4BemI+mweW+z6TGNtfAAecGB4DtsXs+zvO53gM+Egy43Jef5YFFxhefmZESzLdzrHcSbR65+PAx531AnzJifllYHeSPqvl4voqMBRzfO1zlm9yPqcXne/4M0mO65Mxx9ZeYhLaYt//Wh820tsYY0xcMq4NwxhjjDcsYRhjjImLJQxjjDFxsYRhjDEmLpYwjDHGxMUShjHGmLhYwjDGGBMXSxjGGGPi8v8DBYyfu2O87rcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels.plot(kind='kde')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Test Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(x,y,model,test_size=15,train_size=128,epoch=100, writer=None):\n",
    "    train_loss=[]\n",
    "    train_acc=[]\n",
    "    train_mcc=[]\n",
    "    dev_loss=[]\n",
    "    dev_acc=[]\n",
    "    dev_mcc=[]\n",
    "    best_attns=None\n",
    "    best_result=None\n",
    "    best_score=0\n",
    "    global_step=0\n",
    "    best_result_epoch=0\n",
    "    for e in range(epoch):\n",
    "        pointer=0\n",
    "        chunk_train_loss=0\n",
    "        chunk_train_acc=0\n",
    "        chunk_train_mcc=0\n",
    "        train_count=0\n",
    "        chunk_test_loss=0\n",
    "        chunk_test_acc=0\n",
    "        chunk_test_mcc=0\n",
    "        test_count=0\n",
    "        test_yhat=[]\n",
    "        y_true=[]\n",
    "        epoch_attns=[]\n",
    "        while pointer<len(x):\n",
    "            train_x=x[pointer:pointer+train_size]\n",
    "            train_y=y[pointer:pointer+train_size]\n",
    "            train_length=len(train_x)\n",
    "            b_pointer=0\n",
    "            while b_pointer<train_length:\n",
    "                random_batch_size=np.random.randint(low=32,high=128)\n",
    "                batch_x=train_x[b_pointer:b_pointer+random_batch_size]\n",
    "                batch_y=train_y[b_pointer:b_pointer+random_batch_size]\n",
    "                loss,acc,mcc,_,attn=model.train(X=batch_x,y=batch_y)\n",
    "                epoch_attns.extend(attn)\n",
    "                chunk_train_loss+=loss\n",
    "                chunk_train_acc+=acc\n",
    "                chunk_train_mcc+=mcc\n",
    "                print('train loss',loss,'train acc',acc,'train mcc',mcc,'epoch:',e,'iter:',pointer)\n",
    "                if writer:\n",
    "                    writer.add_scalar(tag='train loss', scalar_value=loss, global_step=global_step)\n",
    "                    writer.add_scalar(tag='train acc', scalar_value=acc, global_step=global_step)\n",
    "                    writer.add_scalar(tag='train mcc', scalar_value=mcc, global_step=global_step)\n",
    "                b_pointer+=len(batch_x)\n",
    "                pointer+=len(batch_x)\n",
    "                global_step+=1\n",
    "                train_count+=1\n",
    "            if pointer>=len(x):\n",
    "                break\n",
    "            test_x=x[pointer:pointer+test_size]\n",
    "            test_y=y[pointer:pointer+test_size]\n",
    "            loss,acc,mcc,y_hat,attn=model.test(X=test_x,y=test_y)\n",
    "            epoch_attns.extend(attn)\n",
    "            test_yhat.append(y_hat)\n",
    "            y_true.append(test_y)\n",
    "            print('test loss',loss,'test acc',acc,'test mcc',mcc,'epoch:',e,'iter:',pointer)\n",
    "            chunk_test_loss+=loss\n",
    "            chunk_test_acc+=acc\n",
    "            chunk_test_mcc+=mcc\n",
    "            test_count+=1\n",
    "            pointer+=len(test_x)\n",
    "        model.reset_model()\n",
    "        chunk_train_loss=chunk_train_loss/train_count\n",
    "        chunk_train_acc=chunk_train_acc/train_count\n",
    "        chunk_train_mcc=chunk_train_mcc/train_count\n",
    "        \n",
    "        train_loss.append(chunk_train_loss)\n",
    "        train_acc.append(chunk_train_acc)\n",
    "        train_mcc.append(chunk_train_mcc)\n",
    "        \n",
    "        test_yhat=np.concatenate(test_yhat)\n",
    "        y_true=np.concatenate(y_true)\n",
    "        chunk_test_loss=chunk_test_loss/test_count\n",
    "        chunk_test_acc=accuracy_score(y_pred=test_yhat,y_true=y_true)\n",
    "        chunk_test_mcc=matthews_corrcoef(y_pred=test_yhat,y_true=y_true)\n",
    "        \n",
    "        if chunk_test_mcc+chunk_test_acc>best_score:\n",
    "            best_result=test_yhat.copy()\n",
    "            best_score=chunk_test_mcc+chunk_test_acc\n",
    "            best_result_epoch=e\n",
    "            best_attns=epoch_attns\n",
    "            \n",
    "        \n",
    "        \n",
    "        dev_loss.append(chunk_test_loss)\n",
    "        dev_acc.append(chunk_test_acc)\n",
    "        dev_mcc.append(chunk_test_mcc)\n",
    "        if writer:\n",
    "            writer.add_scalar(tag='dev loss', scalar_value=chunk_test_loss, global_step=global_step)\n",
    "            writer.add_scalar(tag='dev acc', scalar_value=chunk_test_acc, global_step=global_step)\n",
    "            writer.add_scalar(tag='dev mcc', scalar_value=chunk_test_mcc, global_step=global_step)\n",
    "        print('best_result_epoch',best_result_epoch)\n",
    "    return np.array(train_loss),np.array(train_acc),np.array(train_mcc),np.array(dev_loss),np.array(dev_acc), np.array(dev_mcc), best_result,y_true.flatten(), np.array(best_attns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trade(a,price_diff,initial_price,test_size=15,train_size=128):\n",
    "    pointer=0\n",
    "    test_r=initial_price\n",
    "    bnh_r=initial_price\n",
    "    while pointer<len(x):\n",
    "        pointer+=train_size\n",
    "        if pointer>=len(x):\n",
    "            break\n",
    "        test_a=a[pointer:pointer+test_size]\n",
    "        test_a=test_a*2-1\n",
    "        test_p=price_diff[pointer:pointer+test_size]\n",
    "        test_r+=(test_p*test_a).sum()\n",
    "        bnh_r+=test_p.sum()\n",
    "        pointer+=len(test_p)\n",
    "    return test_r,bnh_r,test_r/initial_price,bnh_r/initial_price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding:utf-8 -*-\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import accuracy_score, matthews_corrcoef\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "lmap = lambda func, it: list(map(lambda x: func(x), it))\n",
    "\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, d_x, d_h, temporal=False, dp=0.5):\n",
    "        super(Attention, self).__init__()\n",
    "        self.temporal = temporal\n",
    "        self.w = nn.Linear(d_x, d_h)\n",
    "        self.dropout = nn.Dropout(p=dp)\n",
    "        if self.temporal:\n",
    "            self.u = nn.Linear(d_h, d_h, bias=False)\n",
    "    \n",
    "    def forward(self, x_t, hidden=None):\n",
    "        if self.temporal and hidden is not None:\n",
    "            v_t = self.dropout(F.relu(self.w(x_t) + self.u(hidden)))\n",
    "        else:\n",
    "            v_t = self.dropout(F.relu(self.w(x_t)))\n",
    "        e_t = torch.matmul(v_t, torch.transpose(v_t, 2, 1))\n",
    "        e_t = e_t.sum(dim=1, keepdim=True)\n",
    "        e_t = F.softmax(e_t, dim=-1)\n",
    "        a_t = torch.matmul(e_t, x_t)\n",
    "        return a_t, e_t\n",
    "\n",
    "\n",
    "class AttnRNN(nn.Module):\n",
    "    def __init__(self, d_x, d_h, d_o, rnn_layers=2, ffn_layers=5, cell=nn.GRU, temporal=False, dp=0.5):\n",
    "        super(AttnRNN, self).__init__()\n",
    "        self.temporal = temporal\n",
    "        self.dropout = nn.Dropout(p=dp)\n",
    "        self.f_in = nn.Linear(in_features=d_x,out_features=d_h)\n",
    "        self.attn = Attention(d_h, d_h, temporal=self.temporal)\n",
    "        self.rnn = cell(input_size=d_h, hidden_size=d_h, num_layers=rnn_layers, batch_first=True, dropout=dp)\n",
    "        self.hls = nn.ModuleList([nn.Linear(in_features=d_h, out_features=d_h) for _ in range(ffn_layers)])\n",
    "        self.lms = nn.ModuleList([nn.LayerNorm(d_h) for _ in range(ffn_layers)])\n",
    "        self.f_out = nn.Linear(in_features=d_h, out_features=d_o)\n",
    "        for p in self.attn.parameters():\n",
    "            if p.dim() > 1:\n",
    "                nn.init.xavier_uniform_(p)\n",
    "    \n",
    "    def forward(self, x_t, hidden):\n",
    "        x_t=self.f_in(x_t)\n",
    "        if self.temporal and hidden is not None:\n",
    "            if type(hidden) == tuple:\n",
    "                h_t_1 = hidden[0]\n",
    "            elif type(hidden) == torch.Tensor:\n",
    "                h_t_1 = hidden\n",
    "            else:\n",
    "                raise RuntimeError(\"h_t_1 is None\")\n",
    "            a_t, e_t = self.attn(x_t, h_t_1[-1, None, :, :])\n",
    "        else:\n",
    "            a_t, e_t = self.attn(x_t, hidden=None)\n",
    "        o_t, hidden = self.rnn(a_t, hidden)\n",
    "        o_t_initial = o_t\n",
    "        for hl, lm in zip(self.hls, self.lms):\n",
    "            o_t = self.dropout(lm(F.leaky_relu(hl(o_t))))\n",
    "        o_t = self.f_out(o_t + o_t_initial)\n",
    "        return o_t, hidden, e_t\n",
    "\n",
    "\n",
    "class AttnFFN(nn.Module):\n",
    "    def __init__(self, d_x, d_h, d_o, ffn_layers=5, dp=0.5):\n",
    "        super(AttnFFN, self).__init__()\n",
    "        self.f_in = nn.Linear(in_features=d_x,out_features=d_h)\n",
    "        self.attn = Attention(d_h, d_h, temporal=False)\n",
    "        self.hiddens = nn.ModuleList([nn.Linear(in_features=d_h, out_features=d_h) for _ in range(ffn_layers)])\n",
    "        self.lms = nn.ModuleList([nn.LayerNorm(d_h) for _ in range(ffn_layers)])\n",
    "        self.f_out = nn.Linear(in_features=d_h, out_features=d_o)\n",
    "        self.dropout = nn.Dropout(p=dp)\n",
    "    \n",
    "    def forward(self, x_t, hidden=None):\n",
    "        # use hidden here for compatible\n",
    "        o_t=self.f_in(x_t)\n",
    "        o_t, e_t = self.attn(o_t)\n",
    "        o_t_initial = o_t\n",
    "        for hl, lm in zip(self.hiddens, self.lms):\n",
    "            o_t = self.dropout(lm(F.leaky_relu(hl(o_t))))\n",
    "        o_t = self.f_out(o_t + o_t_initial)\n",
    "        return o_t, hidden, e_t\n",
    "\n",
    "\n",
    "class AttentionClassifier(object):\n",
    "    def __init__(self, d_x, d_h, d_o, attn_model, lr=1e-3):\n",
    "        super(AttentionClassifier, self).__init__()\n",
    "        self.d_h = d_h\n",
    "        self.attn_model = attn_model\n",
    "        self.loss_func = nn.CrossEntropyLoss()\n",
    "        self.optimizer = optim.Adam(list(self.attn_model.parameters()), lr=lr)\n",
    "        self.tmp_hidden = None\n",
    "        for p in self.attn_model.parameters():\n",
    "            if p.dim() > 1:\n",
    "                nn.init.xavier_uniform_(p)\n",
    "    \n",
    "    def train(self, X, y):\n",
    "        self.attn_model.train(True)\n",
    "        self.attn_model.attn.train(True)\n",
    "        self.optimizer.zero_grad()\n",
    "        loss = 0\n",
    "        y_hats = []\n",
    "        attns = []\n",
    "        for i, x in enumerate(X):\n",
    "            y_hat_t, self.tmp_hidden, e_t = self.attn_model(\n",
    "                torch.tensor(x[None, :, :], dtype=torch.float32),\n",
    "                self.tmp_hidden\n",
    "            )\n",
    "            y_hat_t = y_hat_t.squeeze(0)\n",
    "            y_true_t = torch.tensor([y[i]], dtype=torch.long)\n",
    "            loss += self.loss_func(y_hat_t, y_true_t)\n",
    "            topv, topi = y_hat_t.topk(1)\n",
    "            y_hat = topi.detach().item()\n",
    "            y_hats.append(y_hat)\n",
    "            attns.append(e_t.view(-1).detach().numpy())\n",
    "        loss = loss / len(X)\n",
    "        loss.backward()\n",
    "        clip_grad_norm_(self.attn_model.parameters(), 1)\n",
    "        self.optimizer.step()\n",
    "        y_hat = np.array(y_hats)\n",
    "        if type(self.tmp_hidden) == tuple:\n",
    "            self.tmp_hidden = tuple([h.detach() for h in self.tmp_hidden])\n",
    "        elif type(self.tmp_hidden) == torch.Tensor:\n",
    "            self.tmp_hidden = self.tmp_hidden.detach()\n",
    "        else:\n",
    "            self.tmp_hidden = None\n",
    "        acc = accuracy_score(y_true=y.flatten(), y_pred=y_hat)\n",
    "        mcc = matthews_corrcoef(y_true=y.flatten(), y_pred=y_hat)\n",
    "        return loss.item(), acc, mcc, y_hat, attns\n",
    "    \n",
    "    def test(self, X, y):\n",
    "        self.attn_model.eval()\n",
    "        self.attn_model.attn.eval()\n",
    "        with torch.no_grad():\n",
    "            loss = 0\n",
    "            y_hats = []\n",
    "            attns = []\n",
    "            for i, x in enumerate(X):\n",
    "                y_hat_t, self.tmp_hidden, e_t = self.attn_model(torch.tensor(x[None, :, :], dtype=torch.float32), self.tmp_hidden)\n",
    "                y_hat_t = y_hat_t.squeeze(0)\n",
    "                y_t = torch.tensor([y[i]], dtype=torch.long)\n",
    "                loss += self.loss_func(y_hat_t, y_t)\n",
    "                topv, topi = y_hat_t.topk(1)\n",
    "                y_hat = topi.detach().item()\n",
    "                y_hats.append(y_hat)\n",
    "                attns.append(e_t.view(-1).detach().numpy())\n",
    "            loss = loss / len(X)\n",
    "            y_hat = np.array(y_hats)\n",
    "            acc = accuracy_score(y_true=y.flatten(), y_pred=y_hat)\n",
    "            mcc = matthews_corrcoef(y_true=y.flatten(), y_pred=y_hat)\n",
    "            return loss.item(), acc, mcc, y_hat, attns\n",
    "    \n",
    "    def inference(self, X):\n",
    "        self.attn_model.eval()\n",
    "        self.attn_model.attn.eval()\n",
    "        with torch.no_grad():\n",
    "            y_hats = []\n",
    "            attns = []\n",
    "            for i, x in enumerate(X):\n",
    "                y_hat_t, self.tmp_hidden, e_t = self.attn_model(torch.tensor(x[None, :, :], dtype=torch.float32), self.tmp_hidden)\n",
    "                y_hat_t = y_hat_t.squeeze(0)\n",
    "                topv, topi = y_hat_t.topk(1)\n",
    "                y_hat = topi.detach().item()\n",
    "                y_hats.append(y_hat)\n",
    "                attns.append(e_t.view(-1).detach().numpy())\n",
    "            y_hat = np.array(y_hats)\n",
    "            return y_hat, attns\n",
    "    \n",
    "    def reset_model(self):\n",
    "        self.tmp_hidden = None\n",
    "    \n",
    "    def load_model(self, model_path='./AttnModel'):\n",
    "        self.attn_model = torch.load(model_path + '/model.pkl')\n",
    "    \n",
    "    def save_model(self, model_path='./AttnModel'):\n",
    "        if not os.path.exists(model_path):\n",
    "            os.mkdir(model_path)\n",
    "        torch.save(self.attn_model, model_path + '/model.pkl')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment of pre-trained elmo encoded title, (mean with date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each news title is a vector produced by mean of Elmo representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_index=np.load('data/title_elmo_index.npy')\n",
    "content=np.load('data/title_elmo.npy')\n",
    "content=pd.DataFrame(content,index=date_index)\n",
    "content=content.join(labels).fillna(method='bfill').dropna()['2010-06':]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_=sorted(list(content.groupby(content.index).groups.keys()))\n",
    "\n",
    "x=[]\n",
    "for i in range(len(index_)):\n",
    "    x_=content.loc[index_[i]].values\n",
    "    if len(x_.shape)<2:\n",
    "        x.append(x_[None,:content.shape[-1]-1])\n",
    "    else:\n",
    "        x.append(x_[:,:content.shape[-1]-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=content.groupby(content.index)[content.columns[-1]].mean().values.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "del content\n",
    "del date_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "ffn_model=AttnFFN(d_x=x[0].shape[-1],d_h=300,d_o=class_number,ffn_layers=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "rnn_model=AttnRNN(d_x=x[0].shape[-1],d_h=300,d_o=class_number,cell=nn.GRU,rnn_layers=2,ffn_layers=5,temporal=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf=AttentionClassifier(attn_model=ffn_model,d_x=x[0].shape[-1],d_h=300,d_o=class_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss 1.6390303373336792 train acc 0.4927536231884058 train mcc 0.05221015967643629 epoch: 0 iter: 0\n",
      "train loss 2.5550293922424316 train acc 0.4772727272727273 train mcc -0.009929257812241655 epoch: 0 iter: 69\n",
      "train loss 1.9877315759658813 train acc 0.5135135135135135 train mcc 0.22038926600773587 epoch: 0 iter: 113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/mnt/miniconda3/envs/py36/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test loss 2.525205135345459 test acc 0.3333333333333333 test mcc 0.0 epoch: 0 iter: 150\n",
      "train loss 1.1163990497589111 train acc 0.6341463414634146 train mcc 0.22382147335384786 epoch: 0 iter: 165\n",
      "train loss 1.7476894855499268 train acc 0.5233644859813084 train mcc 0.0028733141061374375 epoch: 0 iter: 206\n",
      "train loss 1.7025482654571533 train acc 0.0 train mcc 0.0 epoch: 0 iter: 313\n",
      "test loss 0.8474798798561096 test acc 0.4666666666666667 test mcc -0.026207120918047958 epoch: 0 iter: 315\n",
      "train loss 1.1416571140289307 train acc 0.5495495495495496 train mcc 0.10301070542879115 epoch: 0 iter: 330\n",
      "train loss 2.3887083530426025 train acc 0.41025641025641024 train mcc -0.10857142857142857 epoch: 0 iter: 441\n",
      "test loss 0.8729877471923828 test acc 0.6 test mcc 0.16666666666666666 epoch: 0 iter: 480\n",
      "train loss 1.3628945350646973 train acc 0.6041666666666666 train mcc 0.16972502573910517 epoch: 0 iter: 495\n",
      "train loss 1.408629298210144 train acc 0.45454545454545453 train mcc -0.11654135338345864 epoch: 0 iter: 543\n",
      "train loss 1.9184731245040894 train acc 0.5072463768115942 train mcc 0.10694747536858147 epoch: 0 iter: 576\n",
      "test loss 1.4394054412841797 test acc 0.6 test mcc 0.0 epoch: 0 iter: 645\n",
      "train loss 1.9721965789794922 train acc 0.4854368932038835 train mcc 0.001946474093603066 epoch: 0 iter: 660\n",
      "train loss 1.672438144683838 train acc 0.34210526315789475 train mcc -0.31241556593546993 epoch: 0 iter: 763\n",
      "train loss 1.386427402496338 train acc 0.7777777777777778 train mcc 0.0 epoch: 0 iter: 801\n",
      "test loss 2.787534236907959 test acc 0.5333333333333333 test mcc 0.0 epoch: 0 iter: 810\n",
      "train loss 2.134219169616699 train acc 0.5975609756097561 train mcc 0.09969134130438027 epoch: 0 iter: 825\n",
      "train loss 2.2801878452301025 train acc 0.36538461538461536 train mcc -0.08070002078437537 epoch: 0 iter: 907\n",
      "train loss 1.199530005455017 train acc 0.4375 train mcc -0.034815531191139566 epoch: 0 iter: 959\n",
      "test loss 1.8684993982315063 test acc 0.4666666666666667 test mcc 0.0 epoch: 0 iter: 975\n",
      "train loss 1.1324806213378906 train acc 0.6379310344827587 train mcc 0.18508635975354779 epoch: 0 iter: 990\n",
      "train loss 3.1102707386016846 train acc 0.4117647058823529 train mcc 0.13696356726888637 epoch: 0 iter: 1106\n",
      "test loss 2.4334630966186523 test acc 0.6 test mcc 0.0 epoch: 0 iter: 1140\n",
      "train loss 1.6747395992279053 train acc 0.55 train mcc -0.08478078067985947 epoch: 0 iter: 1155\n",
      "train loss 1.7533237934112549 train acc 0.4098360655737705 train mcc -0.040453268641197686 epoch: 0 iter: 1215\n",
      "train loss 1.1076816320419312 train acc 0.5172413793103449 train mcc -0.048615279266412334 epoch: 0 iter: 1276\n",
      "test loss 2.386260986328125 test acc 0.26666666666666666 test mcc 0.0 epoch: 0 iter: 1305\n",
      "train loss 2.277146339416504 train acc 0.45121951219512196 train mcc -0.010622167009578043 epoch: 0 iter: 1320\n",
      "train loss 1.7137514352798462 train acc 0.5 train mcc 0.06956740755887705 epoch: 0 iter: 1402\n",
      "test loss 0.9151223301887512 test acc 0.4 test mcc -0.16666666666666666 epoch: 0 iter: 1470\n",
      "train loss 1.3642648458480835 train acc 0.5378151260504201 train mcc 0.05774100531289517 epoch: 0 iter: 1485\n",
      "train loss 1.3875824213027954 train acc 0.4838709677419355 train mcc -0.029288959306450306 epoch: 0 iter: 1604\n",
      "test loss 2.9057376384735107 test acc 0.26666666666666666 test mcc 0.0 epoch: 0 iter: 1635\n",
      "train loss 1.5839838981628418 train acc 0.49206349206349204 train mcc 0.0456520872472285 epoch: 0 iter: 1650\n",
      "train loss 0.9902177453041077 train acc 0.5416666666666666 train mcc 0.05913123959890826 epoch: 0 iter: 1776\n",
      "test loss 0.8100066781044006 test acc 0.6 test mcc -0.23652495839563303 epoch: 0 iter: 1800\n",
      "train loss 1.5828659534454346 train acc 0.5111111111111111 train mcc 0.00871842563632695 epoch: 0 iter: 1815\n",
      "train loss 1.5187374353408813 train acc 0.3902439024390244 train mcc -0.15897435897435896 epoch: 0 iter: 1860\n",
      "train loss 1.0839040279388428 train acc 0.5641025641025641 train mcc 0.018569533817705187 epoch: 0 iter: 1901\n",
      "train loss 1.4084945917129517 train acc 0.52 train mcc -0.040291148201269014 epoch: 0 iter: 1940\n",
      "test loss 0.7944287061691284 test acc 0.5333333333333333 test mcc 0.25 epoch: 0 iter: 1965\n",
      "train loss 1.4772238731384277 train acc 0.46236559139784944 train mcc -0.052088178069247565 epoch: 0 iter: 1980\n",
      "train loss 1.1978849172592163 train acc 0.5740740740740741 train mcc 0.13717366518418428 epoch: 0 iter: 2073\n",
      "train loss 1.7557188272476196 train acc 0.3333333333333333 train mcc 0.0 epoch: 0 iter: 2127\n",
      "test loss 1.987249732017517 test acc 0.4 test mcc 0.0 epoch: 0 iter: 2130\n",
      "train loss 1.0035350322723389 train acc 0.48484848484848486 train mcc -0.12676842711697725 epoch: 0 iter: 2145\n",
      "train loss 0.6867074370384216 train acc 0.65625 train mcc 0.22179547112466663 epoch: 0 iter: 2178\n",
      "train loss 1.4376935958862305 train acc 0.49411764705882355 train mcc 0.07962362179157415 epoch: 0 iter: 2210\n",
      "test loss 0.5441754460334778 test acc 0.7333333333333333 test mcc 0.35355339059327373 epoch: 0 iter: 2295\n",
      "train loss 1.358499526977539 train acc 0.4690265486725664 train mcc -0.06946146394368648 epoch: 0 iter: 2310\n",
      "train loss 1.0653499364852905 train acc 0.5945945945945946 train mcc 0.13348476249438293 epoch: 0 iter: 2423\n",
      "test loss 0.38021668791770935 test acc 0.8666666666666667 test mcc 0.0 epoch: 0 iter: 2460\n",
      "train loss 1.0902693271636963 train acc 0.6333333333333333 train mcc 0.2019871588275667 epoch: 0 iter: 2475\n",
      "train loss 1.3497110605239868 train acc 0.5 train mcc 0.0564782494724905 epoch: 0 iter: 2595\n",
      "test loss 0.6263039112091064 test acc 0.6666666666666666 test mcc 0.46770717334674267 epoch: 0 iter: 2625\n",
      "train loss 1.2702642679214478 train acc 0.5189873417721519 train mcc 0.004667600280093366 epoch: 0 iter: 2640\n",
      "train loss 1.056474208831787 train acc 0.5492957746478874 train mcc 0.07904049358111372 epoch: 0 iter: 2719\n",
      "test loss 1.288007378578186 test acc 0.6 test mcc 0.0 epoch: 0 iter: 2790\n",
      "train loss 1.4470139741897583 train acc 0.5172413793103449 train mcc 0.1180480411624714 epoch: 0 iter: 2805\n",
      "best_result_epoch 0\n"
     ]
    }
   ],
   "source": [
    "train_loss,train_acc,train_mcc,dev_loss,dev_acc,dev_mcc,best_result,y_true,best_attns=test(model=clf,x=x,y=y,train_size=150,test_size=15,epoch=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf=AttentionClassifier(attn_model=rnn_model,d_x=x[0].shape[-1],d_h=300,d_o=class_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss 1.4858139753341675 train acc 0.4854368932038835 train mcc -0.009114591746350924 epoch: 0 iter: 0\n",
      "train loss 0.9853986501693726 train acc 0.5714285714285714 train mcc 0.1877669040497027 epoch: 0 iter: 103\n",
      "train loss 2.1266348361968994 train acc 0.4166666666666667 train mcc -0.1690308509457033 epoch: 0 iter: 138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/mnt/miniconda3/envs/py36/lib/python3.6/site-packages/sklearn/metrics/classification.py:543: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test loss 1.4121356010437012 test acc 0.3333333333333333 test mcc 0.0 epoch: 0 iter: 150\n",
      "train loss 1.3618730306625366 train acc 0.49056603773584906 train mcc -0.04227477114349428 epoch: 0 iter: 165\n",
      "train loss 1.6150058507919312 train acc 0.4567901234567901 train mcc -0.09182622582426504 epoch: 0 iter: 218\n",
      "train loss 1.9375121593475342 train acc 0.3125 train mcc -0.28941438070815306 epoch: 0 iter: 299\n",
      "test loss 2.0551984310150146 test acc 0.4666666666666667 test mcc 0.0 epoch: 0 iter: 315\n",
      "train loss 1.2440690994262695 train acc 0.45714285714285713 train mcc -0.0993399267798783 epoch: 0 iter: 330\n",
      "train loss 1.3825035095214844 train acc 0.5 train mcc 0.010101010101010102 epoch: 0 iter: 365\n",
      "train loss 1.539506435394287 train acc 0.6 train mcc 0.2004459314343183 epoch: 0 iter: 465\n",
      "test loss 0.8569240570068359 test acc 0.6 test mcc 0.0 epoch: 0 iter: 480\n",
      "train loss 1.4350095987319946 train acc 0.5066666666666667 train mcc -0.020145574100634507 epoch: 0 iter: 495\n",
      "train loss 1.3407212495803833 train acc 0.4666666666666667 train mcc -0.014413242771443622 epoch: 0 iter: 570\n",
      "test loss 1.0644251108169556 test acc 0.4 test mcc 0.0 epoch: 0 iter: 645\n",
      "train loss 1.4648407697677612 train acc 0.4827586206896552 train mcc -0.03333333333333333 epoch: 0 iter: 660\n",
      "train loss 1.5863332748413086 train acc 0.42857142857142855 train mcc -0.14871794871794872 epoch: 0 iter: 718\n",
      "train loss 1.8826993703842163 train acc 0.125 train mcc -0.6546536707079772 epoch: 0 iter: 802\n",
      "test loss 2.7779219150543213 test acc 0.5333333333333333 test mcc 0.0 epoch: 0 iter: 810\n",
      "train loss 1.6679909229278564 train acc 0.5068493150684932 train mcc -0.05318978854080616 epoch: 0 iter: 825\n",
      "train loss 1.6457487344741821 train acc 0.42857142857142855 train mcc -0.06201736729460423 epoch: 0 iter: 898\n",
      "test loss 1.6968709230422974 test acc 0.5333333333333333 test mcc 0.0 epoch: 0 iter: 975\n",
      "train loss 1.8262215852737427 train acc 0.3829787234042553 train mcc 0.004780477385098126 epoch: 0 iter: 990\n",
      "train loss 1.5630031824111938 train acc 0.4375 train mcc -0.1239671297859498 epoch: 0 iter: 1037\n",
      "train loss 2.1947004795074463 train acc 0.2857142857142857 train mcc -0.47140452079103173 epoch: 0 iter: 1133\n",
      "test loss 2.253809690475464 test acc 0.6 test mcc 0.0 epoch: 0 iter: 1140\n",
      "train loss 1.3657170534133911 train acc 0.5614035087719298 train mcc -0.04681408379492694 epoch: 0 iter: 1155\n",
      "train loss 1.6734322309494019 train acc 0.4731182795698925 train mcc 0.14935379986151404 epoch: 0 iter: 1212\n",
      "test loss 1.1925694942474365 test acc 0.7333333333333333 test mcc 0.0 epoch: 0 iter: 1305\n",
      "train loss 1.3676681518554688 train acc 0.38461538461538464 train mcc -0.21957751641341997 epoch: 0 iter: 1320\n",
      "train loss 0.924824595451355 train acc 0.6206896551724138 train mcc 0.18045073225950925 epoch: 0 iter: 1359\n",
      "train loss 1.690972924232483 train acc 0.4716981132075472 train mcc -0.0477609253551835 epoch: 0 iter: 1417\n",
      "test loss 0.7517318725585938 test acc 0.4 test mcc 0.0 epoch: 0 iter: 1470\n",
      "train loss 1.5227952003479004 train acc 0.5370370370370371 train mcc -0.06091449038731727 epoch: 0 iter: 1485\n",
      "train loss 1.5419330596923828 train acc 0.4634146341463415 train mcc -0.16169387039666994 epoch: 0 iter: 1539\n",
      "train loss 1.264662742614746 train acc 0.45454545454545453 train mcc -0.0759405848796268 epoch: 0 iter: 1580\n",
      "test loss 1.8694065809249878 test acc 0.26666666666666666 test mcc 0.0 epoch: 0 iter: 1635\n",
      "train loss 1.0844148397445679 train acc 0.509090909090909 train mcc 0.0013924713751557241 epoch: 0 iter: 1650\n",
      "train loss 1.126224160194397 train acc 0.4915254237288136 train mcc 0.011834526708278772 epoch: 0 iter: 1705\n",
      "train loss 1.4133925437927246 train acc 0.3888888888888889 train mcc -0.08770580193070292 epoch: 0 iter: 1764\n",
      "test loss 1.0226911306381226 test acc 0.7333333333333333 test mcc 0.0 epoch: 0 iter: 1800\n",
      "train loss 1.125290870666504 train acc 0.5617977528089888 train mcc 0.15601098655571033 epoch: 0 iter: 1815\n",
      "train loss 1.1857444047927856 train acc 0.45901639344262296 train mcc -0.051111111111111114 epoch: 0 iter: 1904\n",
      "test loss 0.7800131440162659 test acc 0.5333333333333333 test mcc 0.0 epoch: 0 iter: 1965\n",
      "train loss 1.4310609102249146 train acc 0.44285714285714284 train mcc -0.11617340706626102 epoch: 0 iter: 1980\n",
      "train loss 1.0958698987960815 train acc 0.45 train mcc -0.08818077954471167 epoch: 0 iter: 2050\n",
      "train loss 1.276703119277954 train acc 0.45 train mcc -0.0657951694959769 epoch: 0 iter: 2090\n",
      "test loss 1.0037786960601807 test acc 0.4 test mcc 0.0 epoch: 0 iter: 2130\n",
      "train loss 1.2390087842941284 train acc 0.4090909090909091 train mcc -0.11043152607484655 epoch: 0 iter: 2145\n",
      "train loss 0.8359387516975403 train acc 0.6190476190476191 train mcc 0.22106231741762167 epoch: 0 iter: 2211\n",
      "train loss 1.3474851846694946 train acc 0.42857142857142855 train mcc -0.0774455652174615 epoch: 0 iter: 2253\n",
      "test loss 2.3139145374298096 test acc 0.3333333333333333 test mcc 0.0 epoch: 0 iter: 2295\n",
      "train loss 1.000035047531128 train acc 0.5319148936170213 train mcc -0.06950526718618025 epoch: 0 iter: 2310\n",
      "train loss 1.363702416419983 train acc 0.44660194174757284 train mcc -0.07880310495387882 epoch: 0 iter: 2357\n",
      "test loss 2.5246341228485107 test acc 0.13333333333333333 test mcc 0.0 epoch: 0 iter: 2460\n",
      "train loss 1.050820231437683 train acc 0.5263157894736842 train mcc 0.1264304343560434 epoch: 0 iter: 2475\n",
      "train loss 1.0624088048934937 train acc 0.5161290322580645 train mcc 0.029288959306450306 epoch: 0 iter: 2532\n",
      "test loss 0.7470861077308655 test acc 0.4666666666666667 test mcc 0.0 epoch: 0 iter: 2625\n",
      "train loss 1.3297802209854126 train acc 0.45161290322580644 train mcc 0.005986071054452994 epoch: 0 iter: 2640\n",
      "train loss 0.9729106426239014 train acc 0.5769230769230769 train mcc 0.10379985922153641 epoch: 0 iter: 2764\n",
      "test loss 0.6742067933082581 test acc 0.6 test mcc 0.0 epoch: 0 iter: 2790\n",
      "train loss 1.014865756034851 train acc 0.5862068965517241 train mcc 0.14470719035407653 epoch: 0 iter: 2805\n",
      "best_result_epoch 0\n"
     ]
    }
   ],
   "source": [
    "train_loss,train_acc,train_mcc,dev_loss,dev_acc,dev_mcc,best_result,y_true,best_attns=test(model=clf,x=x,y=y,train_size=150,test_size=15,epoch=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}