melanoma-lit-x2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import subprocess\n",
    "\n",
    "PL_PATH = Path(\"/kaggle/input/pytorch-lightning\")\n",
    "subprocess.call(\n",
    "    [\"pip\", \"install\", PL_PATH / \"pytorch_lightning-0.8.1-py3-none-any.whl\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader, random_split, WeightedRandomSampler\n",
    "from PIL import Image\n",
    "import os, random\n",
    "import pytorch_lightning.core.lightning as pl\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from torchvision import transforms, models\n",
    "from pytorch_lightning.metrics.functional import auroc, accuracy\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from pytorch_lightning import Trainer\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from pytorch_lightning.callbacks import *\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "root_train = '../input/melanoma-external-malignant-256/train/train/'\n",
    "root_test = '../input/melanoma-external-malignant-256/test/test/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_cat_variables(x, help_dict = None, N = 100):\n",
    "    uniqs = sorted(np.unique(x).tolist())\n",
    "    if help_dict is None: help_dict = {v:k+1 for k, v in enumerate(uniqs)}\n",
    "    levels = len(help_dict.keys()) + 1\n",
    "    x_t = np.array([help_dict.get(x_i, 0) for x_i in x])\n",
    "    return x_t, help_dict, levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../input/melanoma-external-malignant-256/train_concat.csv')\n",
    "df_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n",
    "\n",
    "df['sex'] = df['sex'].fillna('missing')\n",
    "df_test['sex'] = df_test['sex'].fillna('missing')\n",
    "\n",
    "df['age_missing'] = df['age_approx'].isna()\n",
    "df_test['age_missing'] = df_test['age_approx'].isna()\n",
    "\n",
    "df['age_approx'] = df['age_approx'].fillna(50).astype(np.int8)\n",
    "df_test['age_approx'] = df_test['age_approx'].fillna(50).astype(np.int8)\n",
    "\n",
    "df['anatom_site_general_challenge'] = df['anatom_site_general_challenge'].fillna('missing')\n",
    "df_test['anatom_site_general_challenge'] = df_test['anatom_site_general_challenge'].fillna('missing')\n",
    "\n",
    "df['age_approx'] = (df['age_approx']-df['age_approx'].mean())/df['age_approx'].std()\n",
    "df_test['age_approx'] = (df_test['age_approx']-df_test['age_approx'].mean())/df['age_approx'].std()\n",
    "\n",
    "df['patient_id'] = df['patient_id'].str.split('_', expand=True)[0]\n",
    "df_test['patient_id'] = df_test['patient_id'].str.split('_', expand=True)[0]\n",
    "\n",
    "df['patient_id'] = df['patient_id'].fillna('missing')\n",
    "df_test['patient_id'] = df_test['patient_id'].fillna('missing')\n",
    "\n",
    "cat_cols = ['sex', 'anatom_site_general_challenge', 'age_missing', 'patient_id']\n",
    "df['sex'], sex_dict, _ = encode_cat_variables(df['sex'])\n",
    "df_test['sex'], sex_dict, _ = encode_cat_variables(df_test['sex'])\n",
    "\n",
    "df['anatom_site_general_challenge'], anatomy_dict, _ = encode_cat_variables(df['anatom_site_general_challenge'])\n",
    "df_test['anatom_site_general_challenge'], anatomy_dict, _ = encode_cat_variables(df_test['anatom_site_general_challenge'])\n",
    "\n",
    "df['age_missing'], age_dict, _ = encode_cat_variables(df['age_missing'])\n",
    "df_test['age_missing'], age_dict, _ = encode_cat_variables(df_test['age_missing'])\n",
    "\n",
    "df['patient_id'], id_dict, _ = encode_cat_variables(df['patient_id'].astype('string'))\n",
    "df_test['patient_id'], id_dict, _ = encode_cat_variables(df_test['patient_id'].astype('string'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def open_img(f, root):\n",
    "    return Image.open(os.path.join(root, f+'.jpg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfms = transforms.Compose([\n",
    "                          transforms.ToTensor(),\n",
    "                          transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_c = {col: df[col].nunique()  for col in cat_cols}\n",
    "emb_szs = [(c, min(50, max(6, c**0.5))) for _,c in emb_c.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MelanomaDataset(Dataset):\n",
    "    def __init__(self, df, tfms=tfms, train=True):\n",
    "        self.img_id = df['image_name'].values\n",
    "        self.cat_cols = df[cat_cols].values\n",
    "        self.age = df['age_approx'].astype(np.float32)\n",
    "        self.ys = df['target'] if train else None\n",
    "        self.tfms = tfms\n",
    "        self.train = train\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        root = root_train if self.train else root_test\n",
    "        img = self.tfms(open_img(self.img_id[idx], root))\n",
    "        if self.train:\n",
    "            return img, self.cat_cols[idx], self.age[idx], self.ys[idx]\n",
    "        else:\n",
    "            return img, self.cat_cols[idx], self.age[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.img_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = MelanomaDataset(df)\n",
    "\n",
    "train_ds, valid_ds = random_split(ds, [30118, 7530])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaptiveConcatPool2d(nn.Module):\n",
    "    def __init__(self, sz=1):\n",
    "        super().__init__()\n",
    "        self.output_size = sz\n",
    "        self.ap = nn.AdaptiveAvgPool2d(sz)\n",
    "        self.mp = nn.AdaptiveMaxPool2d(sz)\n",
    "    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n",
    "    \n",
    "class Mish(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x*(torch.tanh(F.softplus(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base():\n",
    "    m = models.resnet34()\n",
    "    m.avgpool = AdaptiveConcatPool2d()\n",
    "    return m\n",
    "\n",
    "class LabelSmoothingLoss(nn.Module):\n",
    "    def __init__(self, smooth_factor=0.05):\n",
    "        super().__init__()\n",
    "        self.smooth_factor = smooth_factor\n",
    "\n",
    "    def _smooth_labels(self, num_classes, target):\n",
    "        target_one_hot = F.one_hot(target, num_classes).float()\n",
    "        target_one_hot[target_one_hot == 1] = 1 - self.smooth_factor\n",
    "        target_one_hot[target_one_hot == 0] = self.smooth_factor\n",
    "        return target_one_hot\n",
    "\n",
    "    def forward(self, input, target):\n",
    "        logp = F.log_softmax(input, dim=1)\n",
    "        target_one_hot = self._smooth_labels(input.size(1), target)\n",
    "        return F.kl_div(logp, target_one_hot, reduction='sum')\n",
    "\n",
    "loss_fn = LabelSmoothingLoss()\n",
    "\n",
    "def calc_score(preds, y):\n",
    "    return roc_auc_score(preds, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(pl.LightningModule):\n",
    "    def __init__(self, emb_szs):\n",
    "        super().__init__()\n",
    "        m = get_base()\n",
    "        self.enc =  nn.Sequential(*list(m.children())[:-1], nn.Flatten())    \n",
    "        nc = list(m.children())[-1].in_features\n",
    "        self.head = nn.Sequential(nn.Linear(2*nc+25,512),Mish(),\n",
    "                                    nn.BatchNorm1d(512), nn.Dropout(0.5),nn.Linear(512,2))\n",
    "        self.embs = nn.ModuleList([nn.Embedding(c, s) for c,s in emb_szs])\n",
    "        \n",
    "    def forward(self, xb, x_cat, x_cont):\n",
    "        x1 = [e(x_cat[:,i]-1) for i,e in enumerate(self.embs)]\n",
    "        x1 = torch.cat(x1, 1)\n",
    "        x_img = self.enc(xb)\n",
    "        x = torch.cat([x1, x_cont.unsqueeze(1)], 1)\n",
    "        x = torch.cat([x, x_img], 1)\n",
    "        return self.head(x)\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        xb, x_cat, x_cont, yb = batch\n",
    "        y_hat = self(xb=xb, x_cat=x_cat, x_cont =x_cont)\n",
    "        loss = loss_fn(y_hat, yb)\n",
    "        tensorboard_logs = {'train_loss': loss}\n",
    "        return {'loss': loss}\n",
    "    \n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        xb, x_cat, x_cont, yb = batch\n",
    "        y_hat = self(xb=xb, x_cat=x_cat, x_cont =x_cont)\n",
    "        loss = loss_fn(y_hat, yb)\n",
    "        return {'valid_loss': loss, 'yb': yb, 'predictions': y_hat}\n",
    "    \n",
    "    def validation_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['valid_loss'] for x in outputs]).mean()\n",
    "        yb = torch.cat([x['yb'] for x in outputs],0)\n",
    "        y_hat = torch.cat([x['predictions'] for x in outputs],0)\n",
    "        preds = torch.argmax(y_hat, dim=1)\n",
    "        acc = accuracy(preds, yb, num_classes=2)\n",
    "        score = calc_score(preds.detach().cpu().numpy(), yb.detach().cpu().numpy()) if preds.shape[0]>50 else 1\n",
    "        print(f'accuracy: {acc}, score: {score}')\n",
    "        return {'val_loss': avg_loss, 'score': accuracy}\n",
    "    \n",
    "    def configure_optimizers(self):\n",
    "        opt = optim.Adam(self.parameters(), lr=0.0001)\n",
    "        scheduler = optim.lr_scheduler.OneCycleLR(opt, max_lr = 5e-4, epochs=10, steps_per_epoch=104)\n",
    "        return [opt], [scheduler]\n",
    "    \n",
    "    def train_dataloader(self):\n",
    "        train_dl = DataLoader(train_ds, pin_memory=True, shuffle=True, batch_size=32, num_workers=4)\n",
    "        return train_dl\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        valid_dl = DataLoader(valid_ds, pin_memory=True, batch_size=64, num_workers=4)\n",
    "        return valid_dl\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        ds = MelanomaDataset(df, train=False)\n",
    "        test_dl = DataLoader(ds, pin_memory=True, batch_size=64, num_workers=4)\n",
    "        return test_dl\n",
    "    \n",
    "    def test_step(self, batch, batch_idx):\n",
    "        xb, x_cat, x_cont = batch\n",
    "        y_hat = self(xb=xb, x_cat= x_cat, x_cont=x_cont)\n",
    "        return {'test_pred_b': y_hat}\n",
    "    \n",
    "    def test_epoch_end(self, outputs):\n",
    "        ys = torch.cat([x['test_pred_b'] for x in outputs], 0)\n",
    "        preds = torch.argmax(ys, dim=1)\n",
    "        return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(emb_szs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(monitor = 'val_loss',\n",
    "                              min_delta = 1e-4,\n",
    "                              patience=3,\n",
    "                              mode= 'min',\n",
    "                              verbose=True)\n",
    "\n",
    "checkpoint_callback = ModelCheckpoint(\n",
    "    filepath=os.path.join(os.getcwd(), 'model_dir'),\n",
    "#     save_top_k=True,\n",
    "    verbose=True,\n",
    "    monitor='val_loss',\n",
    "    mode='min',\n",
    "    prefix=''\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(max_epochs=15, \n",
    "                  early_stop_callback = early_stopping,\n",
    "                  gpus=1,\n",
    "                  gradient_clip_val=1.0,\n",
    "                  weights_save_path=os.getcwd(),\n",
    "                  checkpoint_callback = checkpoint_callback,\n",
    "                  num_sanity_val_steps=0\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  | Name | Type       | Params\n",
      "------------------------------------\n",
      "0 | enc  | Sequential | 21 M  \n",
      "1 | head | Sequential | 539 K \n",
      "2 | embs | ModuleList | 120   \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "698e1f8c5c3248449487756953f909d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef7f22e80e21430baf8f2965cd122867",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss reached 12.34969 (best 12.34969), saving model to /kaggle/working/_ckpt_epoch_1.ckpt as top 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.968393087387085, score: 0.9301843804565458\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6af0cdbe2fc480f8ede64e96800c045",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss reached 3.00059 (best 3.00059), saving model to /kaggle/working/_ckpt_epoch_3.ckpt as top 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9837981462478638, score: 0.988358272025335\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39901222d8b24dcf9a68f56e45ba55ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss  was not in top 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9745019674301147, score: 0.9708491803875549\n"
     ]
    }
   ],
   "source": [
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Error(s) in loading state_dict for Model:\n\tUnexpected key(s) in state_dict: \"embs.0.weight\", \"embs.1.weight\", \"embs.2.weight\", \"embs.3.weight\". ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-24edee734c4e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mtest\u001b[0;34m(self, model, test_dataloaders, ckpt_path)\u001b[0m\n\u001b[1;32m   1165\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mckpt_path\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'best'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1166\u001b[0m                 \u001b[0mckpt_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheckpoint_callback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_model_path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1167\u001b[0;31m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_from_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1169\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtesting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pytorch_lightning/core/saving.py\u001b[0m in \u001b[0;36mload_from_checkpoint\u001b[0;34m(cls, checkpoint_path, map_location, hparams_file, tags_csv, *args, **kwargs)\u001b[0m\n\u001b[1;32m    169\u001b[0m         \u001b[0mcheckpoint\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCHECKPOINT_HYPER_PARAMS_KEY\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_model_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/pytorch_lightning/core/saving.py\u001b[0m in \u001b[0;36m_load_model_state\u001b[0;34m(cls, checkpoint, *args, **kwargs)\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;31m# load the state_dict on the model automatically\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 202\u001b[0;31m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'state_dict'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    204\u001b[0m         \u001b[0;31m# give model a chance to load something\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[0;34m(self, state_dict, strict)\u001b[0m\n\u001b[1;32m    845\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_msgs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m             raise RuntimeError('Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n\u001b[0;32m--> 847\u001b[0;31m                                self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n\u001b[0m\u001b[1;32m    848\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_IncompatibleKeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmissing_keys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munexpected_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Error(s) in loading state_dict for Model:\n\tUnexpected key(s) in state_dict: \"embs.0.weight\", \"embs.1.weight\", \"embs.2.weight\", \"embs.3.weight\". "
     ]
    }
   ],
   "source": [
    "trainer.test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "01b169179a574c2dbf5040f021bbeecc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "0d1da176a26b49758c05e6a3e0eba6f4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": "inline-flex",
       "flex": null,
       "flex_flow": "row wrap",
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": "100%"
      }
     },
     "2c399947bed5465ca9f20ae7fe3e85c1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": "2",
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2cec48dd442849728849421cb67997e9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "3584799d89334c20bb7f83b7a70e5f0b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "info",
       "description": "Validating: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2c399947bed5465ca9f20ae7fe3e85c1",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2cec48dd442849728849421cb67997e9",
       "value": 1.0
      }
     },
     "39901222d8b24dcf9a68f56e45ba55ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_79deac1b23634b5098266630de156362",
        "IPY_MODEL_6d0a9e4aee8d47e7b053ed712af1b012"
       ],
       "layout": "IPY_MODEL_0d1da176a26b49758c05e6a3e0eba6f4"
      }
     },
     "40961ec1d87f4a96a9e3ca8b9bdb0788": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "445ad043906942ed9f7657ac801655a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4aefef8a464d43d1afe4dad949d32303": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bae75370ddc84c1d8e9f68d1bc30b3f2",
       "placeholder": "​",
       "style": "IPY_MODEL_a89eb33f5b264467a2eb64ca624ef014",
       "value": " 118/118 [00:32&lt;00:00,  4.12it/s]"
      }
     },
     "4e684287d12b4363bf410442527dae25": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "info",
       "description": "Epoch 5:   7%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e26db23226984e509ea05c55b458b465",
       "max": 1060.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e7746a99675b4ef79dc68a88a0c78935",
       "value": 69.0
      }
     },
     "55490f68583f4c4e982e873d6cd0db6e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5b4e4fbb82454fa894313643ff8d242b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": "2",
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "698e1f8c5c3248449487756953f909d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4e684287d12b4363bf410442527dae25",
        "IPY_MODEL_7ba31e40363e4ea894212cccf66196d4"
       ],
       "layout": "IPY_MODEL_a5aadbea39a7419284aa8e8ab080abde"
      }
     },
     "6d0a9e4aee8d47e7b053ed712af1b012": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_849216eede694b8b9e7c3c3986c54937",
       "placeholder": "​",
       "style": "IPY_MODEL_e2074ea7f45744c1bf466ddf018ae82a",
       "value": " 118/118 [00:31&lt;00:00,  5.31it/s]"
      }
     },
     "79deac1b23634b5098266630de156362": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "info",
       "description": "Validating: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5b4e4fbb82454fa894313643ff8d242b",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_92eb59cf8848475cb736311170add519",
       "value": 1.0
      }
     },
     "7ba31e40363e4ea894212cccf66196d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_eb7251ebeedf43d1b05a29f84c22029c",
       "placeholder": "​",
       "style": "IPY_MODEL_55490f68583f4c4e982e873d6cd0db6e",
       "value": " 68/1060 [00:13&lt;03:21,  4.93it/s, loss=3.106, v_num=0]"
      }
     },
     "849216eede694b8b9e7c3c3986c54937": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "92eb59cf8848475cb736311170add519": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "a5aadbea39a7419284aa8e8ab080abde": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": "inline-flex",
       "flex": null,
       "flex_flow": "row wrap",
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": "100%"
      }
     },
     "a89eb33f5b264467a2eb64ca624ef014": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b2583dfb5d90403bad0d1bdbd5d5f585": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": "inline-flex",
       "flex": null,
       "flex_flow": "row wrap",
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": "100%"
      }
     },
     "bae75370ddc84c1d8e9f68d1bc30b3f2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c4a1bbdafdf54b7ba2a98463a66ef728": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "info",
       "description": "Validating: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ffc061f9fa8f4201b175a1623aee37ef",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_01b169179a574c2dbf5040f021bbeecc",
       "value": 1.0
      }
     },
     "dad641b23b50452eab39e7f9b0710013": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_40961ec1d87f4a96a9e3ca8b9bdb0788",
       "placeholder": "​",
       "style": "IPY_MODEL_445ad043906942ed9f7657ac801655a4",
       "value": " 118/118 [00:30&lt;00:00,  5.20it/s]"
      }
     },
     "dc01f20e505b4722bbe7283cec455b09": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": "inline-flex",
       "flex": null,
       "flex_flow": "row wrap",
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": "100%"
      }
     },
     "e2074ea7f45744c1bf466ddf018ae82a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e26db23226984e509ea05c55b458b465": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": "2",
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e6af0cdbe2fc480f8ede64e96800c045": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c4a1bbdafdf54b7ba2a98463a66ef728",
        "IPY_MODEL_dad641b23b50452eab39e7f9b0710013"
       ],
       "layout": "IPY_MODEL_dc01f20e505b4722bbe7283cec455b09"
      }
     },
     "e7746a99675b4ef79dc68a88a0c78935": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "eb7251ebeedf43d1b05a29f84c22029c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ef7f22e80e21430baf8f2965cd122867": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_3584799d89334c20bb7f83b7a70e5f0b",
        "IPY_MODEL_4aefef8a464d43d1afe4dad949d32303"
       ],
       "layout": "IPY_MODEL_b2583dfb5d90403bad0d1bdbd5d5f585"
      }
     },
     "ffc061f9fa8f4201b175a1623aee37ef": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": "2",
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}