Generate dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:46:22.394107Z",
     "start_time": "2019-10-23T20:46:22.389153Z"
    }
   },
   "outputs": [],
   "source": [
    "from tt_model import TTModel\n",
    "import torch\n",
    "from torch import nn\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:31.234256Z",
     "start_time": "2019-10-23T20:45:31.221358Z"
    }
   },
   "outputs": [],
   "source": [
    "config = {\n",
    "    'in_factors': (4, 4, 2, 2, 2),\n",
    "    'l1_ranks': (1, 1, 1, 1),\n",
    "    'hidd_out_factors': (2, 2, 2, 2, 2),\n",
    "    'ein_string1': \"nabcde,aoiv,bijw,cjkx,dkly,elpz\",\n",
    "    \n",
    "    'hidd_in_factors': (1, 32),\n",
    "    'l2_ranks': (1,),\n",
    "    'out_factors': (1, 2),\n",
    "    'ein_string2': 'nab,aoix,bipy',\n",
    "}\n",
    "\n",
    "class AttrDict(dict):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super(AttrDict, self).__init__(*args, **kwargs)\n",
    "        self.__dict__ = self\n",
    "        \n",
    "cfg = AttrDict(config)\n",
    "model = TTModel(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:31.965111Z",
     "start_time": "2019-10-23T20:45:31.949478Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TTModel(\n",
       "  (net): Sequential(\n",
       "    (tt0): TTLayer(\n",
       "      (cores): ParameterList(\n",
       "          (0): Parameter containing: [torch.FloatTensor of size 4x1x1x2]\n",
       "          (1): Parameter containing: [torch.FloatTensor of size 4x1x1x2]\n",
       "          (2): Parameter containing: [torch.FloatTensor of size 2x1x1x2]\n",
       "          (3): Parameter containing: [torch.FloatTensor of size 2x1x1x2]\n",
       "          (4): Parameter containing: [torch.FloatTensor of size 2x1x1x2]\n",
       "      )\n",
       "    )\n",
       "    (relu): ReLU()\n",
       "    (tt1): TTLayer(\n",
       "      (cores): ParameterList(\n",
       "          (0): Parameter containing: [torch.FloatTensor of size 1x1x1x1]\n",
       "          (1): Parameter containing: [torch.FloatTensor of size 32x1x1x2]\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:33.864090Z",
     "start_time": "2019-10-23T20:45:33.859965Z"
    }
   },
   "outputs": [],
   "source": [
    "t = torch.randn(1, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:34.367762Z",
     "start_time": "2019-10-23T20:45:34.357346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0345, -0.0638]], grad_fn=<AsStridedBackward>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:37.818055Z",
     "start_time": "2019-10-23T20:45:37.663563Z"
    }
   },
   "outputs": [],
   "source": [
    "torch.random.manual_seed(42)\n",
    "data = torch.randn(100000, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:39.886783Z",
     "start_time": "2019-10-23T20:45:39.526098Z"
    }
   },
   "outputs": [],
   "source": [
    "target = torch.argmax(model(data), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:12:56.952604Z",
     "start_time": "2019-10-23T21:12:56.838963Z"
    }
   },
   "outputs": [],
   "source": [
    "np.savez('synth_data', {'data': data, 'target': target})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:40.151607Z",
     "start_time": "2019-10-23T20:45:40.147576Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 1,  ..., 0, 1, 0])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:47.262371Z",
     "start_time": "2019-10-23T20:45:47.226428Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset = torch.utils.data.TensorDataset(data, target)\n",
    "train_size = int(0.8 * len(dataset))\n",
    "test_size = len(dataset) - train_size\n",
    "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:45:51.145280Z",
     "start_time": "2019-10-23T20:45:51.081375Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[-0.4786, -1.0465,  0.9237,  ...,  0.2253,  1.2762,  0.4853],\n",
       "         [ 0.3496,  0.6459,  0.1231,  ..., -0.5505, -1.4439, -0.9298],\n",
       "         [-0.5105, -0.6921,  3.0666,  ..., -0.4517, -1.4604,  0.6890],\n",
       "         ...,\n",
       "         [ 0.4699, -0.1049,  1.1463,  ..., -0.1402,  2.1803, -0.3842],\n",
       "         [ 0.4893, -0.5197, -2.5331,  ...,  0.9431, -0.3671, -0.5147],\n",
       "         [-0.8304, -1.0810,  1.0682,  ..., -0.5584,  1.3846, -0.0069]]),\n",
       " tensor([0, 0, 1,  ..., 1, 0, 0]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:47:34.921868Z",
     "start_time": "2019-10-23T20:47:34.916668Z"
    }
   },
   "outputs": [],
   "source": [
    "data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T20:50:21.479610Z",
     "start_time": "2019-10-23T20:50:21.475286Z"
    }
   },
   "outputs": [],
   "source": [
    "DEV = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:10:14.671223Z",
     "start_time": "2019-10-23T21:10:14.648813Z"
    }
   },
   "outputs": [],
   "source": [
    "model = nn.Sequential(\n",
    "    nn.Linear(128, 32),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(32, 2),\n",
    "    nn.Softmax(dim=1)\n",
    ").to(DEV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:10:14.901221Z",
     "start_time": "2019-10-23T21:10:14.896946Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:11:37.295940Z",
     "start_time": "2019-10-23T21:10:17.101931Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0.550980325527252 0.46881173029541967 0.858837890625\n",
      "2 0.4463082136242253 0.43861790373921394 0.87314453125\n",
      "3 0.4275215023262486 0.42570673376321794 0.88427734375\n",
      "4 0.413534686443912 0.41188603192567824 0.90009765625\n",
      "5 0.3988949443883957 0.3979099452495575 0.9154296875\n",
      "6 0.38707061216330074 0.38882550224661827 0.92626953125\n",
      "7 0.3795865003470403 0.38352841287851336 0.931640625\n",
      "8 0.3744413862182836 0.38005381971597674 0.9337890625\n",
      "9 0.370511502407159 0.37752060517668723 0.9361328125\n",
      "10 0.36726545433329927 0.37547651380300523 0.93740234375\n",
      "11 0.3644474017771946 0.37378252297639847 0.93935546875\n",
      "12 0.36195406298728505 0.37228023409843447 0.94072265625\n",
      "13 0.3597514502182128 0.37102048099040985 0.941455078125\n",
      "14 0.3577718920768446 0.3698779411613941 0.941796875\n",
      "15 0.3560138191007505 0.36885996013879774 0.94248046875\n",
      "16 0.35443125560784794 0.36797242388129237 0.944287109375\n",
      "17 0.35300500928216677 0.3671853855252266 0.9451171875\n",
      "18 0.3516938851517477 0.36652852296829225 0.94541015625\n",
      "19 0.3504997964497584 0.36592225432395936 0.9458984375\n",
      "20 0.34940238743071345 0.36541820466518404 0.946435546875\n",
      "21 0.34835790221098883 0.364974121004343 0.946142578125\n",
      "22 0.3474026178098788 0.3645111121237278 0.946826171875\n",
      "23 0.3464704209072575 0.3641354776918888 0.94755859375\n",
      "24 0.3456126199026776 0.3638535015285015 0.948095703125\n",
      "25 0.34479939557944134 0.3635639913380146 0.948095703125\n",
      "26 0.3440414610182404 0.36333945766091347 0.9474609375\n",
      "27 0.34331351176948305 0.3631212837994099 0.9470703125\n",
      "28 0.3426224754494467 0.3629018381237984 0.94755859375\n",
      "29 0.34196497338592624 0.36271800100803375 0.947802734375\n",
      "30 0.34133875237149036 0.3624886006116867 0.948291015625\n",
      "31 0.34073630089213136 0.3622972846031189 0.948388671875\n",
      "32 0.34017439632658747 0.3621064983308315 0.948876953125\n",
      "33 0.3396438276691801 0.36196015402674675 0.948876953125\n",
      "34 0.33914064506816255 0.36185803562402724 0.948828125\n",
      "35 0.3386695851945573 0.36171861216425893 0.949169921875\n",
      "36 0.3382239618878456 0.3616117529571056 0.94912109375\n",
      "37 0.3377901524495167 0.36151635348796846 0.949658203125\n",
      "38 0.3373900891109637 0.36139376536011697 0.94970703125\n",
      "39 0.3369944486648414 0.3613170400261879 0.94912109375\n",
      "40 0.3366352312124459 0.3611882947385311 0.949609375\n",
      "41 0.3362719167949288 0.36109588369727136 0.94970703125\n",
      "42 0.33593896771692167 0.36097878888249396 0.950048828125\n",
      "43 0.33561764553094364 0.36085010766983033 0.94990234375\n",
      "44 0.33530178115626047 0.3607689872384071 0.94970703125\n",
      "45 0.33500281014260214 0.36064341068267824 0.949853515625\n",
      "46 0.33470589378077514 0.36059684827923777 0.949951171875\n",
      "47 0.33443087567189694 0.3605394333600998 0.949951171875\n",
      "48 0.33414849126414886 0.3604609616100788 0.9498046875\n",
      "49 0.3338674648552184 0.36038086339831354 0.94990234375\n",
      "50 0.33360014409775945 0.3603563211858273 0.949755859375\n",
      "51 0.3333322318496218 0.36030329018831253 0.949853515625\n",
      "52 0.3330827569885618 0.36031920909881593 0.94970703125\n",
      "53 0.3328438298717426 0.36030672788619994 0.94970703125\n",
      "54 0.33261494860527624 0.3602975785732269 0.9498046875\n",
      "55 0.33238611574385574 0.3602850526571274 0.9505859375\n",
      "56 0.3321798054655646 0.36025536954402926 0.95048828125\n",
      "57 0.3319717049598694 0.36026459783315656 0.95068359375\n",
      "58 0.33175602042750946 0.3602270007133484 0.9505859375\n",
      "59 0.3315333834119663 0.36028549894690515 0.950341796875\n",
      "60 0.3313164291488137 0.3603090688586235 0.95029296875\n",
      "61 0.3311094848593329 0.36032288521528244 0.949951171875\n",
      "62 0.33089319080304186 0.36031804606318474 0.950048828125\n",
      "63 0.33068787235363273 0.3603065811097622 0.9505859375\n",
      "64 0.3304984375929377 0.36035130396485326 0.949609375\n",
      "65 0.3303115626049649 0.3603408269584179 0.949658203125\n",
      "66 0.3301276234304829 0.36034243628382684 0.949365234375\n",
      "67 0.32994707431762843 0.3603819914162159 0.949169921875\n",
      "68 0.32978093927832924 0.3603587359189987 0.94931640625\n",
      "69 0.3296058400041738 0.3603920333087444 0.94951171875\n",
      "70 0.32945870214207157 0.3603865303099155 0.949462890625\n",
      "71 0.3292920640699423 0.360354545712471 0.949560546875\n",
      "72 0.32914054488680167 0.360395335406065 0.949658203125\n",
      "73 0.3289880382407243 0.36039345264434813 0.949560546875\n",
      "74 0.3288581090368283 0.36039004400372504 0.94970703125\n",
      "75 0.3287033212792342 0.36041702777147294 0.949853515625\n",
      "76 0.32857108154114645 0.3604170553386211 0.950390625\n",
      "77 0.32842716745510225 0.360510578751564 0.95048828125\n",
      "78 0.328304745778916 0.3604396604001522 0.95048828125\n",
      "79 0.32819783194049906 0.36045927852392196 0.950537109375\n",
      "80 0.3280731655989483 0.3604750372469425 0.950537109375\n",
      "81 0.32796405550021274 0.36045742109417916 0.9505859375\n",
      "82 0.3278471106176923 0.3604862757027149 0.950341796875\n",
      "83 0.32773574883011497 0.36049275398254393 0.95009765625\n",
      "84 0.32762878089194086 0.3604888528585434 0.9501953125\n",
      "85 0.3275096581620016 0.3605507232248783 0.9501953125\n",
      "86 0.32740185025391305 0.3605042859911919 0.95029296875\n",
      "87 0.32728792945290824 0.36057077050209047 0.950244140625\n",
      "88 0.3271763227927457 0.360631849616766 0.950341796875\n",
      "89 0.3270770495484589 0.36061261147260665 0.95068359375\n",
      "90 0.3269529522983891 0.3606127381324768 0.950537109375\n",
      "91 0.32684438452599157 0.36058808490633965 0.95078125\n",
      "92 0.32673875142814246 0.36060835868120195 0.95068359375\n",
      "93 0.3266316654196211 0.36057779714465144 0.95078125\n",
      "94 0.32652781438675654 0.36060992181301116 0.950634765625\n",
      "95 0.32643441580663063 0.36060169115662577 0.950634765625\n",
      "96 0.3263413268289748 0.3606003329157829 0.95048828125\n",
      "97 0.3262564268461458 0.3605898581445217 0.9505859375\n",
      "98 0.3261590848682792 0.3605645149946213 0.950634765625\n",
      "99 0.32607422721613744 0.3606032893061638 0.950390625\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 100):\n",
    "    model.train()\n",
    "    sum_ = 0\n",
    "    count_ = 0\n",
    "    for X, y in data_loader:\n",
    "        out = model(X.to(DEV))\n",
    "        loss = torch.nn.functional.cross_entropy(out, y.to(DEV))\n",
    "        sum_ += loss.item()\n",
    "        count_ += 1\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    model.eval()\n",
    "    sum1_ = 0\n",
    "    count1_ = 0\n",
    "    sum2_ = 0\n",
    "    count2_ = 0\n",
    "    for X, y in test_loader:\n",
    "        out = model(X.to(DEV))\n",
    "        bin_out = torch.argmax(out, axis=1)\n",
    "        sum2_ += (bin_out == y.to(DEV)).sum().item() / X.shape[0]\n",
    "        count2_ += 1\n",
    "        sum1_ += torch.nn.functional.cross_entropy(out, y.to(DEV)).item()\n",
    "        count1_ += 1\n",
    "    print(epoch, sum_ / count_, sum1_ / count1_, sum2_ / count2_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:42:36.977900Z",
     "start_time": "2019-10-23T21:42:36.956644Z"
    }
   },
   "outputs": [],
   "source": [
    "model = TTModel(cfg).to(DEV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:42:43.730131Z",
     "start_time": "2019-10-23T21:42:43.725280Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T21:45:14.714009Z",
     "start_time": "2019-10-23T21:42:53.290508Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0.6887694156853257 0.656994016468525 0.664013671875\n",
      "2 0.6059344893048524 0.5519380688667297 0.70966796875\n",
      "3 0.4952477258481797 0.45318569913506507 0.773486328125\n",
      "4 0.435436566923834 0.4163321189582348 0.79853515625\n",
      "5 0.3749590307284313 0.3254692979156971 0.850537109375\n",
      "6 0.2577802103226352 0.1991395901888609 0.912451171875\n",
      "7 0.16380377379572317 0.14377107061445712 0.938134765625\n",
      "8 0.12663134899298856 0.11694768136367202 0.949267578125\n",
      "9 0.1045853113577624 0.09737274991348385 0.9587890625\n",
      "10 0.0873448651544987 0.08135504191741347 0.96484375\n",
      "11 0.07248587638234637 0.067342540435493 0.9712890625\n",
      "12 0.059311243628335605 0.055155406100675465 0.977099609375\n",
      "13 0.048016602944606426 0.04515776135958731 0.981884765625\n",
      "14 0.03866149792981565 0.036880869837477806 0.98505859375\n",
      "15 0.03129530260280059 0.030392512492835522 0.988232421875\n",
      "16 0.025665058167117418 0.025487600197084247 0.990771484375\n",
      "17 0.021435917580535837 0.021896723099052905 0.992724609375\n",
      "18 0.018293157463098408 0.01914860587567091 0.9939453125\n",
      "19 0.015972694239703714 0.017037848010659218 0.994775390625\n",
      "20 0.014239972089506258 0.015408930950798094 0.995751953125\n",
      "21 0.012929004353061792 0.014127571519929915 0.995947265625\n",
      "22 0.011923369156659409 0.013151919387746602 0.996240234375\n",
      "23 0.01114394032592132 0.012372865504585207 0.99638671875\n",
      "24 0.010520223717970453 0.01174623896367848 0.996630859375\n",
      "25 0.010003786184083504 0.011220276530366391 0.996728515625\n",
      "26 0.009569235940932468 0.010778350662440062 0.996630859375\n",
      "27 0.0091944192046786 0.010393971181474625 0.9966796875\n",
      "28 0.008866391439809445 0.0100569756294135 0.996630859375\n",
      "29 0.008577706372353491 0.00975859880563803 0.996728515625\n",
      "30 0.008318022884143765 0.009491203364450484 0.996728515625\n",
      "31 0.008084484631088889 0.009250078280456365 0.996728515625\n",
      "32 0.007872345878084185 0.009030527749564499 0.996728515625\n",
      "33 0.0076810534951532155 0.00882364969002083 0.996728515625\n",
      "34 0.007500449898445113 0.008629108231980353 0.996728515625\n",
      "35 0.0073384398262307145 0.008461627474753186 0.996728515625\n",
      "36 0.007189639172202368 0.008309375337557868 0.996826171875\n",
      "37 0.0070538974958798215 0.008169875148450956 0.996826171875\n",
      "38 0.00692739746693498 0.00804104219423607 0.996875\n",
      "39 0.006809846199248105 0.007920763839501887 0.996875\n",
      "40 0.006700770113065507 0.00780742228962481 0.996826171875\n",
      "41 0.006599565738794293 0.007766140945022926 0.996826171875\n",
      "42 0.006515856414060494 0.0076585806062212216 0.996826171875\n",
      "43 0.0064209874861749115 0.007576435909140855 0.996826171875\n",
      "44 0.006338771995435807 0.007495224173180759 0.996826171875\n",
      "45 0.006263139519015932 0.007396828426863067 0.996875\n",
      "46 0.006184634295066783 0.007330693065887317 0.996875\n",
      "47 0.006116897298562916 0.007266718434402719 0.996923828125\n",
      "48 0.0060520046435082984 0.007217238435987383 0.996875\n",
      "49 0.0059919568856845904 0.007161330006783828 0.996826171875\n",
      "50 0.0059350351298785515 0.007108551444252953 0.99677734375\n",
      "51 0.005877878545089418 0.007011260360013693 0.996826171875\n",
      "52 0.005817729567480125 0.006962586287409067 0.996826171875\n",
      "53 0.005770968751548819 0.0069179168494883925 0.996875\n",
      "54 0.005726917575641422 0.006876623019343242 0.996826171875\n",
      "55 0.00568540649678392 0.006837183592142537 0.996826171875\n",
      "56 0.005646064583135021 0.006800080278480891 0.99677734375\n",
      "57 0.005608920568517249 0.006765080546028912 0.99677734375\n",
      "58 0.00557375370352796 0.006729417599854059 0.99677734375\n",
      "59 0.005539935594572905 0.006697363195416983 0.99677734375\n",
      "60 0.0055086000846212456 0.006667905002541374 0.99677734375\n",
      "61 0.0054790864870616586 0.006639532299595885 0.99677734375\n",
      "62 0.005451160355524462 0.006612718792166561 0.99677734375\n",
      "63 0.005425422201022078 0.0066060612589353696 0.99677734375\n",
      "64 0.005402047365049648 0.006582547223661095 0.99677734375\n",
      "65 0.005378850514757311 0.006558856439369265 0.99677734375\n",
      "66 0.005357499379712116 0.006537965792813338 0.996826171875\n",
      "67 0.005336803102355664 0.006516744031978305 0.996826171875\n",
      "68 0.005315403557815559 0.006490076833870262 0.996826171875\n",
      "69 0.005296225628035535 0.006472386677342001 0.996826171875\n",
      "70 0.00527843100887836 0.006453431626141537 0.996826171875\n",
      "71 0.005261578934510375 0.006435279213474132 0.996826171875\n",
      "72 0.0052456484282994344 0.006417604940361343 0.996826171875\n",
      "73 0.005230519166483194 0.006400557977030985 0.996826171875\n",
      "74 0.00521607933029129 0.0063838224014034495 0.996826171875\n",
      "75 0.0052023509333420336 0.006367862717888783 0.996826171875\n",
      "76 0.005189374608660983 0.006352216649975162 0.996875\n",
      "77 0.005177009134823278 0.006337104526755866 0.996875\n",
      "78 0.005165276268517871 0.00632258558034664 0.996875\n",
      "79 0.0051541280566424034 0.006308216710749548 0.996875\n",
      "80 0.005141844463187632 0.006300898191693704 0.996875\n",
      "81 0.005131939899511517 0.006277136066637467 0.996875\n",
      "82 0.005123221557913645 0.0062674393513589164 0.996875\n",
      "83 0.005113184170287316 0.00626124746631831 0.996875\n",
      "84 0.005104706350530077 0.006240548472851515 0.996875\n",
      "85 0.005095882872178866 0.006237426865845918 0.996875\n",
      "86 0.00508863629868171 0.006217695533996448 0.996875\n",
      "87 0.00508069494593841 0.006215140514541417 0.996923828125\n",
      "88 0.005074163330837515 0.00619931528926827 0.996923828125\n",
      "89 0.0050670658979731 0.00619408042402938 0.996923828125\n",
      "90 0.005059049998641394 0.006180502893403173 0.99697265625\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-55-82977c0dcc33>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mcount_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEV\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEV\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0msum_\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/BM/Project/lrbtnn/tt_model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;31m# ('softmax', nn.Softmax(dim=1))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             ]),)\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m         \u001b[0;31m# self.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/BM/Project/lrbtnn/tt_model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     40\u001b[0m         \u001b[0;31m# o, i, j, k, l, p - indices corresponding to the 4 tensor train ranks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m         \u001b[0;31m# v, w, x, y, z - indices corresponding to o1, o2, o3, o4, o5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m         result = torch.einsum(\n\u001b[1;32m     44\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mein_string\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/torch/lib/python3.7/site-packages/torch/functional.py\u001b[0m in \u001b[0;36meinsum\u001b[0;34m(equation, *operands)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;31m# the old interface of passing the operands as one list argument\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0moperands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moperands\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_VariableFunctions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meinsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mequation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperands\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 100):\n",
    "    model.train()\n",
    "    sum_ = 0\n",
    "    count_ = 0\n",
    "    for X, y in data_loader:\n",
    "        out = model(X.to(DEV))\n",
    "        loss = torch.nn.functional.cross_entropy(out, y.to(DEV))\n",
    "        sum_ += loss.item()\n",
    "        count_ += 1\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    model.eval()\n",
    "    sum1_ = 0\n",
    "    count1_ = 0\n",
    "    sum2_ = 0\n",
    "    count2_ = 0\n",
    "    for X, y in test_loader:\n",
    "        out = model(X.to(DEV))\n",
    "        bin_out = torch.argmax(out, axis=1)\n",
    "        sum2_ += (bin_out == y.to(DEV)).sum().item() / X.shape[0]\n",
    "        count2_ += 1\n",
    "        sum1_ += torch.nn.functional.cross_entropy(out, y.to(DEV)).item()\n",
    "        count1_ += 1\n",
    "    print(epoch, sum_ / count_, sum1_ / count1_, sum2_ / count2_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}