RNN/RNNcell.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.12 64-bit ('pytorch_dl': conda)",
   "metadata": {
    "interpreter": {
     "hash": "6b962ed846ab7a2f9c4286c5c9f447c6ba721d3b496bd62adf19906831909870"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "来源：[b站刘二大人--RNN基础篇](https://www.bilibili.com/video/BV1Y7411d7Ys?p=12)\n",
    "# How to use RNNCell\n",
    "## 注意几个参数\n",
    "1. 输入和隐层（输出）维度\n",
    "2. 序列长度\n",
    "3. 批处理大小\n",
    "\n",
    "- **注 调用RNNCell这个需要循环，循环长度就是序列长度**"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "==================== 0 ====================\nInput size: torch.Size([1, 4]) tensor([[ 1.9129, -0.7440,  0.2329,  1.3065]])\nhidden size: torch.Size([1, 2]) tensor([[-0.0790, -0.8957]], grad_fn=<TanhBackward>)\ntensor([[-0.0790, -0.8957]], grad_fn=<TanhBackward>)\n==================== 1 ====================\nInput size: torch.Size([1, 4]) tensor([[-0.6290, -0.2338, -0.2949,  0.3956]])\nhidden size: torch.Size([1, 2]) tensor([[ 0.0170, -0.0005]], grad_fn=<TanhBackward>)\ntensor([[ 0.0170, -0.0005]], grad_fn=<TanhBackward>)\n==================== 2 ====================\nInput size: torch.Size([1, 4]) tensor([[-0.6959,  1.0590, -0.6798,  0.6989]])\nhidden size: torch.Size([1, 2]) tensor([[0.4216, 0.6813]], grad_fn=<TanhBackward>)\ntensor([[0.4216, 0.6813]], grad_fn=<TanhBackward>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "batch_size = 1  # 批处理大小\n",
    "seq_len = 3     # 序列长度\n",
    "input_size = 4  # 输入维度\n",
    "hidden_size = 2 # 隐层维度\n",
    "\n",
    "cell = torch.nn.RNNCell(input_size=input_size, hidden_size=hidden_size)\n",
    "\n",
    "# (seq, batch, features)\n",
    "dataset = torch.randn(seq_len, batch_size, input_size)\n",
    "hidden = torch.zeros(batch_size, hidden_size)\n",
    "\n",
    "# 这个循环就是处理seq_len长度的数据\n",
    "for idx, data in enumerate(dataset):\n",
    "    print('=' * 20, idx, '=' * 20)\n",
    "    print('Input size:', data.shape, data)\n",
    "\n",
    "    hidden = cell(data, hidden)\n",
    "\n",
    "    print('hidden size:', hidden.shape, hidden)\n",
    "    print(hidden)"
   ]
  },
  {
   "source": [
    "# How to use RNN\n",
    "## 确定几个参数\n",
    "1. input_size和hidden_size: 输入维度和隐层维度\n",
    "2. batch_size: 批处理大小\n",
    "3. seq_len: 序列长度\n",
    "4. num_layers: 隐层数目\n",
    "\n",
    "- **注 直接调用RNN这个不用循环**\n",
    "- **注：如果使用batch_first: if True, the input and output tensors are provided as:(batch_size, seq_len, input_size)**"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "import torch\n",
    "\n",
    "batch_size = 1\n",
    "seq_len = 3\n",
    "input_size = 4\n",
    "hidden_size = 2\n",
    "num_layers = 1\n",
    "\n",
    "cell = torch.nn.RNN(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers)\n",
    "\n",
    "# (seqLen, batchSize, inputSize)\n",
    "inputs = torch.randn(seq_len, batch_size, input_size)\n",
    "hidden = torch.zeros(num_layers, batch_size, hidden_size)\n",
    "\n",
    "out, hidden = cell(inputs, hidden)\n",
    "\n",
    "print('Output size:', out.shape)        # (seq_len, batch_size, hidden_size)\n",
    "print('Output:', out)\n",
    "print('Hidden size:', hidden.shape)     # (num_layers, batch_size, hidden_size)\n",
    "print('Hidden:', hidden)"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 23,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Output size: torch.Size([3, 1, 2])\nOutput: tensor([[[ 0.3689,  0.5982]],\n\n        [[ 0.1233,  0.2617]],\n\n        [[-0.3517, -0.7246]]], grad_fn=<StackBackward>)\nHidden size: torch.Size([1, 1, 2])\nHidden: tensor([[[-0.3517, -0.7246]]], grad_fn=<StackBackward>)\n"
     ]
    }
   ]
  },
  {
   "source": [
    "# Example: Using RNNCell\n",
    "## Hello --> ohlol\n",
    "1. 首先需要将输入的单词转成向量`one-hot vector`\n",
    "2. 注意input_size，如下图\n",
    "\n",
    "![转化成向量](https://s2.loli.net/2022/01/11/DwGu2hUaopHljtY.png)\n",
    "\n",
    "## 注意交叉熵在计算loss的时候维度关系，这里的hidden是`([1, 4])`, label是 `([1])`"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "torch.Size([5, 1, 4]) torch.Size([5, 1])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "input_size = 4\n",
    "hidden_size = 4\n",
    "batch_size = 1\n",
    "\n",
    "idx2char = ['e', 'h', 'l', 'o']\n",
    "x_data = [1, 0, 2, 3, 3]    # hello中各个字符的下标\n",
    "y_data = [3, 1, 2, 3, 2]    # ohlol中各个字符的下标\n",
    "\n",
    "one_hot_lookup = [[1, 0, 0, 0],\n",
    "                  [0, 1, 0, 0],\n",
    "                  [0, 0, 1, 0],\n",
    "                  [0, 0, 0, 1]]\n",
    "x_one_hot = [one_hot_lookup[x] for x in x_data] # (seqLen, inputSize)\n",
    "\n",
    "inputs = torch.Tensor(x_one_hot).view(-1, batch_size, input_size)\n",
    "labels = torch.LongTensor(y_data).view(-1, 1)   # torch.Tensor默认是torch.FloatTensor是32位浮点类型数据，torch.LongTensor是64位整型\n",
    "print(inputs.shape, labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, batch_size):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.rnncell = nn.RNNCell(input_size=self.input_size, hidden_size=self.hidden_size)\n",
    "\n",
    "    def forward(self, inputs, hidden):\n",
    "        hidden = self.rnncell(inputs, hidden)   # (batch_size, hidden_size)\n",
    "        return hidden\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return torch.zeros(self.batch_size, self.hidden_size)\n",
    "\n",
    "net = Model(input_size, hidden_size, batch_size)\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Predicted string:lhlhh, Epoch [1/15] loss=6.8407\nPredicted string:lllll, Epoch [2/15] loss=5.2957\nPredicted string:lllol, Epoch [3/15] loss=4.9344\nPredicted string:lllol, Epoch [4/15] loss=4.7035\nPredicted string:oolol, Epoch [5/15] loss=4.4781\nPredicted string:oolol, Epoch [6/15] loss=4.2419\nPredicted string:ohlol, Epoch [7/15] loss=3.9733\nPredicted string:ohlol, Epoch [8/15] loss=3.6942\nPredicted string:ohlol, Epoch [9/15] loss=3.4917\nPredicted string:ohloo, Epoch [10/15] loss=3.3837\nPredicted string:ohloo, Epoch [11/15] loss=3.2953\nPredicted string:ohlol, Epoch [12/15] loss=3.1331\nPredicted string:ohlol, Epoch [13/15] loss=2.9294\nPredicted string:ohlol, Epoch [14/15] loss=2.7344\nPredicted string:ohlol, Epoch [15/15] loss=2.5680\n"
     ]
    }
   ],
   "source": [
    "epochs = 15\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    loss = 0\n",
    "    optimizer.zero_grad()\n",
    "    hidden = net.init_hidden()\n",
    "    print('Predicted string:', end='')\n",
    "    for input, label in zip(inputs, labels):\n",
    "        hidden = net(input, hidden)\n",
    "        # 注意交叉熵在计算loss的时候维度关系，这里的hidden是([1, 4]), label是 ([1])\n",
    "        loss += criterion(hidden, label)\n",
    "        _, idx = hidden.max(dim = 1)\n",
    "        print(idx2char[idx.item()], end='')\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    print(', Epoch [%d/15] loss=%.4f' % (epoch+1, loss.item()))"
   ]
  },
  {
   "source": [
    "# Example: Using RNN\n",
    "## 注意`inputs`和`labels`的维度\n",
    "- `inputs`维度是: (seqLen, batch_size, input_size)\n",
    "- `labels`维度是: (seqLen * batch_size)\n",
    "\n",
    "## 注意`outputs`维度，对应和`labels`做交叉熵的维度\n",
    "- `outputs`维度是: (seqLen, batch_size, hidden_size)\n",
    "- 为了能和labels做交叉熵，需要reshape一下: outputs.view(-1, hidden_size)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "torch.Size([5, 1, 4]) torch.Size([5])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "input_size = 4\n",
    "hidden_size = 4\n",
    "batch_size = 1\n",
    "seq_len = 5\n",
    "num_layers = 1\n",
    "\n",
    "idx2char = ['e', 'h', 'l', 'o']\n",
    "x_data = [1, 0, 2, 3, 3]    # hello中各个字符的下标\n",
    "y_data = [3, 1, 2, 3, 2]    # ohlol中各个字符的下标\n",
    "\n",
    "one_hot_lookup = [[1, 0, 0, 0],\n",
    "                  [0, 1, 0, 0],\n",
    "                  [0, 0, 1, 0],\n",
    "                  [0, 0, 0, 1]]\n",
    "x_one_hot = [one_hot_lookup[x] for x in x_data] # (seqLen, inputSize)\n",
    "\n",
    "inputs = torch.Tensor(x_one_hot).view(seq_len, batch_size, input_size)\n",
    "labels = torch.LongTensor(y_data)  \n",
    "print(inputs.shape, labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, batch_size, num_layers=1):\n",
    "        super(Model, self).__init__()\n",
    "        self.num_layers = num_layers\n",
    "        self.batch_size = batch_size\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.rnn = nn.RNN(input_size=self.input_size, hidden_size=self.hidden_size, )\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        hidden = torch.zeros(self.num_layers, self.batch_size, self.hidden_size)\n",
    "        out, _ = self.rnn(inputs, hidden)    # 注意维度是(seqLen, batch_size, hidden_size)\n",
    "        return out.view(-1, self.hidden_size) # 为了容易计算交叉熵这里调整维度为(seqLen * batch_size, hidden_size)\n",
    "\n",
    "net = Model(input_size, hidden_size, batch_size)\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Predicted:  ololl, Epoch [1/15] loss = 1.189\nPredicted:  ollll, Epoch [2/15] loss = 1.070\nPredicted:  ollll, Epoch [3/15] loss = 0.976\nPredicted:  ohlll, Epoch [4/15] loss = 0.883\nPredicted:  ohlol, Epoch [5/15] loss = 0.788\nPredicted:  ohlol, Epoch [6/15] loss = 0.715\nPredicted:  ohlol, Epoch [7/15] loss = 0.652\nPredicted:  ohlol, Epoch [8/15] loss = 0.603\nPredicted:  ohlol, Epoch [9/15] loss = 0.570\nPredicted:  ohlol, Epoch [10/15] loss = 0.548\nPredicted:  ohlol, Epoch [11/15] loss = 0.530\nPredicted:  ohlol, Epoch [12/15] loss = 0.511\nPredicted:  ohlol, Epoch [13/15] loss = 0.488\nPredicted:  ohlol, Epoch [14/15] loss = 0.462\nPredicted:  ohlol, Epoch [15/15] loss = 0.439\n"
     ]
    }
   ],
   "source": [
    "epochs = 15\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    optimizer.zero_grad()\n",
    "    outputs = net(inputs) \n",
    "    # print(outputs.shape, labels.shape)\n",
    "    # 这里的outputs维度是([seqLen * batch_size, hidden]), labels维度是([seqLen])\n",
    "    loss = criterion(outputs, labels) \n",
    "    loss.backward() \n",
    "    optimizer.step()\n",
    "\n",
    "    _, idx = outputs.max(dim=1) \n",
    "    idx = idx.data.numpy() \n",
    "    print('Predicted: ', ''.join([idx2char[x] for x in idx]), end='') \n",
    "    print(', Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))"
   ]
  },
  {
   "source": [
    "# 将一个单词变成vector\n",
    "## One-hot encoding of words and characters\n",
    "- one-hot vectors high-dimension --> lower-dimension\n",
    "- one-hot vectors sparse --> dense\n",
    "- one-hot vectors hardcoded   --> learn from data\n",
    "\n",
    "## Embedding\n",
    "![ont_hot_vector VS embedding](https://ericpengshuai.github.io/shen-du-xue-xi/ac81e297adc0/ont_hot_VS_embedding.png)\n",
    "![embedding](https://ericpengshuai.github.io/shen-du-xue-xi/ac81e297adc0/embedding.png)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "# parameters\n",
    "num_class = 4 \n",
    "input_size = 4 \n",
    "hidden_size = 8 \n",
    "embedding_size = 10 \n",
    "num_layers = 2 \n",
    "batch_size = 1 \n",
    "seq_len = 5\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.emb = torch.nn.Embedding(input_size, embedding_size)\n",
    "        self.rnn = nn.RNN(input_size=embedding_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, num_class)\n",
    "\n",
    "    def forward(self, x):\n",
    "        hidden = torch.zeros(num_layers, x.size(0), hidden_size)\n",
    "        x = self.emb(x)                 # (batch, seqLen, embeddingSize) \n",
    "        x, _ = self.rnn(x, hidden)      # 输出(𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆, 𝒔𝒆𝒒𝑳𝒆𝒏, hidden_size)\n",
    "        x = self.fc(x)                  # 输出(𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆, 𝒔𝒆𝒒𝑳𝒆𝒏, 𝒏𝒖𝒎𝑪𝒍𝒂𝒔𝒔)\n",
    "        return x.view(-1, num_class)    # reshape to use Cross Entropy: (𝒃𝒂𝒕𝒄𝒉𝑺𝒊𝒛𝒆×𝒔𝒆𝒒𝑳𝒆𝒏, 𝒏𝒖𝒎𝑪𝒍𝒂𝒔𝒔)\n",
    "        \n",
    "net = Model()\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Predicted:  ollll, Epoch [1/15] loss = 1.290\nPredicted:  olooo, Epoch [2/15] loss = 1.071\nPredicted:  ollol, Epoch [3/15] loss = 0.913\nPredicted:  ollol, Epoch [4/15] loss = 0.785\nPredicted:  ollol, Epoch [5/15] loss = 0.660\nPredicted:  ohlol, Epoch [6/15] loss = 0.541\nPredicted:  ohlol, Epoch [7/15] loss = 0.435\nPredicted:  ohlol, Epoch [8/15] loss = 0.343\nPredicted:  ohlol, Epoch [9/15] loss = 0.251\nPredicted:  ohlol, Epoch [10/15] loss = 0.171\nPredicted:  ohlol, Epoch [11/15] loss = 0.121\nPredicted:  ohlol, Epoch [12/15] loss = 0.081\nPredicted:  ohlol, Epoch [13/15] loss = 0.052\nPredicted:  ohlol, Epoch [14/15] loss = 0.036\nPredicted:  ohlol, Epoch [15/15] loss = 0.025\n"
     ]
    }
   ],
   "source": [
    "idx2char = ['e', 'h', 'l', 'o'] \n",
    "x_data = [[1, 0, 2, 2, 3]]  # (batch, seq_len) \n",
    "y_data = [3, 1, 2, 3, 2]    # (batch * seq_len)\n",
    "\n",
    "inputs = torch.LongTensor(x_data)   # Input should be LongTensor: (batchSize, seqLen)\n",
    "labels = torch.LongTensor(y_data)   # Target should be LongTensor: (batchSize * seqLen)\n",
    "\n",
    "epochs = 15\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    optimizer.zero_grad()\n",
    "    outputs = net(inputs) \n",
    "    loss = criterion(outputs, labels) \n",
    "    loss.backward() \n",
    "    optimizer.step()\n",
    "\n",
    "    _, idx = outputs.max(dim=1) \n",
    "    idx = idx.data.numpy() \n",
    "    print('Predicted: ', ''.join([idx2char[x] for x in idx]), end='') \n",
    "    print(', Epoch [%d/15] loss = %.3f' % (epoch + 1, loss.item()))"
   ]
  }
 ]
}