workshop/1a_torch_tensors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9e126499",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "# The package pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "72dd3ff0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36d6bae8",
   "metadata": {},
   "source": [
    "## Tensor manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6aa30a9",
   "metadata": {
    "origin_pos": 14,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x = torch.arange(12.)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "640cadaf",
   "metadata": {
    "origin_pos": 21,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.numel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e0a9616",
   "metadata": {
    "origin_pos": 24,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f9312ce",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Change the shape of a tensor\n",
    "without altering its size or values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6092207c",
   "metadata": {
    "origin_pos": 26,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "A = x.reshape(3, 4)\n",
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc6681ad",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Supplying the exact values for each element"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b26863d8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.153567Z",
     "iopub.status.busy": "2023-08-18T19:32:57.153222Z",
     "iopub.status.idle": "2023-08-18T19:32:57.160436Z",
     "shell.execute_reply": "2023-08-18T19:32:57.159548Z"
    },
    "origin_pos": 45,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "A = torch.tensor([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "255741b1",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "`[-1]` selects the last row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9049a53",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.164537Z",
     "iopub.status.busy": "2023-08-18T19:32:57.163812Z",
     "iopub.status.idle": "2023-08-18T19:32:57.171699Z",
     "shell.execute_reply": "2023-08-18T19:32:57.170451Z"
    },
    "origin_pos": 49,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "A[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a94e2e35",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "`[1:3]` selects the second and third rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a948477",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A[1:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8ecea71",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Apply a function component-wise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5677d82",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "torch.exp(A)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30e6e22b",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Write elements by specifying indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9246619c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.176047Z",
     "iopub.status.busy": "2023-08-18T19:32:57.175685Z",
     "iopub.status.idle": "2023-08-18T19:32:57.182893Z",
     "shell.execute_reply": "2023-08-18T19:32:57.181890Z"
    },
    "origin_pos": 52,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "B = A.clone()\n",
    "B[1, 2] = 17\n",
    "B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8708c166",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "To assign multiple elements the same value,\n",
    "we apply the indexing on the left-hand side \n",
    "of the assignment operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0532f024",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.186970Z",
     "iopub.status.busy": "2023-08-18T19:32:57.186270Z",
     "iopub.status.idle": "2023-08-18T19:32:57.193303Z",
     "shell.execute_reply": "2023-08-18T19:32:57.192338Z"
    },
    "origin_pos": 56,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "B[:2, :] = 12\n",
    "B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4217e631",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "*concatenate* multiple tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43aa9012",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.223534Z",
     "iopub.status.busy": "2023-08-18T19:32:57.222711Z",
     "iopub.status.idle": "2023-08-18T19:32:57.233166Z",
     "shell.execute_reply": "2023-08-18T19:32:57.232145Z"
    },
    "origin_pos": 71,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "torch.cat((A, B), dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f0c2350",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "torch.cat((A, B), dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55bcaa66",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Construct a binary tensor via *logical statements*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91d39e58",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.237276Z",
     "iopub.status.busy": "2023-08-18T19:32:57.236485Z",
     "iopub.status.idle": "2023-08-18T19:32:57.243133Z",
     "shell.execute_reply": "2023-08-18T19:32:57.242117Z"
    },
    "origin_pos": 75,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "A == B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8861ed40",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "**Broadcasting**: Perform elementwise binary operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be37d2de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.256932Z",
     "iopub.status.busy": "2023-08-18T19:32:57.256264Z",
     "iopub.status.idle": "2023-08-18T19:32:57.263823Z",
     "shell.execute_reply": "2023-08-18T19:32:57.262881Z"
    },
    "origin_pos": 81,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "a = torch.arange(3).reshape((3, 1))\n",
    "b = torch.arange(2).reshape((1, 2))\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f62e827",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:32:57.267856Z",
     "iopub.status.busy": "2023-08-18T19:32:57.267172Z",
     "iopub.status.idle": "2023-08-18T19:32:57.273497Z",
     "shell.execute_reply": "2023-08-18T19:32:57.272587Z"
    },
    "origin_pos": 85,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "a + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "51b50749",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "A = torch.arange(6.).reshape(2, 3)\n",
    "B = A"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24bca25d",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Elementwise sum or product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d2fcd62",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A + B, A * B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f96ab73",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Adding or multiplying a scalar and a tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dc555bf",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A + 3, A * 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77b2f30b",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Sums over the elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c95fc36",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c5840eb",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22f845dc",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Specify the axes \n",
    "along which the tensor should be reduced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55994d12",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b516b065",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b97f6b5",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum(axis=[0, 1]) == A.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d4d538d",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Keep the number of axes unchanged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52a3da79",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0984a540",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A.sum(axis=0, keepdim=True).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61440daf",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The *dot product* of two vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66d9cf58",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.arange(3.)\n",
    "y = torch.ones(3, dtype = torch.float32)\n",
    "x, y, torch.dot(x, y), x @ y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c010fb5",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Matrix-vector product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29d1cf2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "A.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73680e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.matmul(A, x), A @ x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef402938",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Matrix product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f4fa10f",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "B = torch.ones(3, 4)\n",
    "torch.matmul(A, B), A @ B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b005fc6",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The $\\ell_2$ *norm*\n",
    "$$\\|\\mathbf{x}\\|_2 = \\sqrt{\\sum_{i=1}^n x_i^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d3080f6",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "u = torch.tensor([3.0, -4.0])\n",
    "torch.norm(u)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48307c32",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "The $\\ell_1$ norm\n",
    "$$\\|\\mathbf{x}\\|_1 = \\sum_{i=1}^n \\left|x_i \\right|$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3ce7fb4",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "torch.abs(u).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bcf3f32",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "The *Frobenius norm*, \n",
    "which is much easier to compute\n",
    "$$\\|\\mathbf{X}\\|_\\textrm{F} = \\sqrt{\\sum_{i=1}^m \\sum_{j=1}^n x_{ij}^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79db6268",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "A = torch.ones((4, 9))\n",
    "torch.norm(A)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2110a2da",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Automatic Differentiation\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "130439cd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:08.286501Z",
     "iopub.status.busy": "2023-08-18T19:26:08.285693Z",
     "iopub.status.idle": "2023-08-18T19:26:10.052257Z",
     "shell.execute_reply": "2023-08-18T19:26:10.050994Z"
    },
    "origin_pos": 3,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1b3f966",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Differentiating the function\n",
    "$y = 2\\mathbf{x}^{\\top}\\mathbf{x}$\n",
    "with respect to the column vector $\\mathbf{x}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4253cfab",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "7"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.056833Z",
     "iopub.status.busy": "2023-08-18T19:26:10.055871Z",
     "iopub.status.idle": "2023-08-18T19:26:10.084858Z",
     "shell.execute_reply": "2023-08-18T19:26:10.083727Z"
    },
    "origin_pos": 8,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x = torch.arange(4.0)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35a0803e",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Before we calculate the gradient\n",
    "of $y$ with respect to $\\mathbf{x}$,\n",
    "we need a place to store it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2a001d1e",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "9"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.088716Z",
     "iopub.status.busy": "2023-08-18T19:26:10.087816Z",
     "iopub.status.idle": "2023-08-18T19:26:10.092878Z",
     "shell.execute_reply": "2023-08-18T19:26:10.091740Z"
    },
    "origin_pos": 13,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.requires_grad_(True)\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34223f79",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "We now calculate our function of `x` and assign the result to `y`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e3bd777",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "11"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.096336Z",
     "iopub.status.busy": "2023-08-18T19:26:10.095772Z",
     "iopub.status.idle": "2023-08-18T19:26:10.105236Z",
     "shell.execute_reply": "2023-08-18T19:26:10.104075Z"
    },
    "origin_pos": 17,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "y = 2 * torch.dot(x, x)\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2f52bd4",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "We can now take the gradient of `y`\n",
    "with respect to `x`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21b134ae",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "12"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.108600Z",
     "iopub.status.busy": "2023-08-18T19:26:10.108011Z",
     "iopub.status.idle": "2023-08-18T19:26:10.160854Z",
     "shell.execute_reply": "2023-08-18T19:26:10.159702Z"
    },
    "origin_pos": 25,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "y.backward()\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74e7747a",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "We already know that the gradient of the function $y = 2\\mathbf{x}^{\\top}\\mathbf{x}$\n",
    "with respect to $\\mathbf{x}$ should be $4\\mathbf{x}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5030e37d",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "14"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.164665Z",
     "iopub.status.busy": "2023-08-18T19:26:10.163930Z",
     "iopub.status.idle": "2023-08-18T19:26:10.171033Z",
     "shell.execute_reply": "2023-08-18T19:26:10.169923Z"
    },
    "origin_pos": 30,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.grad == 4 * x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e19d827",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Now let's calculate \n",
    "another function of `x`\n",
    "and take its gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "add5cf4b",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "20"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.174691Z",
     "iopub.status.busy": "2023-08-18T19:26:10.173957Z",
     "iopub.status.idle": "2023-08-18T19:26:10.181847Z",
     "shell.execute_reply": "2023-08-18T19:26:10.180759Z"
    },
    "origin_pos": 37,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.grad.zero_()\n",
    "y = x.sum()\n",
    "y.backward()\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32049daa",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Sum up the gradients\n",
    "computed individually for each example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1baa40bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.185096Z",
     "iopub.status.busy": "2023-08-18T19:26:10.184685Z",
     "iopub.status.idle": "2023-08-18T19:26:10.192537Z",
     "shell.execute_reply": "2023-08-18T19:26:10.191435Z"
    },
    "origin_pos": 45,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.grad.zero_()\n",
    "y = x * x\n",
    "y.backward(gradient=torch.ones(len(y)))\n",
    "x.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5c51cbd",
   "metadata": {},
   "source": [
    "Note: The argument `gradient` specifies a vector to multiply with the Jacobian matrix."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47c4826",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Move some calculations\n",
    "outside of the recorded computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "107ac041",
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "21"
    },
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.196001Z",
     "iopub.status.busy": "2023-08-18T19:26:10.195456Z",
     "iopub.status.idle": "2023-08-18T19:26:10.203246Z",
     "shell.execute_reply": "2023-08-18T19:26:10.202155Z"
    },
    "origin_pos": 50,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.grad.zero_()\n",
    "y = x * x\n",
    "u = y.detach()\n",
    "z = u * x\n",
    "\n",
    "z.sum().backward()\n",
    "x.grad == u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb8c674b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.206880Z",
     "iopub.status.busy": "2023-08-18T19:26:10.206001Z",
     "iopub.status.idle": "2023-08-18T19:26:10.213592Z",
     "shell.execute_reply": "2023-08-18T19:26:10.212476Z"
    },
    "origin_pos": 55,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "x.grad.zero_()\n",
    "y.sum().backward()\n",
    "x.grad == 2 * x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe357b0b",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Even if\n",
    "a function required passing through a maze of Python control flow\n",
    "we can still calculate the gradient of the resulting variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab14ef91",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-18T19:26:10.237298Z",
     "iopub.status.busy": "2023-08-18T19:26:10.236886Z",
     "iopub.status.idle": "2023-08-18T19:26:10.243577Z",
     "shell.execute_reply": "2023-08-18T19:26:10.242480Z"
    },
    "origin_pos": 70,
    "tab": [
     "pytorch"
    ]
   },
   "outputs": [],
   "source": [
    "def f(a):\n",
    "    b = a * 2\n",
    "    while b.norm() < 1000:\n",
    "        b = b * 2\n",
    "    if b.sum() > 0:\n",
    "        c = b\n",
    "    else:\n",
    "        c = 100 * b\n",
    "    return c\n",
    "\n",
    "a = torch.randn(size=(), requires_grad=True)\n",
    "d = f(a)\n",
    "d.backward()\n",
    "\n",
    "a.grad == d / a"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "required_libs": [],
  "rise": {
   "autolaunch": false,
   "enable_chalkboard": true,
   "overlay": "<div class='my-top-right'><img height=80px src='../d2l-en/static/logo-with-text.png'/></div><div class='my-top-left'></div>",
   "scroll": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}