BERT/Custom_Named_Entity_Recognition_with_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Custom Named Entity Recognition with BERT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3f0c39538ccc43eb800099ee1e8aaaf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f2003724d2f9402291b90cf643debde3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d4f35d8c9a2c43b0a293abc1b6bfabea",
              "IPY_MODEL_f0a1fefe73bb4c97a8082a70f79da8b2",
              "IPY_MODEL_b5035eff71e94419815074dcf523291d",
              "IPY_MODEL_a80d70df10194fbca3149e66aaf08c78",
              "IPY_MODEL_45a51b4b0b1049c8981fa8eab19d6b11"
            ]
          }
        },
        "f2003724d2f9402291b90cf643debde3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "column",
            "width": "50%",
            "min_width": null,
            "border": null,
            "align_items": "center",
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "flex",
            "left": null
          }
        },
        "d4f35d8c9a2c43b0a293abc1b6bfabea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c79560a41bb54490ba870245062560f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "<center>\n<img src=https://huggingface.co/front/assets/huggingface_logo-noborder.svg alt='Hugging Face'>\n<br>\nCopy a token from <a href=\"https://huggingface.co/settings/token\" target=\"_blank\">your Hugging Face tokens page</a> and paste it below.\n<br>\nImmediately click login after copying your token or it might be stored in plain text in this notebook file.\n</center>",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90b362e5517049688e67b61d4a8acfe4"
          }
        },
        "f0a1fefe73bb4c97a8082a70f79da8b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "PasswordView",
            "style": "IPY_MODEL_ae1235da1aab4859a24974c1592363df",
            "_dom_classes": [],
            "description": "Token:",
            "_model_name": "PasswordModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "continuous_update": true,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8198e84854b94ac591bc2cfedc1771ae"
          }
        },
        "b5035eff71e94419815074dcf523291d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ButtonView",
            "style": "IPY_MODEL_d3e0b329daf6496fb79323bf120a44bd",
            "_dom_classes": [],
            "description": "Login",
            "_model_name": "ButtonModel",
            "button_style": "",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "tooltip": "",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "layout": "IPY_MODEL_9fa72d570d29489e8f0b7e57634ef92a",
            "_model_module": "@jupyter-widgets/controls",
            "icon": ""
          }
        },
        "a80d70df10194fbca3149e66aaf08c78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7fe522823a8840c7aea0c9728054f78b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated 'notebooks' token with 'write' access, that you can then easily reuse for all notebooks.\n<br>\n<i>Logging in with your username and password is deprecated and won't be possible anymore in the near future. You can still use them for now by clicking below.</i>\n</center>",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_455be84e2e184e37a425753e7106653e"
          }
        },
        "45a51b4b0b1049c8981fa8eab19d6b11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ButtonView",
            "style": "IPY_MODEL_558544f011614ea590d1abeb79689b20",
            "_dom_classes": [],
            "description": "Use password",
            "_model_name": "ButtonModel",
            "button_style": "",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "tooltip": "",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "layout": "IPY_MODEL_6dceb012fcdf4d8b942909d28821c633",
            "_model_module": "@jupyter-widgets/controls",
            "icon": ""
          }
        },
        "c79560a41bb54490ba870245062560f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90b362e5517049688e67b61d4a8acfe4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae1235da1aab4859a24974c1592363df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8198e84854b94ac591bc2cfedc1771ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3e0b329daf6496fb79323bf120a44bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ButtonStyleModel",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "button_color": null,
            "font_weight": "",
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9fa72d570d29489e8f0b7e57634ef92a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7fe522823a8840c7aea0c9728054f78b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "455be84e2e184e37a425753e7106653e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "558544f011614ea590d1abeb79689b20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ButtonStyleModel",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "button_color": null,
            "font_weight": "",
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6dceb012fcdf4d8b942909d28821c633": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2233d8919b64cfcb6fa898c400b5673": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_35add86a7ae34843bc18db2da48b3255",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_96587923209f4a958d340d2992e16c3b",
              "IPY_MODEL_0c4f1a279f2f47c897947f426ddc1569",
              "IPY_MODEL_cc8464c15946471f9863040358c9b071"
            ]
          }
        },
        "35add86a7ae34843bc18db2da48b3255": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "96587923209f4a958d340d2992e16c3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7ee8cc9ac56f4b2cadb73a99bc064c5b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Upload file pytorch_model.bin: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_198b3b068e54406da2ab611a19ae8008"
          }
        },
        "0c4f1a279f2f47c897947f426ddc1569": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0ae79201bd3a432bad399ad26427ecc2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435681969,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435681969,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1208952534f143b994c3c7087c194b51"
          }
        },
        "cc8464c15946471f9863040358c9b071": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a60e72178fed4eefb0fee3c6d3f2c1fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 415M/415M [04:58&lt;00:00, 948kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4afd39e7f1d493d9db20b1fa852eb5b"
          }
        },
        "7ee8cc9ac56f4b2cadb73a99bc064c5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "198b3b068e54406da2ab611a19ae8008": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ae79201bd3a432bad399ad26427ecc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1208952534f143b994c3c7087c194b51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a60e72178fed4eefb0fee3c6d3f2c1fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4afd39e7f1d493d9db20b1fa852eb5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b45067ba34c745078632a7ed1ecfcfde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e1055dc6d54c46d68b6b6ad3fd523d8b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0101cdf1aae54ee9be3fab68706ffb44",
              "IPY_MODEL_9b990881eb394e998149c3d92065dde3",
              "IPY_MODEL_7d223f7b9c9c420393f7daa409382119"
            ]
          }
        },
        "e1055dc6d54c46d68b6b6ad3fd523d8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0101cdf1aae54ee9be3fab68706ffb44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_83319745e64f48e3a5446e86d441a0e2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a52852a97c284a77b5ce1e8683745a73"
          }
        },
        "9b990881eb394e998149c3d92065dde3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ed3cd1339e8347a3aa9c4eb9af3fa042",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 369,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 369,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_913e92e121f04aaaa1591c24ad2aaa0d"
          }
        },
        "7d223f7b9c9c420393f7daa409382119": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_579240ba60aa4352a4ab4be9c754a061",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 369/369 [00:00&lt;00:00, 4.22kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20d4708da2b444bd9886edfff45e8f1b"
          }
        },
        "83319745e64f48e3a5446e86d441a0e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a52852a97c284a77b5ce1e8683745a73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ed3cd1339e8347a3aa9c4eb9af3fa042": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "913e92e121f04aaaa1591c24ad2aaa0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "579240ba60aa4352a4ab4be9c754a061": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20d4708da2b444bd9886edfff45e8f1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f163ad4caa4348d98765f0058d8ebaa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d733118516b14c9e98104eb0bf88650f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_83620b5680a245169672012f68b588a9",
              "IPY_MODEL_171a6288d6b944ebbb591e7bed62d152",
              "IPY_MODEL_6f32331b10a14d96a5e649d2368fbe0e"
            ]
          }
        },
        "d733118516b14c9e98104eb0bf88650f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83620b5680a245169672012f68b588a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5a632929f2a8485eb02b1dd9b339f348",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8cfd1e69f8824a1db8348c7fd5e21ede"
          }
        },
        "171a6288d6b944ebbb591e7bed62d152": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_14f370fb3da1464c9e880cd0075f1299",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d53f286986d948fb84658b0e37b3354a"
          }
        },
        "6f32331b10a14d96a5e649d2368fbe0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5c6f67ff349f4f35954d93a02250f5dd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 226k/226k [00:00&lt;00:00, 254kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ee787678bc414ec5bd099d04ac5817f8"
          }
        },
        "5a632929f2a8485eb02b1dd9b339f348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8cfd1e69f8824a1db8348c7fd5e21ede": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14f370fb3da1464c9e880cd0075f1299": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d53f286986d948fb84658b0e37b3354a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c6f67ff349f4f35954d93a02250f5dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ee787678bc414ec5bd099d04ac5817f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20fb4d560bca4edea60c047ba1e0de78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4cea747e3f464f6fbe0555822eccfc14",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1a2dc487acc94df1811d78ead637d04c",
              "IPY_MODEL_6a0143b1d2e24c849bc0897c9778cf13",
              "IPY_MODEL_43bbd28475be4e47a7e041da4675743b"
            ]
          }
        },
        "4cea747e3f464f6fbe0555822eccfc14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a2dc487acc94df1811d78ead637d04c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5ca95ca54fba454aa71a81533346647a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8654fb2c0edd4b74b8f52cf5a7c3a40d"
          }
        },
        "6a0143b1d2e24c849bc0897c9778cf13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d36262841f154e17a2751025a4a2834c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_954936e2061e4dc39f759e68589c4e39"
          }
        },
        "43bbd28475be4e47a7e041da4675743b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e33f7a7f413a4f4f9afac764f73d27d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 2.79kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7ae9246c54b447f1948378269ca8e6ac"
          }
        },
        "5ca95ca54fba454aa71a81533346647a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8654fb2c0edd4b74b8f52cf5a7c3a40d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d36262841f154e17a2751025a4a2834c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "954936e2061e4dc39f759e68589c4e39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e33f7a7f413a4f4f9afac764f73d27d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7ae9246c54b447f1948378269ca8e6ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d3fb9f31e5b474e8893a8ede56d16b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7deead08b65c434db06cdde6c2e72232",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c96156ea9c1e49aab7c065ebbfc0537f",
              "IPY_MODEL_0360c745f569434190a7a72514eea049",
              "IPY_MODEL_47fa90493b9c4e74b2d933934b1a5702"
            ]
          }
        },
        "7deead08b65c434db06cdde6c2e72232": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c96156ea9c1e49aab7c065ebbfc0537f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_add89a77779441ef883ad6106e4184d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f43f095774df4a75b95a5ee11472c64a"
          }
        },
        "0360c745f569434190a7a72514eea049": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1460bbcdf06540a4912d17cdf3d4043c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1083,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1083,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_201ec7a20b6e471bbccda65bc520ac24"
          }
        },
        "47fa90493b9c4e74b2d933934b1a5702": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_068cad028be244b9ab5e26dff7908935",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.06k/1.06k [00:00&lt;00:00, 32.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bf624b00ab134acda8bb505844731081"
          }
        },
        "add89a77779441ef883ad6106e4184d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f43f095774df4a75b95a5ee11472c64a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1460bbcdf06540a4912d17cdf3d4043c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "201ec7a20b6e471bbccda65bc520ac24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "068cad028be244b9ab5e26dff7908935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bf624b00ab134acda8bb505844731081": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f25b7d5f21646e3ae6e34682de0e559": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c51b5a92894840e58736a106f521ea8d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_252090bca3794efca06337bb0bced176",
              "IPY_MODEL_d7b0c9293306465da88262e66c4bb1cf",
              "IPY_MODEL_91cb71ee71814869b986f307d8e6caac"
            ]
          }
        },
        "c51b5a92894840e58736a106f521ea8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "252090bca3794efca06337bb0bced176": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f1821d9437874771bf3a5f6e01597749",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_17983bd0617c4b1f8e35dd12cbaad18e"
          }
        },
        "d7b0c9293306465da88262e66c4bb1cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fbf9c204f1bb4439ad33aa95a6e52562",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435681969,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435681969,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_999fba878ce940928cbfb35cef43c9b8"
          }
        },
        "91cb71ee71814869b986f307d8e6caac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_33d0f4dc3b2d4f428aa858ac8f76c0b7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 415M/415M [00:27&lt;00:00, 14.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_281efa82b7624d4d9c441c9691924757"
          }
        },
        "f1821d9437874771bf3a5f6e01597749": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "17983bd0617c4b1f8e35dd12cbaad18e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fbf9c204f1bb4439ad33aa95a6e52562": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "999fba878ce940928cbfb35cef43c9b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33d0f4dc3b2d4f428aa858ac8f76c0b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "281efa82b7624d4d9c441c9691924757": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/BERT/Custom_Named_Entity_Recognition_with_BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MyETdB-dkBsX"
      },
      "source": [
        "## **Fine-tuning BERT for named-entity recognition**\n",
        "\n",
        "In this notebook, we are going to use **BertForTokenClassification** which is included in the [Transformers library](https://github.com/huggingface/transformers) by HuggingFace. This model has BERT as its base architecture, with a token classification head on top, allowing it to make predictions at the token level, rather than the sequence level. Named entity recognition is typically treated as a token classification problem, so that's what we are going to use it for.\n",
        "\n",
        "This tutorial uses the idea of **transfer learning**, i.e. first pretraining a large neural network in an unsupervised way, and then fine-tuning that neural network on a task of interest. In this case, BERT is a neural network pretrained on 2 tasks: masked language modeling and next sentence prediction. Now, we are going to fine-tune this network on a NER dataset. Fine-tuning is supervised learning, so this means we will need a labeled dataset.\n",
        "\n",
        "If you want to know more about BERT, I suggest the following resources:\n",
        "* the original [paper](https://arxiv.org/abs/1810.04805)\n",
        "* Jay Allamar's [blog post](http://jalammar.github.io/illustrated-bert/) as well as his [tutorial](http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/)\n",
        "* Chris Mccormick's [Youtube channel](https://www.youtube.com/channel/UCoRX98PLOsaN8PtekB9kWrw)\n",
        "* Abbishek Kumar Mishra's [Youtube channel](https://www.youtube.com/user/abhisheksvnit)\n",
        "\n",
        "The following notebook largely follows the same structure as the tutorials by Abhishek Kumar Mishra. For his tutorials on the Transformers library, see his [Github repository](https://github.com/abhimishra91/transformers-tutorials).\n",
        "\n",
        "NOTE: this notebook assumes basic knowledge about deep learning, BERT, and native PyTorch. If you want to learn more Python, deep learning and PyTorch, I highly recommend cs231n by Stanford University and the FastAI course by Jeremy Howard et al. Both are freely available on the web.  \n",
        "\n",
        "Now, let's move on to the real stuff!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e7wfLWyYkvDi"
      },
      "source": [
        "#### **Importing Python Libraries and preparing the environment**\n",
        "\n",
        "This notebook assumes that you have the following libraries installed:\n",
        "* pandas\n",
        "* numpy\n",
        "* sklearn\n",
        "* pytorch\n",
        "* transformers\n",
        "* seqeval\n",
        "\n",
        "As we are running this in Google Colab, the only libraries we need to additionally install are transformers and seqeval (GPU version):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d4_YJqjR_Gjw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a483a22e-a3d8-4ba9-a5b7-bc833848602f"
      },
      "source": [
        "!pip install transformers seqeval[gpu]"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.17.0)\n",
            "Requirement already satisfied: seqeval[gpu] in /usr/local/lib/python3.7/dist-packages (1.2.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.47)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.11.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval[gpu]) (1.0.2)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval[gpu]) (1.4.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval[gpu]) (3.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IEnlUbgm8z3B"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import accuracy_score\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from transformers import BertTokenizer, BertConfig, BertForTokenClassification"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jzq1w3L1K5M-"
      },
      "source": [
        "As deep learning can be accellerated a lot using a GPU instead of a CPU, make sure you can run this notebook in a GPU runtime (which Google Colab provides for free! - check \"Runtime\" - \"Change runtime type\" - and set the hardware accelerator to \"GPU\").\n",
        "\n",
        "We can set the default device to GPU using the following code (if it prints \"cuda\", it means the GPU has been recognized):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sm1krxJtKxpx",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10b11257-e569-48de-e6ad-ba3e1ff94ba7"
      },
      "source": [
        "from torch import cuda\n",
        "device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "print(device)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahwMsmyG5ZPE"
      },
      "source": [
        "#### **Downloading and preprocessing the data**\n",
        "Named entity recognition (NER) uses a specific annotation scheme, which is defined (at least for European languages) at the *word* level. An annotation scheme that is widely used is called **[IOB-tagging](https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging))**, which stands for Inside-Outside-Beginning. Each tag indicates whether the corresponding word is *inside*, *outside* or at the *beginning* of a specific named entity. The reason this is used is because named entities usually comprise more than 1 word. \n",
        "\n",
        "Let's have a look at an example. If you have a sentence like \"Barack Obama was born in Hawaï\", then the corresponding tags would be   [B-PERS, I-PERS, O, O, O, B-GEO]. B-PERS means that the word \"Barack\" is the beginning of a person, I-PERS means that the word \"Obama\" is inside a person, \"O\" means that the word \"was\" is outside a named entity, and so on. So one typically has as many tags as there are words in a sentence.\n",
        "\n",
        "So if you want to train a deep learning model for NER, it requires that you have your data in this IOB format (or similar formats such as [BILOU](https://stackoverflow.com/questions/17116446/what-do-the-bilou-tags-mean-in-named-entity-recognition)). There exist many annotation tools which let you create these kind of annotations automatically (such as Spacy's [Prodigy](https://prodi.gy/), [Tagtog](https://docs.tagtog.net/) or [Doccano](https://github.com/doccano/doccano)). You can also use Spacy's [biluo_tags_from_offsets](https://spacy.io/api/goldparse#biluo_tags_from_offsets) function to convert annotations at the character level to IOB format.\n",
        "\n",
        "Here, we will use a NER dataset from [Kaggle](https://www.kaggle.com/namanj27/ner-dataset) that is already in IOB format. One has to go to this web page, download the dataset, unzip it, and upload the csv file to this notebook. Let's print out the first few rows of this csv file:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "deLB9HVX5I6F",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "13dbe700-7d7c-4b1b-f28b-cc8b4010fe61"
      },
      "source": [
        "data = pd.read_csv(\"ner_datasetreference.csv\", encoding='unicode_escape')\n",
        "data.head()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-60d85617-0a83-45fe-b8de-b165c129c778\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-60d85617-0a83-45fe-b8de-b165c129c778')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-60d85617-0a83-45fe-b8de-b165c129c778 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-60d85617-0a83-45fe-b8de-b165c129c778');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag\n",
              "0  Sentence: 1      Thousands  NNS   O\n",
              "1          NaN             of   IN   O\n",
              "2          NaN  demonstrators  NNS   O\n",
              "3          NaN           have  VBP   O\n",
              "4          NaN        marched  VBN   O"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ucYjhq6uRAmY"
      },
      "source": [
        "Let's check how many sentences and words (and corresponding tags) there are in this dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gMibEJXTKDw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69132b39-d771-42fd-d272-d777475a9fe0"
      },
      "source": [
        "data.count()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Sentence #      47959\n",
              "Word          1048575\n",
              "POS           1048575\n",
              "Tag           1048575\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sUGXZOfE_GkO"
      },
      "source": [
        "As we can see, there are approximately 48,000 sentences in the dataset, comprising more than 1 million words and tags (quite huge!). This corresponds to approximately 20 words per sentence. \n",
        "\n",
        "Let's have a look at the different NER tags, and their frequency: "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "s4Jn1fVT_GkO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3d3d8ee-0062-4e32-9a48-d6927c03eec9"
      },
      "source": [
        "print(\"Number of tags: {}\".format(len(data.Tag.unique())))\n",
        "frequencies = data.Tag.value_counts()\n",
        "frequencies"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of tags: 17\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "O        887908\n",
              "B-geo     37644\n",
              "B-tim     20333\n",
              "B-org     20143\n",
              "I-per     17251\n",
              "B-per     16990\n",
              "I-org     16784\n",
              "B-gpe     15870\n",
              "I-geo      7414\n",
              "I-tim      6528\n",
              "B-art       402\n",
              "B-eve       308\n",
              "I-art       297\n",
              "I-eve       253\n",
              "B-nat       201\n",
              "I-gpe       198\n",
              "I-nat        51\n",
              "Name: Tag, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gsX0ebih_GkS"
      },
      "source": [
        "There are 8 category tags, each with a \"beginning\" and \"inside\" variant, and the \"outside\" tag. It is not really clear what these tags mean - \"geo\" probably stands for geographical entity, \"gpe\" for geopolitical entity, and so on. They do not seem to correspond with what the publisher says on Kaggle. Some tags seem to be underrepresented. Let's print them by frequency (highest to lowest): "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "CmjbHirJ_GkS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c4da2e8a-c0a1-433a-b3e8-45919de835b7"
      },
      "source": [
        "tags = {}\n",
        "for tag, count in zip(frequencies.index, frequencies):\n",
        "    if tag != \"O\":\n",
        "        if tag[2:5] not in tags.keys():\n",
        "            tags[tag[2:5]] = count\n",
        "        else:\n",
        "            tags[tag[2:5]] += count\n",
        "    continue\n",
        "\n",
        "print(sorted(tags.items(), key=lambda x: x[1], reverse=True))"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('geo', 45058), ('org', 36927), ('per', 34241), ('tim', 26861), ('gpe', 16068), ('art', 699), ('eve', 561), ('nat', 252)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aGUQemBz_GkV"
      },
      "source": [
        "Let's remove \"art\", \"eve\" and \"nat\" named entities, as performance on them will probably be not comparable to the other named entities. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8iorLrU4_GkW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "01d49910-f2e5-4d9d-a492-0cae03439aa9"
      },
      "source": [
        "entities_to_remove = [\"B-art\", \"I-art\", \"B-eve\", \"I-eve\", \"B-nat\", \"I-nat\"]\n",
        "data = data[~data.Tag.isin(entities_to_remove)]\n",
        "data.head()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ce18f138-596b-4a51-b332-f9d89ccd28f1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ce18f138-596b-4a51-b332-f9d89ccd28f1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ce18f138-596b-4a51-b332-f9d89ccd28f1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ce18f138-596b-4a51-b332-f9d89ccd28f1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag\n",
              "0  Sentence: 1      Thousands  NNS   O\n",
              "1          NaN             of   IN   O\n",
              "2          NaN  demonstrators  NNS   O\n",
              "3          NaN           have  VBP   O\n",
              "4          NaN        marched  VBN   O"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mskU4h0oRKEF"
      },
      "source": [
        "Now, we have to ask ourself the question: what is a training example in the case of NER, which is provided in a single forward pass? A training example is typically a **sentence**, with corresponding IOB tags. Let's group the words and corresponding tags by sentence:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zkW2vNcO-uMH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "641e1171-929e-41d7-f229-fcaf130807f8"
      },
      "source": [
        "# pandas has a very handy \"forward fill\" function to fill missing values based on the last upper non-nan value\n",
        "data = data.fillna(method='ffill')\n",
        "data.head()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-7a37e70a-803c-4bba-89f3-195e20eb06c8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7a37e70a-803c-4bba-89f3-195e20eb06c8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7a37e70a-803c-4bba-89f3-195e20eb06c8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7a37e70a-803c-4bba-89f3-195e20eb06c8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag\n",
              "0  Sentence: 1      Thousands  NNS   O\n",
              "1  Sentence: 1             of   IN   O\n",
              "2  Sentence: 1  demonstrators  NNS   O\n",
              "3  Sentence: 1           have  VBP   O\n",
              "4  Sentence: 1        marched  VBN   O"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hmd-ow389k6Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "9c00a203-14df-4078-be2a-f3e9ec36af53"
      },
      "source": [
        "# let's create a new column called \"sentence\" which groups the words by sentence \n",
        "data['sentence'] = data[['Sentence #','Word','Tag']].groupby(['Sentence #'])['Word'].transform(lambda x: ' '.join(x))\n",
        "# let's also create a new column called \"word_labels\" which groups the tags by sentence \n",
        "data['word_labels'] = data[['Sentence #','Word','Tag']].groupby(['Sentence #'])['Tag'].transform(lambda x: ','.join(x))\n",
        "data.head()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-60e43875-628d-4392-8a71-ec50b325e470\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "      <th>sentence</th>\n",
              "      <th>word_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-60e43875-628d-4392-8a71-ec50b325e470')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-60e43875-628d-4392-8a71-ec50b325e470 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-60e43875-628d-4392-8a71-ec50b325e470');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag  \\\n",
              "0  Sentence: 1      Thousands  NNS   O   \n",
              "1  Sentence: 1             of   IN   O   \n",
              "2  Sentence: 1  demonstrators  NNS   O   \n",
              "3  Sentence: 1           have  VBP   O   \n",
              "4  Sentence: 1        marched  VBN   O   \n",
              "\n",
              "                                            sentence  \\\n",
              "0  Thousands of demonstrators have marched throug...   \n",
              "1  Thousands of demonstrators have marched throug...   \n",
              "2  Thousands of demonstrators have marched throug...   \n",
              "3  Thousands of demonstrators have marched throug...   \n",
              "4  Thousands of demonstrators have marched throug...   \n",
              "\n",
              "                                         word_labels  \n",
              "0  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  \n",
              "1  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  \n",
              "2  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  \n",
              "3  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  \n",
              "4  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  "
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsjhdQbE-Lve"
      },
      "source": [
        "Let's have a look at the different NER tags. \n",
        "\n",
        "We create 2 dictionaries: one that maps individual tags to indices, and one that maps indices to their individual tags. This is necessary in order to create the labels (as computers work with numbers = indices, rather than words = tags) - see further in this notebook."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFRDM8WsQXvL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03354a90-7477-44e3-e6c2-a8dc1fc35c61"
      },
      "source": [
        "label2id = {k: v for v, k in enumerate(data.Tag.unique())}\n",
        "id2label = {v: k for v, k in enumerate(data.Tag.unique())}\n",
        "label2id"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'B-geo': 1,\n",
              " 'B-gpe': 2,\n",
              " 'B-org': 5,\n",
              " 'B-per': 3,\n",
              " 'B-tim': 7,\n",
              " 'I-geo': 4,\n",
              " 'I-gpe': 9,\n",
              " 'I-org': 6,\n",
              " 'I-per': 8,\n",
              " 'I-tim': 10,\n",
              " 'O': 0}"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J08Cvk_USgbM"
      },
      "source": [
        "As we can see, there are now only 10 different tags. \n",
        "\n",
        "Let's only keep the \"sentence\" and \"word_labels\" columns, and drop duplicates:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SrEgd4PZUgmF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "2ee4ce4b-a6f4-4100-ed49-2993312a5139"
      },
      "source": [
        "data = data[[\"sentence\", \"word_labels\"]].drop_duplicates().reset_index(drop=True)\n",
        "data.head()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4c105197-96df-40d7-a7f0-eb0d98b14363\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>word_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Families of soldiers killed in the conflict jo...</td>\n",
              "      <td>O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,B-per,O,O,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>They marched from the Houses of Parliament to ...</td>\n",
              "      <td>O,O,O,O,O,O,O,O,O,O,O,B-geo,I-geo,O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Police put the number of marchers at 10,000 wh...</td>\n",
              "      <td>O,O,O,O,O,O,O,O,O,O,O,O,O,O,O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The protest comes on the eve of the annual con...</td>\n",
              "      <td>O,O,O,O,O,O,O,O,O,O,O,B-geo,O,O,B-org,I-org,O,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c105197-96df-40d7-a7f0-eb0d98b14363')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4c105197-96df-40d7-a7f0-eb0d98b14363 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4c105197-96df-40d7-a7f0-eb0d98b14363');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                            sentence  \\\n",
              "0  Thousands of demonstrators have marched throug...   \n",
              "1  Families of soldiers killed in the conflict jo...   \n",
              "2  They marched from the Houses of Parliament to ...   \n",
              "3  Police put the number of marchers at 10,000 wh...   \n",
              "4  The protest comes on the eve of the annual con...   \n",
              "\n",
              "                                         word_labels  \n",
              "0  O,O,O,O,O,O,B-geo,O,O,O,O,O,B-geo,O,O,O,O,O,B-...  \n",
              "1  O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,O,B-per,O,O,...  \n",
              "2                O,O,O,O,O,O,O,O,O,O,O,B-geo,I-geo,O  \n",
              "3                      O,O,O,O,O,O,O,O,O,O,O,O,O,O,O  \n",
              "4  O,O,O,O,O,O,O,O,O,O,O,B-geo,O,O,B-org,I-org,O,...  "
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3ArUiVRqw0C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06f540c5-fa9d-46bc-f911-975be97714c8"
      },
      "source": [
        "len(data)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "47571"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U8obZumRTBrT"
      },
      "source": [
        "Let's verify that a random sentence and its corresponding tags are correct:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eUvupomW_fbe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "25f75911-9a00-488c-9f1c-cae363a9c937"
      },
      "source": [
        "data.iloc[41].sentence"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Bedfordshire police said Tuesday that Omar Khayam was arrested in Bedford for breaching the conditions of his parole .'"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0dLyY3Oi_lvp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "6823eff2-f905-43fd-f0ba-6e9e6753dea6"
      },
      "source": [
        "data.iloc[41].word_labels"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'B-gpe,O,O,B-tim,O,B-per,I-per,O,O,O,B-geo,O,O,O,O,O,O,O,O'"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f5EHpuB78pIa"
      },
      "source": [
        "#### **Preparing the dataset and dataloader**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "15x7zmZnTgFx"
      },
      "source": [
        "Now that our data is preprocessed, we can turn it into PyTorch tensors such that we can provide it to the model. Let's start by defining some key variables that will be used later on in the training/evaluation process:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lgNSM8Xz79Mg",
        "tags": []
      },
      "source": [
        "MAX_LEN = 128\n",
        "TRAIN_BATCH_SIZE = 4\n",
        "VALID_BATCH_SIZE = 2\n",
        "EPOCHS = 1\n",
        "LEARNING_RATE = 1e-05\n",
        "MAX_GRAD_NORM = 10\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wPYV2Ld6Tr5I"
      },
      "source": [
        "A tricky part of NER with BERT is that BERT relies on **wordpiece tokenization**, rather than word tokenization. This means that we should also define the labels at the wordpiece-level, rather than the word-level! \n",
        "\n",
        "For example, if you have word like \"Washington\" which is labeled as \"b-gpe\", but it gets tokenized to \"Wash\", \"##ing\", \"##ton\", then we will have to propagate the word’s original label to all of its wordpieces: \"b-gpe\", \"b-gpe\", \"b-gpe\". The model should be able to produce the correct labels for each individual wordpiece. The function below (taken from [here](https://github.com/chambliss/Multilingual_NER/blob/master/python/utils/main_utils.py#L118)) implements this.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNzSgZTfGUd8"
      },
      "source": [
        "def tokenize_and_preserve_labels(sentence, text_labels, tokenizer):\n",
        "    \"\"\"\n",
        "    Word piece tokenization makes it difficult to match word labels\n",
        "    back up with individual word pieces. This function tokenizes each\n",
        "    word one at a time so that it is easier to preserve the correct\n",
        "    label for each subword. It is, of course, a bit slower in processing\n",
        "    time, but it will help our model achieve higher accuracy.\n",
        "    \"\"\"\n",
        "\n",
        "    tokenized_sentence = []\n",
        "    labels = []\n",
        "\n",
        "    sentence = sentence.strip()\n",
        "\n",
        "    for word, label in zip(sentence.split(), text_labels.split(\",\")):\n",
        "\n",
        "        # Tokenize the word and count # of subwords the word is broken into\n",
        "        tokenized_word = tokenizer.tokenize(word)\n",
        "        n_subwords = len(tokenized_word)\n",
        "\n",
        "        # Add the tokenized word to the final tokenized word list\n",
        "        tokenized_sentence.extend(tokenized_word)\n",
        "\n",
        "        # Add the same label to the new list of labels `n_subwords` times\n",
        "        labels.extend([label] * n_subwords)\n",
        "\n",
        "    return tokenized_sentence, labels"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ez7qlFHl56ZW"
      },
      "source": [
        "Note that this is a **design decision**. You could also decide to only label the first wordpiece of each word and let the model only learn this (this is what was done in the original BERT paper, see Github discussion [here](https://github.com/huggingface/transformers/issues/64#issuecomment-443703063)). Another design decision could be to give the first wordpiece of each word the original word label, and then use the label “X” for all subsequent subwords of that word.\n",
        "\n",
        "All of them lead to good performance.\n",
        "\n",
        "Next, we define a regular PyTorch [dataset class](https://pytorch.org/docs/stable/data.html) (which transforms examples of a dataframe to PyTorch tensors). Here, each sentence gets tokenized, the special tokens that BERT expects are added, the tokens are padded or truncated based on the max length of the model, the attention mask is created and the labels are created based on the dictionary which we defined above. \n",
        "\n",
        "For more information about BERT's inputs, see [here](https://huggingface.co/transformers/glossary.html).  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJty_Abw8_xK"
      },
      "source": [
        "class dataset(Dataset):\n",
        "    def __init__(self, dataframe, tokenizer, max_len):\n",
        "        self.len = len(dataframe)\n",
        "        self.data = dataframe\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_len = max_len\n",
        "        \n",
        "    def __getitem__(self, index):\n",
        "        # step 1: tokenize (and adapt corresponding labels)\n",
        "        sentence = self.data.sentence[index]  \n",
        "        word_labels = self.data.word_labels[index]  \n",
        "        tokenized_sentence, labels = tokenize_and_preserve_labels(sentence, word_labels, self.tokenizer)\n",
        "        \n",
        "        # step 2: add special tokens (and corresponding labels)\n",
        "        tokenized_sentence = [\"[CLS]\"] + tokenized_sentence + [\"[SEP]\"] # add special tokens\n",
        "        labels.insert(0, \"O\") # add outside label for [CLS] token\n",
        "        labels.insert(-1, \"O\") # add outside label for [SEP] token\n",
        "\n",
        "        # step 3: truncating/padding\n",
        "        maxlen = self.max_len\n",
        "\n",
        "        if (len(tokenized_sentence) > maxlen):\n",
        "          # truncate\n",
        "          tokenized_sentence = tokenized_sentence[:maxlen]\n",
        "          labels = labels[:maxlen]\n",
        "        else:\n",
        "          # pad\n",
        "          tokenized_sentence = tokenized_sentence + ['[PAD]'for _ in range(maxlen - len(tokenized_sentence))]\n",
        "          labels = labels + [\"O\" for _ in range(maxlen - len(labels))]\n",
        "\n",
        "        # step 4: obtain the attention mask\n",
        "        attn_mask = [1 if tok != '[PAD]' else 0 for tok in tokenized_sentence]\n",
        "        \n",
        "        # step 5: convert tokens to input ids\n",
        "        ids = self.tokenizer.convert_tokens_to_ids(tokenized_sentence)\n",
        "\n",
        "        label_ids = [label2id[label] for label in labels]\n",
        "        # the following line is deprecated\n",
        "        #label_ids = [label if label != 0 else -100 for label in label_ids]\n",
        "        \n",
        "        return {\n",
        "              'ids': torch.tensor(ids, dtype=torch.long),\n",
        "              'mask': torch.tensor(attn_mask, dtype=torch.long),\n",
        "              #'token_type_ids': torch.tensor(token_ids, dtype=torch.long),\n",
        "              'targets': torch.tensor(label_ids, dtype=torch.long)\n",
        "        } \n",
        "    \n",
        "    def __len__(self):\n",
        "        return self.len"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hTP7zuWGWGUd"
      },
      "source": [
        "Now, based on the class we defined above, we can create 2 datasets, one for training and one for testing. Let's use a 80/20 split:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jrkdZBLYHVcB",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cca99c3e-f007-4bfa-8948-366daf68b9eb"
      },
      "source": [
        "train_size = 0.8\n",
        "train_dataset = data.sample(frac=train_size,random_state=200)\n",
        "test_dataset = data.drop(train_dataset.index).reset_index(drop=True)\n",
        "train_dataset = train_dataset.reset_index(drop=True)\n",
        "\n",
        "print(\"FULL Dataset: {}\".format(data.shape))\n",
        "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
        "print(\"TEST Dataset: {}\".format(test_dataset.shape))\n",
        "\n",
        "training_set = dataset(train_dataset, tokenizer, MAX_LEN)\n",
        "testing_set = dataset(test_dataset, tokenizer, MAX_LEN)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FULL Dataset: (47571, 2)\n",
            "TRAIN Dataset: (38057, 2)\n",
            "TEST Dataset: (9514, 2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ptv5AT_iTb7W"
      },
      "source": [
        "Let's have a look at the first training example:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phmPylgAm8Xy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6bd7ea5c-47d3-4f55-9fb5-3bf1b860bdcd"
      },
      "source": [
        "training_set[0]"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ids': tensor([  101, 23564, 21030,  2099,  4967,  2001,  9388,  1011,  6109,  2005,\n",
              "          2634,  1012,   102,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0]),\n",
              " 'mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " 'targets': tensor([0, 3, 3, 3, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0])}"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VvU4nzL2W2Xo"
      },
      "source": [
        "Let's verify that the input ids and corresponding targets are correct:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_set[0][\"ids\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZHoufyakY18x",
        "outputId": "8a447091-602b-48c6-fc6e-d98a649d4f65"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([  101, 23564, 21030,  2099,  4967,  2001,  9388,  1011,  6109,  2005,\n",
              "         2634,  1012,   102,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0])"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DWgnNJrYW2GP",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e949191c-fa89-470f-906e-bd2cad569fc4"
      },
      "source": [
        "# print the first 30 tokens and corresponding labels\n",
        "for token, label in zip(tokenizer.convert_ids_to_tokens(training_set[0][\"ids\"][:30]), training_set[0][\"targets\"][:30]):\n",
        "  print('{0:10}  {1}'.format(token, id2label[label.item()]))"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CLS]       O\n",
            "za          B-per\n",
            "##hee       B-per\n",
            "##r         B-per\n",
            "khan        I-per\n",
            "was         O\n",
            "mar         O\n",
            "-           O\n",
            "93          O\n",
            "for         O\n",
            "india       B-geo\n",
            ".           O\n",
            "[SEP]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n",
            "[PAD]       O\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ky68FcTgWnfN"
      },
      "source": [
        "Now, let's define the corresponding PyTorch dataloaders:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIw793myWOmi"
      },
      "source": [
        "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "testing_loader = DataLoader(testing_set, **test_params)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "73OzU7oXRxR8"
      },
      "source": [
        "#### **Defining the model**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T-iGhnhdLNdP"
      },
      "source": [
        "Here we define the model, BertForTokenClassification, and load it with the pretrained weights of \"bert-base-uncased\". The only thing we need to additionally specify is the number of labels (as this will determine the architecture of the classification head).\n",
        "\n",
        "Note that only the base layers are initialized with the pretrained weights. The token classification head of top has just randomly initialized weights, which we will train, together with the pretrained weights, using our labelled dataset. This is also printed as a warning when you run the code cell below.\n",
        "\n",
        "Then, we move the model to the GPU."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cB9MR3KcWXUs",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ef5d359-f132-4a6b-8998-3a3d85f36a4a"
      },
      "source": [
        "model = BertForTokenClassification.from_pretrained('bert-base-uncased', \n",
        "                                                   num_labels=len(id2label),\n",
        "                                                   id2label=id2label,\n",
        "                                                   label2id=label2id)\n",
        "model.to(device)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForTokenClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForTokenClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=11, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pp7Yl4JyWhDj"
      },
      "source": [
        "#### **Training the model**\n",
        "\n",
        "Before training the model, let's perform a sanity check, which I learned thanks to Andrej Karpathy's wonderful [cs231n course](http://cs231n.stanford.edu/) at Stanford (see also his [blog post about debugging neural networks](http://karpathy.github.io/2019/04/25/recipe/)). The initial loss of your model should be close to -ln(1/number of classes) = -ln(1/17) = 2.83. \n",
        "\n",
        "Why? Because we are using cross entropy loss. The cross entropy loss is defined as -ln(probability score of the model for the correct class). In the beginning, the weights are random, so the probability distribution for all of the classes for a given token will be uniform, meaning that the probability for the correct class will be near 1/17. The loss for a given token will thus be -ln(1/17). As PyTorch's [CrossEntropyLoss](https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html) (which is used by `BertForTokenClassification`) uses *mean reduction* by default, it will compute the mean loss for each of the tokens in the sequence (in other words, for all of the 512 tokens). The mean of 512 times -log(1/17) is, you guessed it, -log(1/17).  \n",
        "\n",
        "Let's verify this:\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eqAN7YVIjKTr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "414ed440-5505-4155-d4a2-0403674d3b3f"
      },
      "source": [
        "ids = training_set[0][\"ids\"].unsqueeze(0)\n",
        "mask = training_set[0][\"mask\"].unsqueeze(0)\n",
        "targets = training_set[0][\"targets\"].unsqueeze(0)\n",
        "ids = ids.to(device)\n",
        "mask = mask.to(device)\n",
        "targets = targets.to(device)\n",
        "outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
        "initial_loss = outputs[0]\n",
        "initial_loss"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(2.4042, device='cuda:0', grad_fn=<NllLossBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yLdwsru9Mh7U"
      },
      "source": [
        "This looks good. Let's also verify that the logits of the neural network have a shape of (batch_size, sequence_length, num_labels):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X-z6YCpGnvfj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e1126112-34aa-4a41-fb82-db6ef263bad7"
      },
      "source": [
        "tr_logits = outputs[1]\n",
        "tr_logits.shape"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 128, 11])"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kwDLXxOVOCvD"
      },
      "source": [
        "Next, we define the optimizer. Here, we are just going to use Adam with a default learning rate. One can also decide to use more advanced ones such as AdamW (Adam with weight decay fix), which is [included](https://huggingface.co/transformers/main_classes/optimizer_schedules.html) in the Transformers repository, and a learning rate scheduler, but we are not going to do that here."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kznSQfGIWdU4"
      },
      "source": [
        "optimizer = torch.optim.Adam(params=model.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vZQ8JMF0NOe1"
      },
      "source": [
        "Now let's define a regular PyTorch training function. It is partly based on [a really good repository about multilingual NER](https://github.com/chambliss/Multilingual_NER/blob/master/python/utils/main_utils.py#L344)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLFivpkwW1HY"
      },
      "source": [
        "# Defining the training function on the 80% of the dataset for tuning the bert model\n",
        "def train(epoch):\n",
        "    tr_loss, tr_accuracy = 0, 0\n",
        "    nb_tr_examples, nb_tr_steps = 0, 0\n",
        "    tr_preds, tr_labels = [], []\n",
        "    # put model in training mode\n",
        "    model.train()\n",
        "    \n",
        "    for idx, batch in enumerate(training_loader):\n",
        "        \n",
        "        ids = batch['ids'].to(device, dtype = torch.long)\n",
        "        mask = batch['mask'].to(device, dtype = torch.long)\n",
        "        targets = batch['targets'].to(device, dtype = torch.long)\n",
        "\n",
        "        outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
        "        loss, tr_logits = outputs.loss, outputs.logits\n",
        "        tr_loss += loss.item()\n",
        "\n",
        "        nb_tr_steps += 1\n",
        "        nb_tr_examples += targets.size(0)\n",
        "        \n",
        "        if idx % 100==0:\n",
        "            loss_step = tr_loss/nb_tr_steps\n",
        "            print(f\"Training loss per 100 training steps: {loss_step}\")\n",
        "           \n",
        "        # compute training accuracy\n",
        "        flattened_targets = targets.view(-1) # shape (batch_size * seq_len,)\n",
        "        active_logits = tr_logits.view(-1, model.num_labels) # shape (batch_size * seq_len, num_labels)\n",
        "        flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * seq_len,)\n",
        "        # now, use mask to determine where we should compare predictions with targets (includes [CLS] and [SEP] token predictions)\n",
        "        active_accuracy = mask.view(-1) == 1 # active accuracy is also of shape (batch_size * seq_len,)\n",
        "        targets = torch.masked_select(flattened_targets, active_accuracy)\n",
        "        predictions = torch.masked_select(flattened_predictions, active_accuracy)\n",
        "        \n",
        "        tr_preds.extend(predictions)\n",
        "        tr_labels.extend(targets)\n",
        "        \n",
        "        tmp_tr_accuracy = accuracy_score(targets.cpu().numpy(), predictions.cpu().numpy())\n",
        "        tr_accuracy += tmp_tr_accuracy\n",
        "    \n",
        "        # gradient clipping\n",
        "        torch.nn.utils.clip_grad_norm_(\n",
        "            parameters=model.parameters(), max_norm=MAX_GRAD_NORM\n",
        "        )\n",
        "        \n",
        "        # backward pass\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "    epoch_loss = tr_loss / nb_tr_steps\n",
        "    tr_accuracy = tr_accuracy / nb_tr_steps\n",
        "    print(f\"Training loss epoch: {epoch_loss}\")\n",
        "    print(f\"Training accuracy epoch: {tr_accuracy}\")"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k2dsCyP7dcF3"
      },
      "source": [
        "And let's train the model!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y07Ybw8rZeZ7",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7adeee4-baae-4c61-8de6-d1682dd749cf"
      },
      "source": [
        "for epoch in range(EPOCHS):\n",
        "    print(f\"Training epoch: {epoch + 1}\")\n",
        "    train(epoch)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training epoch: 1\n",
            "Training loss per 100 training steps: 2.257507801055908\n",
            "Training loss per 100 training steps: 0.4154649484570664\n",
            "Training loss per 100 training steps: 0.2701939059263883\n",
            "Training loss per 100 training steps: 0.21136601985218714\n",
            "Training loss per 100 training steps: 0.17849522581941765\n",
            "Training loss per 100 training steps: 0.15673487058872293\n",
            "Training loss per 100 training steps: 0.14080958732228707\n",
            "Training loss per 100 training steps: 0.12864232085480456\n",
            "Training loss per 100 training steps: 0.11885192252104909\n",
            "Training loss per 100 training steps: 0.11099200748030166\n",
            "Training loss per 100 training steps: 0.10436300416272339\n",
            "Training loss per 100 training steps: 0.098732150021685\n",
            "Training loss per 100 training steps: 0.09388644995146914\n",
            "Training loss per 100 training steps: 0.09015688475426845\n",
            "Training loss per 100 training steps: 0.08655438412643629\n",
            "Training loss per 100 training steps: 0.08367351255923147\n",
            "Training loss per 100 training steps: 0.0807475139461702\n",
            "Training loss per 100 training steps: 0.07850061704871135\n",
            "Training loss per 100 training steps: 0.07638459064483721\n",
            "Training loss per 100 training steps: 0.07452762559943513\n",
            "Training loss per 100 training steps: 0.07258037018027903\n",
            "Training loss per 100 training steps: 0.07086636544410707\n",
            "Training loss per 100 training steps: 0.06927561587050027\n",
            "Training loss per 100 training steps: 0.06810071642911494\n",
            "Training loss per 100 training steps: 0.06653289777042111\n",
            "Training loss per 100 training steps: 0.06534103016837871\n",
            "Training loss per 100 training steps: 0.06419615173464056\n",
            "Training loss per 100 training steps: 0.06301664189689492\n",
            "Training loss per 100 training steps: 0.06187959020901325\n",
            "Training loss per 100 training steps: 0.060933663363434296\n",
            "Training loss per 100 training steps: 0.060110324684206816\n",
            "Training loss per 100 training steps: 0.05930020764297561\n",
            "Training loss per 100 training steps: 0.058522181899654274\n",
            "Training loss per 100 training steps: 0.057836721014128185\n",
            "Training loss per 100 training steps: 0.057086359761872756\n",
            "Training loss per 100 training steps: 0.056444525056608\n",
            "Training loss per 100 training steps: 0.05573567320973274\n",
            "Training loss per 100 training steps: 0.055108766209429005\n",
            "Training loss per 100 training steps: 0.05447007775604953\n",
            "Training loss per 100 training steps: 0.05389548315754293\n",
            "Training loss epoch: 0.05368644366631744\n",
            "Training accuracy epoch: 0.9385354910971212\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4jcSOJr680a"
      },
      "source": [
        "#### **Evaluating the model**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rYUTuOEUdfFJ"
      },
      "source": [
        "Now that we've trained our model, we can evaluate its performance on the held-out test set (which is 20% of the data). Note that here, no gradient updates are performed, the model just outputs its logits. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RIVVfFHi7Aw7"
      },
      "source": [
        "def valid(model, testing_loader):\n",
        "    # put model in evaluation mode\n",
        "    model.eval()\n",
        "    \n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_examples, nb_eval_steps = 0, 0\n",
        "    eval_preds, eval_labels = [], []\n",
        "    \n",
        "    with torch.no_grad():\n",
        "        for idx, batch in enumerate(testing_loader):\n",
        "            \n",
        "            ids = batch['ids'].to(device, dtype = torch.long)\n",
        "            mask = batch['mask'].to(device, dtype = torch.long)\n",
        "            targets = batch['targets'].to(device, dtype = torch.long)\n",
        "            \n",
        "            outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
        "            loss, eval_logits = outputs.loss, outputs.logits\n",
        "            \n",
        "            eval_loss += loss.item()\n",
        "\n",
        "            nb_eval_steps += 1\n",
        "            nb_eval_examples += targets.size(0)\n",
        "        \n",
        "            if idx % 100==0:\n",
        "                loss_step = eval_loss/nb_eval_steps\n",
        "                print(f\"Validation loss per 100 evaluation steps: {loss_step}\")\n",
        "              \n",
        "            # compute evaluation accuracy\n",
        "            flattened_targets = targets.view(-1) # shape (batch_size * seq_len,)\n",
        "            active_logits = eval_logits.view(-1, model.num_labels) # shape (batch_size * seq_len, num_labels)\n",
        "            flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * seq_len,)\n",
        "            # now, use mask to determine where we should compare predictions with targets (includes [CLS] and [SEP] token predictions)\n",
        "            active_accuracy = mask.view(-1) == 1 # active accuracy is also of shape (batch_size * seq_len,)\n",
        "            targets = torch.masked_select(flattened_targets, active_accuracy)\n",
        "            predictions = torch.masked_select(flattened_predictions, active_accuracy)\n",
        "            \n",
        "            eval_labels.extend(targets)\n",
        "            eval_preds.extend(predictions)\n",
        "            \n",
        "            tmp_eval_accuracy = accuracy_score(targets.cpu().numpy(), predictions.cpu().numpy())\n",
        "            eval_accuracy += tmp_eval_accuracy\n",
        "    \n",
        "    #print(eval_labels)\n",
        "    #print(eval_preds)\n",
        "\n",
        "    labels = [id2label[id.item()] for id in eval_labels]\n",
        "    predictions = [id2label[id.item()] for id in eval_preds]\n",
        "\n",
        "    #print(labels)\n",
        "    #print(predictions)\n",
        "    \n",
        "    eval_loss = eval_loss / nb_eval_steps\n",
        "    eval_accuracy = eval_accuracy / nb_eval_steps\n",
        "    print(f\"Validation Loss: {eval_loss}\")\n",
        "    print(f\"Validation Accuracy: {eval_accuracy}\")\n",
        "\n",
        "    return labels, predictions"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zJaONluRdq-e"
      },
      "source": [
        "As we can see below, performance is quite good! Accuracy on the test test is > 93%."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2BrxRjvxApY8",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e8b6830-220b-49f8-fa91-b0fd05a72b6d"
      },
      "source": [
        "labels, predictions = valid(model, testing_loader)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validation loss per 100 evaluation steps: 0.0006468583596870303\n",
            "Validation loss per 100 evaluation steps: 0.029117425034885378\n",
            "Validation loss per 100 evaluation steps: 0.02645552399816149\n",
            "Validation loss per 100 evaluation steps: 0.02677391133851494\n",
            "Validation loss per 100 evaluation steps: 0.02708561681612352\n",
            "Validation loss per 100 evaluation steps: 0.027608126274245224\n",
            "Validation loss per 100 evaluation steps: 0.02766878997025017\n",
            "Validation loss per 100 evaluation steps: 0.02802180213946799\n",
            "Validation loss per 100 evaluation steps: 0.02813900749804568\n",
            "Validation loss per 100 evaluation steps: 0.02800608498731886\n",
            "Validation loss per 100 evaluation steps: 0.0283729030002046\n",
            "Validation loss per 100 evaluation steps: 0.02819417062155819\n",
            "Validation loss per 100 evaluation steps: 0.028217743144274266\n",
            "Validation loss per 100 evaluation steps: 0.028680528649606803\n",
            "Validation loss per 100 evaluation steps: 0.02914536368688233\n",
            "Validation loss per 100 evaluation steps: 0.02904094562052864\n",
            "Validation loss per 100 evaluation steps: 0.02893985871928632\n",
            "Validation loss per 100 evaluation steps: 0.02889460223758376\n",
            "Validation loss per 100 evaluation steps: 0.029179134117535057\n",
            "Validation loss per 100 evaluation steps: 0.029369855337453024\n",
            "Validation Loss: 0.02922834331467394\n",
            "Validation Accuracy: 0.9602300367723403\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SAznLDwx_U2X"
      },
      "source": [
        "However, the accuracy metric is misleading, as a lot of labels are \"outside\" (O), even after omitting predictions on the [PAD] tokens. What is important is looking at the precision, recall and f1-score of the individual tags. For this, we use the seqeval Python library: "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jDNXrjr-6BW",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c20a5d1d-2762-4226-9462-2f829a743d3d"
      },
      "source": [
        "from seqeval.metrics import classification_report\n",
        "\n",
        "print(classification_report([labels], [predictions]))"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "         geo       0.79      0.88      0.83      4613\n",
            "         gpe       0.89      0.89      0.89      1523\n",
            "         org       0.71      0.56      0.63      2761\n",
            "         per       0.78      0.81      0.79      2183\n",
            "         tim       0.82      0.81      0.81      1772\n",
            "\n",
            "   micro avg       0.79      0.79      0.79     12852\n",
            "   macro avg       0.80      0.79      0.79     12852\n",
            "weighted avg       0.79      0.79      0.79     12852\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Gz-wHAw3xMk"
      },
      "source": [
        "#### **Inference**\n",
        "\n",
        "The fun part is when we can quickly test the model on new, unseen sentences. \n",
        "Here, we use the prediction of the **first word piece of every word**. Note that the function we used to train our model (`tokenze_and_preserve_labels`) propagated the label to all subsequent word pieces (so you could for example also perform a majority vote on the predicted labels of all word pieces of a word).\n",
        "\n",
        "*In other words, the code below does not take into account when predictions of different word pieces that belong to the same word do not match.*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zPDla1mmZiax",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c949f7be-aa27-4ba4-8f8f-2e6b17c02ac7"
      },
      "source": [
        "sentence = \"India has a capital called Mumbai. On wednesday, the president will give a presentation\"\n",
        "\n",
        "inputs = tokenizer(sentence, padding='max_length', truncation=True, max_length=MAX_LEN, return_tensors=\"pt\")\n",
        "\n",
        "# move to gpu\n",
        "ids = inputs[\"input_ids\"].to(device)\n",
        "mask = inputs[\"attention_mask\"].to(device)\n",
        "# forward pass\n",
        "outputs = model(ids, mask)\n",
        "logits = outputs[0]\n",
        "\n",
        "active_logits = logits.view(-1, model.num_labels) # shape (batch_size * seq_len, num_labels)\n",
        "flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size*seq_len,) - predictions at the token level\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(ids.squeeze().tolist())\n",
        "token_predictions = [id2label[i] for i in flattened_predictions.cpu().numpy()]\n",
        "wp_preds = list(zip(tokens, token_predictions)) # list of tuples. Each tuple = (wordpiece, prediction)\n",
        "\n",
        "word_level_predictions = []\n",
        "for pair in wp_preds:\n",
        "  if (pair[0].startswith(\" ##\")) or (pair[0] in ['[CLS]', '[SEP]', '[PAD]']):\n",
        "    # skip prediction\n",
        "    continue\n",
        "  else:\n",
        "    word_level_predictions.append(pair[1])\n",
        "\n",
        "# we join tokens, if they are not special ones\n",
        "str_rep = \" \".join([t[0] for t in wp_preds if t[0] not in ['[CLS]', '[SEP]', '[PAD]']]).replace(\" ##\", \"\")\n",
        "print(str_rep)\n",
        "print(word_level_predictions)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "india has a capital called mumbai . on wednesday , the president will give a presentation\n",
            "['B-geo', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'B-tim', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note that there's another way to easily perform quick inference with a trained model: the [pipeline API](https://huggingface.co/docs/transformers/main_classes/pipelines). The pipeline API abstracts away all the complexity for you (basically performing what we did above). Here, we'll use the [TokenClassificationPipeline](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/pipelines#transformers.TokenClassificationPipeline) since that's the task we're doing, and we provide a model and tokenizer."
      ],
      "metadata": {
        "id": "bDIqKc-9cXBl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "pipe = pipeline(task=\"token-classification\", model=model.to(\"cpu\"), tokenizer=tokenizer, aggregation_strategy=\"simple\")\n",
        "pipe(\"My name is Niels and New York is a city\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D5KB5TKRcdRT",
        "outputId": "09b500b8-dcfc-4183-c5ae-3611868258f5"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'end': None,\n",
              "  'entity_group': 'per',\n",
              "  'score': 0.71306735,\n",
              "  'start': None,\n",
              "  'word': 'ni'},\n",
              " {'end': None,\n",
              "  'entity_group': 'per',\n",
              "  'score': 0.8921481,\n",
              "  'start': None,\n",
              "  'word': '##els'},\n",
              " {'end': None,\n",
              "  'entity_group': 'geo',\n",
              "  'score': 0.95278734,\n",
              "  'start': None,\n",
              "  'word': 'new york'}]"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqDklprSqB5d"
      },
      "source": [
        "#### **Saving the model for future use**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VuUdX_fImswO"
      },
      "source": [
        "Finally, let's save the model and tokenizer files such that we can easily re-use them later on. There are 2 options:\n",
        "\n",
        "* you can save everything locally, simply by calling `model.save_pretrained()` and `tokenizer.save_pretrained()`, providing a directory path as argument. \n",
        "* you can push the files to the [HuggingFace hub](https://huggingface.co/). This way, you can share your model with the community/your colleagues. All files will be tracked by git, as each model on the hub has its own git repo.\n",
        "\n",
        "Both options allow to re-use the model/tokenizer using the `from_pretrained()` method. Here we'll do the latter. \n",
        "\n",
        "To upload a model to the hub, 2 things need to be setup:\n",
        "* install git-LFS, which is used by the hub\n",
        "* set up authentication token\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!sudo apt-get install git-lfs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZVw5wjcwfGHd",
        "outputId": "09b28f6c-e2e9-4c62-9276-10f5fc38e9bd"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  git-lfs\n",
            "0 upgraded, 1 newly installed, 0 to remove and 39 not upgraded.\n",
            "Need to get 2,129 kB of archives.\n",
            "After this operation, 7,662 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 git-lfs amd64 2.3.4-1 [2,129 kB]\n",
            "Fetched 2,129 kB in 2s (899 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 1.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package git-lfs.\n",
            "(Reading database ... 155332 files and directories currently installed.)\n",
            "Preparing to unpack .../git-lfs_2.3.4-1_amd64.deb ...\n",
            "Unpacking git-lfs (2.3.4-1) ...\n",
            "Setting up git-lfs (2.3.4-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386,
          "referenced_widgets": [
            "3f0c39538ccc43eb800099ee1e8aaaf9",
            "f2003724d2f9402291b90cf643debde3",
            "d4f35d8c9a2c43b0a293abc1b6bfabea",
            "f0a1fefe73bb4c97a8082a70f79da8b2",
            "b5035eff71e94419815074dcf523291d",
            "a80d70df10194fbca3149e66aaf08c78",
            "45a51b4b0b1049c8981fa8eab19d6b11",
            "c79560a41bb54490ba870245062560f8",
            "90b362e5517049688e67b61d4a8acfe4",
            "ae1235da1aab4859a24974c1592363df",
            "8198e84854b94ac591bc2cfedc1771ae",
            "d3e0b329daf6496fb79323bf120a44bd",
            "9fa72d570d29489e8f0b7e57634ef92a",
            "7fe522823a8840c7aea0c9728054f78b",
            "455be84e2e184e37a425753e7106653e",
            "558544f011614ea590d1abeb79689b20",
            "6dceb012fcdf4d8b942909d28821c633"
          ]
        },
        "id": "yxx9U4VgfJ7L",
        "outputId": "51e28986-38ae-476f-81c4-480d7048f7ad"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Login successful\n",
            "Your token has been saved to /root/.huggingface/token\n",
            "\u001b[1m\u001b[31mAuthenticated through git-credential store but this isn't the helper defined on your machine.\n",
            "You might have to re-authenticate when pushing to the Hugging Face Hub. Run the following command in your terminal in case you want to set this credential helper as the default\n",
            "\n",
            "git config --global credential.helper store\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sDZtSsKKntuI",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301,
          "referenced_widgets": [
            "b2233d8919b64cfcb6fa898c400b5673",
            "35add86a7ae34843bc18db2da48b3255",
            "96587923209f4a958d340d2992e16c3b",
            "0c4f1a279f2f47c897947f426ddc1569",
            "cc8464c15946471f9863040358c9b071",
            "7ee8cc9ac56f4b2cadb73a99bc064c5b",
            "198b3b068e54406da2ab611a19ae8008",
            "0ae79201bd3a432bad399ad26427ecc2",
            "1208952534f143b994c3c7087c194b51",
            "a60e72178fed4eefb0fee3c6d3f2c1fb",
            "d4afd39e7f1d493d9db20b1fa852eb5b"
          ]
        },
        "outputId": "130ba8cc-5c85-47cf-fee6-9974697fe0fd"
      },
      "source": [
        "model_name = \"bert-finetuned-ner\"\n",
        "\n",
        "# upload files to the hub\n",
        "tokenizer.push_to_hub(\n",
        "    repo_path_or_name=model_name,\n",
        "    organization=\"nielsr\",\n",
        "    commit_message=\"Add tokenizer\",\n",
        "    use_temp_dir=True,\n",
        ")\n",
        "model.push_to_hub(\n",
        "    repo_path_or_name=model_name,\n",
        "    organization=\"nielsr\",\n",
        "    commit_message=\"Add model\",\n",
        "    use_temp_dir=True,\n",
        ")"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/huggingface_hub/hf_api.py:1004: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
            "  FutureWarning,\n",
            "Cloning https://huggingface.co/nielsr/bert-finetuned-ner into local empty directory.\n",
            "To https://huggingface.co/nielsr/bert-finetuned-ner\n",
            "   7fb33a6..12404bd  main -> main\n",
            "\n",
            "/usr/local/lib/python3.7/dist-packages/huggingface_hub/hf_api.py:1004: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
            "  FutureWarning,\n",
            "Cloning https://huggingface.co/nielsr/bert-finetuned-ner into local empty directory.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b2233d8919b64cfcb6fa898c400b5673",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Upload file pytorch_model.bin:   0%|          | 3.39k/415M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "To https://huggingface.co/nielsr/bert-finetuned-ner\n",
            "   12404bd..77117f5  main -> main\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'https://huggingface.co/nielsr/bert-finetuned-ner/commit/77117f560fc015ea11bb2b1f0496bfc09cf792b3'"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can check the model on the hub: https://huggingface.co/nielsr/bert-finetuned-ner. Awesome, isn't it? Check the \"files and versions\" tab, it includes all our files :) we can now load it back as follows (here I'm using the [Auto API](https://huggingface.co/docs/transformers/model_doc/auto) - which will load the appropriate tokenizer + model for us):"
      ],
      "metadata": {
        "id": "ZlPINMg3hQQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
        "\n",
        "model_name = \"nielsr/bert-finetuned-ner\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModelForTokenClassification.from_pretrained(model_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "b45067ba34c745078632a7ed1ecfcfde",
            "e1055dc6d54c46d68b6b6ad3fd523d8b",
            "0101cdf1aae54ee9be3fab68706ffb44",
            "9b990881eb394e998149c3d92065dde3",
            "7d223f7b9c9c420393f7daa409382119",
            "83319745e64f48e3a5446e86d441a0e2",
            "a52852a97c284a77b5ce1e8683745a73",
            "ed3cd1339e8347a3aa9c4eb9af3fa042",
            "913e92e121f04aaaa1591c24ad2aaa0d",
            "579240ba60aa4352a4ab4be9c754a061",
            "20d4708da2b444bd9886edfff45e8f1b",
            "f163ad4caa4348d98765f0058d8ebaa0",
            "d733118516b14c9e98104eb0bf88650f",
            "83620b5680a245169672012f68b588a9",
            "171a6288d6b944ebbb591e7bed62d152",
            "6f32331b10a14d96a5e649d2368fbe0e",
            "5a632929f2a8485eb02b1dd9b339f348",
            "8cfd1e69f8824a1db8348c7fd5e21ede",
            "14f370fb3da1464c9e880cd0075f1299",
            "d53f286986d948fb84658b0e37b3354a",
            "5c6f67ff349f4f35954d93a02250f5dd",
            "ee787678bc414ec5bd099d04ac5817f8",
            "20fb4d560bca4edea60c047ba1e0de78",
            "4cea747e3f464f6fbe0555822eccfc14",
            "1a2dc487acc94df1811d78ead637d04c",
            "6a0143b1d2e24c849bc0897c9778cf13",
            "43bbd28475be4e47a7e041da4675743b",
            "5ca95ca54fba454aa71a81533346647a",
            "8654fb2c0edd4b74b8f52cf5a7c3a40d",
            "d36262841f154e17a2751025a4a2834c",
            "954936e2061e4dc39f759e68589c4e39",
            "e33f7a7f413a4f4f9afac764f73d27d4",
            "7ae9246c54b447f1948378269ca8e6ac",
            "2d3fb9f31e5b474e8893a8ede56d16b1",
            "7deead08b65c434db06cdde6c2e72232",
            "c96156ea9c1e49aab7c065ebbfc0537f",
            "0360c745f569434190a7a72514eea049",
            "47fa90493b9c4e74b2d933934b1a5702",
            "add89a77779441ef883ad6106e4184d9",
            "f43f095774df4a75b95a5ee11472c64a",
            "1460bbcdf06540a4912d17cdf3d4043c",
            "201ec7a20b6e471bbccda65bc520ac24",
            "068cad028be244b9ab5e26dff7908935",
            "bf624b00ab134acda8bb505844731081",
            "0f25b7d5f21646e3ae6e34682de0e559",
            "c51b5a92894840e58736a106f521ea8d",
            "252090bca3794efca06337bb0bced176",
            "d7b0c9293306465da88262e66c4bb1cf",
            "91cb71ee71814869b986f307d8e6caac",
            "f1821d9437874771bf3a5f6e01597749",
            "17983bd0617c4b1f8e35dd12cbaad18e",
            "fbf9c204f1bb4439ad33aa95a6e52562",
            "999fba878ce940928cbfb35cef43c9b8",
            "33d0f4dc3b2d4f428aa858ac8f76c0b7",
            "281efa82b7624d4d9c441c9691924757"
          ]
        },
        "id": "lGTAfo8bhXrW",
        "outputId": "eb5ad6c5-2e14-4d0f-868c-4b3b94f7ebea"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b45067ba34c745078632a7ed1ecfcfde",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/369 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f163ad4caa4348d98765f0058d8ebaa0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20fb4d560bca4edea60c047ba1e0de78",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d3fb9f31e5b474e8893a8ede56d16b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.06k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0f25b7d5f21646e3ae6e34682de0e559",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/415M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}