ner.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Best Model - COMP5046A2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d1ae17a7e62440ba8820680360083c80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4509574f47a349d88ceaf843be6e021f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8ab8588bb416418ea153a682ca03f2cb",
              "IPY_MODEL_0c957a0c96324e7bb1d321abb70549ee"
            ]
          }
        },
        "4509574f47a349d88ceaf843be6e021f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ab8588bb416418ea153a682ca03f2cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_97da95fbe8b646498dff0f9b9884cba9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ef3f6221b991438e8f7c41ee3e70d548"
          }
        },
        "0c957a0c96324e7bb1d321abb70549ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e16a03662bcd42478138a46dad88b5ac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 1.00kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ebd2a5d2b9704369b4844eb81b3e2a2e"
          }
        },
        "97da95fbe8b646498dff0f9b9884cba9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ef3f6221b991438e8f7c41ee3e70d548": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e16a03662bcd42478138a46dad88b5ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ebd2a5d2b9704369b4844eb81b3e2a2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c4d8469a4a0421e85b3d1984c500bf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_be8dd487b6a147899868c36f839b518d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbd71ccedd774ea49228137f002f23c1",
              "IPY_MODEL_c6c1076226ac42f5a7397429b2ea8e27"
            ]
          }
        },
        "be8dd487b6a147899868c36f839b518d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbd71ccedd774ea49228137f002f23c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_218f6b97e75d464a90c2b2614b91bf6a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0527885a29b4c409e2f5a6ea24c01c9"
          }
        },
        "c6c1076226ac42f5a7397429b2ea8e27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e708d10a6a99463493149c909de37137",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:02&lt;00:00, 115kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9bfc3e8524d48cebf9b9581b5b53e23"
          }
        },
        "218f6b97e75d464a90c2b2614b91bf6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0527885a29b4c409e2f5a6ea24c01c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e708d10a6a99463493149c909de37137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9bfc3e8524d48cebf9b9581b5b53e23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e045b0d4fe0e48349482831434acf6a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_90c747637bf74b43a58f2c2b5eac8d9a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0c0dcb492f3044e3bdee77803a035494",
              "IPY_MODEL_5ce24df45bdc4ce5842422500d8cb531"
            ]
          }
        },
        "90c747637bf74b43a58f2c2b5eac8d9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c0dcb492f3044e3bdee77803a035494": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_07aed3f703d5457b8d0ac2acd5e95c54",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a63c4a3b8174b0499ce026eb0ddb373"
          }
        },
        "5ce24df45bdc4ce5842422500d8cb531": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d1bc6fc9759d4f17ae7e415cbea99cf8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:01&lt;00:00, 354kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a822cf47e974892951050d180fbd665"
          }
        },
        "07aed3f703d5457b8d0ac2acd5e95c54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a63c4a3b8174b0499ce026eb0ddb373": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d1bc6fc9759d4f17ae7e415cbea99cf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a822cf47e974892951050d180fbd665": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6fc42d301203491eb04e15bccbd025e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_621f29c642184281bea825307ec172ec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e2e350772bab4a2e87d3eae95c4227ba",
              "IPY_MODEL_6f43d8fd1bab489795176ef5e76025d6"
            ]
          }
        },
        "621f29c642184281bea825307ec172ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e2e350772bab4a2e87d3eae95c4227ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c14fc45122d84357a32ef5218b56811f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a38f29db22a47d1b72146be820942bc"
          }
        },
        "6f43d8fd1bab489795176ef5e76025d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd7362fd89c54784a45dc8cd05ec737a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.06kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_99f0cb3370df48ce9c4f872c8af5ec90"
          }
        },
        "c14fc45122d84357a32ef5218b56811f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a38f29db22a47d1b72146be820942bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd7362fd89c54784a45dc8cd05ec737a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "99f0cb3370df48ce9c4f872c8af5ec90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08432afde20044aba54f7fef0b82d5db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3725063c34894febbfadcfc8b0599601",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3112f35202c64b6a8dd7fd2abc641113",
              "IPY_MODEL_e97c36468f9b44c6a5870a193c6cb68d"
            ]
          }
        },
        "3725063c34894febbfadcfc8b0599601": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3112f35202c64b6a8dd7fd2abc641113": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8e47c0682da6475ab5ec2178c6725096",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5163c0a80fa2479b8c58202383403a78"
          }
        },
        "e97c36468f9b44c6a5870a193c6cb68d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6fa5490aff3c4190b651e69d74359330",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:07&lt;00:00, 57.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b3c3e8d85cb54029acb97cedc41c5538"
          }
        },
        "8e47c0682da6475ab5ec2178c6725096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5163c0a80fa2479b8c58202383403a78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6fa5490aff3c4190b651e69d74359330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b3c3e8d85cb54029acb97cedc41c5538": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PCI9BOhVs1FD"
      },
      "source": [
        "# Load Packages and Dependency\n",
        "Please restart the environment after runing the following two sections."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4axnA8IV9IzH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9b7daf1-5ef8-4337-dfa5-0c1ca12934ad"
      },
      "source": [
        "import spacy\n",
        "!pip uninstall spacy"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling spacy-2.2.4:\n",
            "  Would remove:\n",
            "    /usr/local/bin/spacy\n",
            "    /usr/local/lib/python3.7/dist-packages/bin/*\n",
            "    /usr/local/lib/python3.7/dist-packages/spacy-2.2.4.dist-info/*\n",
            "    /usr/local/lib/python3.7/dist-packages/spacy/*\n",
            "  Would not remove (might be manually added):\n",
            "    /usr/local/lib/python3.7/dist-packages/bin/theano_cache.py\n",
            "Proceed (y/n)? y\n",
            "  Successfully uninstalled spacy-2.2.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g-46oZMEzMHE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "07be0751-d37e-4280-eaed-09757c26c4e6"
      },
      "source": [
        "!pip install flair\n",
        "!pip install spacy\n",
        "!python -m spacy download en_core_web_trf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting flair\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f0/3a/1b46a0220d6176b22bcb9336619d1731301bc2c75fa926a9ef953e6e4d58/flair-0.8.0.post1-py3-none-any.whl (284kB)\n",
            "\r\u001b[K     |█▏                              | 10kB 23.2MB/s eta 0:00:01\r\u001b[K     |██▎                             | 20kB 23.9MB/s eta 0:00:01\r\u001b[K     |███▌                            | 30kB 17.0MB/s eta 0:00:01\r\u001b[K     |████▋                           | 40kB 14.5MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 51kB 8.2MB/s eta 0:00:01\r\u001b[K     |███████                         | 61kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████                        | 71kB 9.1MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 81kB 8.7MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 92kB 9.6MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 102kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 112kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 122kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 133kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████                | 143kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 153kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 163kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 174kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 184kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 194kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 204kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 215kB 8.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 225kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 235kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 245kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 256kB 8.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 266kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 276kB 8.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 286kB 8.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.41.1)\n",
            "Collecting huggingface-hub\n",
            "  Downloading https://files.pythonhosted.org/packages/3c/e3/fb7b6aefaf0fc7b792cebbbd590b1895c022ab0ff27f389e1019c6f2e68a/huggingface_hub-0.0.10-py3-none-any.whl\n",
            "Collecting segtok>=1.5.7\n",
            "  Downloading https://files.pythonhosted.org/packages/41/08/582dab5f4b1d5ca23bc6927b4bb977c8ff7f3a87a3b98844ef833e2f5623/segtok-1.5.10.tar.gz\n",
            "Collecting janome\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a8/63/98858cbead27df7536c7e300c169da0999e9704d02220dc6700b804eeff0/Janome-0.4.1-py2.py3-none-any.whl (19.7MB)\n",
            "\u001b[K     |████████████████████████████████| 19.7MB 1.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from flair) (0.1.2)\n",
            "Collecting transformers>=4.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/43/cfe4ee779bbd6a678ac6a97c5a5cdeb03c35f9eaebbb9720b036680f9a2d/transformers-4.6.1-py3-none-any.whl (2.2MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 47.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from flair) (3.6.0)\n",
            "Collecting deprecated>=1.2.4\n",
            "  Downloading https://files.pythonhosted.org/packages/fb/73/994edfcba74443146c84b91921fcc269374354118d4f452fb0c54c1cbb12/Deprecated-1.2.12-py2.py3-none-any.whl\n",
            "Collecting sentencepiece==0.1.95\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 67.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from flair) (0.22.2.post1)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from flair) (4.2.6)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from flair) (0.8.9)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from flair) (2019.12.20)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from flair) (2.8.1)\n",
            "Collecting langdetect\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 67.6MB/s \n",
            "\u001b[?25hCollecting sqlitedict>=1.6.0\n",
            "  Downloading https://files.pythonhosted.org/packages/5c/2d/b1d99e9ad157dd7de9cd0d36a8a5876b13b55e4b75f7498bc96035fb4e96/sqlitedict-1.7.0.tar.gz\n",
            "Collecting konoha<5.0.0,>=4.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/71/70/48a0bd55f79c328504fe6fe7ae8ff651f77a2aadbb1911701385d9bb5ca3/konoha-4.6.5-py3-none-any.whl\n",
            "Collecting gdown==3.12.2\n",
            "  Downloading https://files.pythonhosted.org/packages/50/21/92c3cfe56f5c0647145c4b0083d0733dd4890a057eb100a8eeddf949ffe9/gdown-3.12.2.tar.gz\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting torch<=1.7.1,>=1.5.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/90/5d/095ddddc91c8a769a68c791c019c5793f9c4456a688ddd235d6670924ecb/torch-1.7.1-cp37-cp37m-manylinux1_x86_64.whl (776.8MB)\n",
            "\u001b[K     |████████████████████████████████| 776.8MB 23kB/s \n",
            "\u001b[?25hCollecting bpemb>=0.3.2\n",
            "  Downloading https://files.pythonhosted.org/packages/f2/6f/9191b85109772636a8f8accb122900c34db26c091d2793218aa94954524c/bpemb-0.3.3-py3-none-any.whl\n",
            "Collecting ftfy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/da/d215a091986e5f01b80f5145cff6f22e2dc57c6b048aab2e882a07018473/ftfy-6.0.3.tar.gz (64kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 10.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.7/dist-packages (from flair) (3.2.2)\n",
            "Collecting mpld3==0.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/95/a52d3a83d0a29ba0d6898f6727e9858fe7a43f6c2ce81a5fe7e05f0f4912/mpld3-0.3.tar.gz (788kB)\n",
            "\u001b[K     |████████████████████████████████| 798kB 59.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy<1.20.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub->flair) (3.7.4.3)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from huggingface-hub->flair) (4.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub->flair) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from huggingface-hub->flair) (2.23.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (2.5.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (1.15.0)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (3.11.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (1.4.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 50.1MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 59.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (20.9)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (5.0.0)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.7/dist-packages (from deprecated>=1.2.4->flair) (1.12.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->flair) (1.0.1)\n",
            "Collecting overrides<4.0.0,>=3.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/ff/b1/10f69c00947518e6676bbd43e739733048de64b8dd998e9c2d5a71f44c5d/overrides-3.1.0.tar.gz\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->flair) (0.2.5)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (1.3.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->huggingface-hub->flair) (3.4.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->huggingface-hub->flair) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->huggingface-hub->flair) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->huggingface-hub->flair) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->huggingface-hub->flair) (2.10)\n",
            "Requirement already satisfied: decorator<5,>=4.3 in /usr/local/lib/python3.7/dist-packages (from networkx->hyperopt>=0.1.1->flair) (4.4.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers>=4.0.0->flair) (7.1.2)\n",
            "Building wheels for collected packages: gdown\n",
            "  Building wheel for gdown (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gdown: filename=gdown-3.12.2-cp37-none-any.whl size=9705 sha256=c76d491c9bce991c4ec96a431616cab75575c52711013b48b8e116985d2d6cee\n",
            "  Stored in directory: /root/.cache/pip/wheels/81/d0/d7/d9983facc6f2775411803e0e2d30ebf98efbf2fc6e57701e09\n",
            "Successfully built gdown\n",
            "Building wheels for collected packages: segtok, langdetect, sqlitedict, ftfy, mpld3, overrides\n",
            "  Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for segtok: filename=segtok-1.5.10-cp37-none-any.whl size=25031 sha256=d2bd1ea9a82eaa71e11bb253333fffc69c3177b78e66bc480fefd99fe4c19100\n",
            "  Stored in directory: /root/.cache/pip/wheels/b4/39/f6/9ca1c5cabde964d728023b5751c3a206a5c8cc40252321fb6b\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-cp37-none-any.whl size=993242 sha256=dacdc7abb098812b04554572f2b504d935f76e004911bf9f8697094d1c62ba4c\n",
            "  Stored in directory: /root/.cache/pip/wheels/7e/18/13/038c34057808931c7ddc6c92d3aa015cf1a498df5a70268996\n",
            "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sqlitedict: filename=sqlitedict-1.7.0-cp37-none-any.whl size=14393 sha256=ff4598d6402ddd67d4249b0e0e31a919849d2c8ddff9eba1979b141b8a86941e\n",
            "  Stored in directory: /root/.cache/pip/wheels/cf/c6/4f/2c64a43f041415eb8b8740bd80e15e92f0d46c5e464d8e4b9b\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ftfy: filename=ftfy-6.0.3-cp37-none-any.whl size=41935 sha256=f36b649bd95fd112c4ddab04e842208490231126f4e893501d334dfd785939ec\n",
            "  Stored in directory: /root/.cache/pip/wheels/99/2c/e6/109c8a28fef7a443f67ba58df21fe1d0067ac3322e75e6b0b7\n",
            "  Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mpld3: filename=mpld3-0.3-cp37-none-any.whl size=116704 sha256=2a957cdda5081166087a1e9fb5a690344738fa739bbe134979f6ae6525e720b6\n",
            "  Stored in directory: /root/.cache/pip/wheels/c0/47/fb/8a64f89aecfe0059830479308ad42d62e898a3e3cefdf6ba28\n",
            "  Building wheel for overrides (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for overrides: filename=overrides-3.1.0-cp37-none-any.whl size=10187 sha256=6a4bc2c204c45e0b5416677decdb70be7a56cd41d9c91c95ecd3b33295506c4c\n",
            "  Stored in directory: /root/.cache/pip/wheels/5c/24/13/6ef8600e6f147c95e595f1289a86a3cc82ed65df57582c65a9\n",
            "Successfully built segtok langdetect sqlitedict ftfy mpld3 overrides\n",
            "\u001b[31mERROR: fastai 1.0.61 requires spacy>=2.0.18; python_version < \"3.8\", which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: torchvision 0.9.1+cu101 has requirement torch==1.8.1, but you'll have torch 1.7.1 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: torchtext 0.9.1 has requirement torch==1.8.1, but you'll have torch 1.7.1 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: transformers 4.6.1 has requirement huggingface-hub==0.0.8, but you'll have huggingface-hub 0.0.10 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: konoha 4.6.5 has requirement importlib-metadata<4.0.0,>=3.7.0, but you'll have importlib-metadata 4.0.1 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: konoha 4.6.5 has requirement requests<3.0.0,>=2.25.1, but you'll have requests 2.23.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: huggingface-hub, segtok, janome, sacremoses, tokenizers, transformers, deprecated, sentencepiece, langdetect, sqlitedict, overrides, konoha, gdown, torch, bpemb, ftfy, mpld3, flair\n",
            "  Found existing installation: gdown 3.6.4\n",
            "    Uninstalling gdown-3.6.4:\n",
            "      Successfully uninstalled gdown-3.6.4\n",
            "  Found existing installation: torch 1.8.1+cu101\n",
            "    Uninstalling torch-1.8.1+cu101:\n",
            "      Successfully uninstalled torch-1.8.1+cu101\n",
            "Successfully installed bpemb-0.3.3 deprecated-1.2.12 flair-0.8.0.post1 ftfy-6.0.3 gdown-3.12.2 huggingface-hub-0.0.10 janome-0.4.1 konoha-4.6.5 langdetect-1.0.9 mpld3-0.3 overrides-3.1.0 sacremoses-0.0.45 segtok-1.5.10 sentencepiece-0.1.95 sqlitedict-1.7.0 tokenizers-0.10.3 torch-1.7.1 transformers-4.6.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "torch"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Collecting spacy\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1b/d8/0361bbaf7a1ff56b44dca04dace54c82d63dad7475b7d25ea1baefafafb2/spacy-3.0.6-cp37-cp37m-manylinux2014_x86_64.whl (12.8MB)\n",
            "\u001b[K     |████████████████████████████████| 12.8MB 8.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.8.2)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from spacy) (3.7.4.3)\n",
            "Collecting pydantic<1.8.0,>=1.7.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/fa/d43f31874e1f2a9633e4c025be310f2ce7a8350017579e9e837a62630a7e/pydantic-1.7.4-cp37-cp37m-manylinux2014_x86_64.whl (9.1MB)\n",
            "\u001b[K     |████████████████████████████████| 9.1MB 51.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.11.3)\n",
            "Collecting thinc<8.1.0,>=8.0.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/55/e5/6820eccc01d6d8b1d87c3bd021321516af572dcd551e41712913f880f58f/thinc-8.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (618kB)\n",
            "\u001b[K     |████████████████████████████████| 624kB 53.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (57.0.0)\n",
            "Collecting catalogue<2.1.0,>=2.0.3\n",
            "  Downloading https://files.pythonhosted.org/packages/9c/10/dbc1203a4b1367c7b02fddf08cb2981d9aa3e688d398f587cea0ab9e3bec/catalogue-2.0.4-py3-none-any.whl\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.41.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (20.9)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.19.5)\n",
            "Collecting srsly<3.0.0,>=2.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c3/84/dfdfc9f6f04f6b88207d96d9520b911e5fec0c67ff47a0dea31ab5429a1e/srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl (456kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 57.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.5)\n",
            "Collecting typer<0.4.0,>=0.3.0\n",
            "  Downloading https://files.pythonhosted.org/packages/90/34/d138832f6945432c638f32137e6c79a3b682f06a63c488dcfaca6b166c64/typer-0.3.2-py3-none-any.whl\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.5)\n",
            "Collecting spacy-legacy<3.1.0,>=3.0.4\n",
            "  Downloading https://files.pythonhosted.org/packages/8d/67/d4002a18e26bf29b17ab563ddb55232b445ab6a02f97bf17d1345ff34d3f/spacy_legacy-3.0.5-py2.py3-none-any.whl\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.23.0)\n",
            "Collecting pathy>=0.3.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/87/5991d87be8ed60beb172b4062dbafef18b32fa559635a8e2b633c2974f85/pathy-0.5.2-py3-none-any.whl (42kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy) (2.0.1)\n",
            "Requirement already satisfied: zipp>=0.5; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.3->spacy) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy) (2.4.7)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy) (7.1.2)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Collecting smart-open<4.0.0,>=2.2.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/11/9a/ba2d5f67f25e8d5bbf2fcec7a99b1e38428e83cb715f64dd179ca43a11bb/smart_open-3.0.0.tar.gz (113kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 67.5MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: smart-open\n",
            "  Building wheel for smart-open (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for smart-open: filename=smart_open-3.0.0-cp37-none-any.whl size=107107 sha256=a02fdb1016ba8adcefac05d4beea62b6e43cf4fde3ee635442a6af82971a65ed\n",
            "  Stored in directory: /root/.cache/pip/wheels/18/88/7c/f06dabd5e9cabe02d2269167bcacbbf9b47d0c0ff7d6ebcb78\n",
            "Successfully built smart-open\n",
            "Installing collected packages: pydantic, catalogue, srsly, thinc, typer, spacy-legacy, smart-open, pathy, spacy\n",
            "  Found existing installation: catalogue 1.0.0\n",
            "    Uninstalling catalogue-1.0.0:\n",
            "      Successfully uninstalled catalogue-1.0.0\n",
            "  Found existing installation: srsly 1.0.5\n",
            "    Uninstalling srsly-1.0.5:\n",
            "      Successfully uninstalled srsly-1.0.5\n",
            "  Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Found existing installation: smart-open 5.0.0\n",
            "    Uninstalling smart-open-5.0.0:\n",
            "      Successfully uninstalled smart-open-5.0.0\n",
            "Successfully installed catalogue-2.0.4 pathy-0.5.2 pydantic-1.7.4 smart-open-3.0.0 spacy-3.0.6 spacy-legacy-3.0.5 srsly-2.4.1 thinc-8.0.4 typer-0.3.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "catalogue",
                  "spacy",
                  "srsly",
                  "thinc"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2021-06-13 11:00:42.724294: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n",
            "Collecting en-core-web-trf==3.0.0\n",
            "\u001b[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.0.0/en_core_web_trf-3.0.0-py3-none-any.whl (459.7MB)\n",
            "\u001b[K     |████████████████████████████████| 459.7MB 31kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.1.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-trf==3.0.0) (3.0.6)\n",
            "Collecting spacy-transformers<1.1.0,>=1.0.0rc4\n",
            "  Downloading https://files.pythonhosted.org/packages/e8/c5/a156f9c979cc14f5f41cf2e6ecfc55d1128ac0363930ec7cc6fe4d98b4a2/spacy_transformers-1.0.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (1.0.5)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (0.4.1)\n",
            "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (1.7.4)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.0.4)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.0.5)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (4.41.1)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.4.1)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (8.0.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.11.3)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (1.19.5)\n",
            "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (0.3.2)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.0.5)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.7.4.3)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (0.5.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (57.0.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (20.9)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (0.8.2)\n",
            "Collecting spacy-alignments<1.0.0,>=0.7.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/86/a6786d24d1d8f3a6cff2c60b55a7e845725a94919cd94d270ea49d82e59b/spacy_alignments-0.8.3-cp37-cp37m-manylinux2014_x86_64.whl (998kB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (1.7.1)\n",
            "Collecting transformers<4.6.0,>=3.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 14.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.3->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.4.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (1.24.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.0.1)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (7.1.2)\n",
            "Requirement already satisfied: smart-open<4.0.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (3.0.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->en-core-web-trf==3.0.0) (2.4.7)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (4.0.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (0.0.45)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (3.0.12)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (0.10.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (2019.12.20)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<4.6.0,>=3.4.0->spacy-transformers<1.1.0,>=1.0.0rc4->en-core-web-trf==3.0.0) (1.0.1)\n",
            "Installing collected packages: spacy-alignments, transformers, spacy-transformers, en-core-web-trf\n",
            "  Found existing installation: transformers 4.6.1\n",
            "    Uninstalling transformers-4.6.1:\n",
            "      Successfully uninstalled transformers-4.6.1\n",
            "Successfully installed en-core-web-trf-3.0.0 spacy-alignments-0.8.3 spacy-transformers-1.0.2 transformers-4.5.1\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_trf')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j5JjrrrR983p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17667be0-94e7-4042-c6bf-0f9a4ae6f2b9"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import nltk\n",
        "import os\n",
        "import torch\n",
        "import gensim.downloader as api\n",
        "import gensim\n",
        "import gensim.corpora as corpora\n",
        "import torch\n",
        "import torch.autograd as autograd\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from flair.embeddings import TransformerWordEmbeddings, FlairEmbeddings, CharacterEmbeddings, StackedEmbeddings, WordEmbeddings\n",
        "from flair.data import Sentence\n",
        "import spacy\n",
        "from spacy.tokens import Doc\n",
        "from nltk.tag import hmm\n",
        "from nltk import word_tokenize\n",
        "nltk.download('brown')\n",
        "from nltk.corpus import brown\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/brown.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0LIPe7wJt1L_"
      },
      "source": [
        "# Data And Input"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NBfuuCoUs9Le"
      },
      "source": [
        "## Data Download and Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C5GreNAzs6xT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8889b313-11e5-44a3-c729-f486a9054c32"
      },
      "source": [
        "!pip install -U -q PyDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "# Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "if not os.path.exists('/content/train.csv'):\n",
        "    link = '1_M43BgxQXXy1L9W0kcUANS8NMuO2ta38'  # Restricted shared link\n",
        "    downloaded = drive.CreateFile({'id':link}) \n",
        "    downloaded.GetContentFile('train.csv')\n",
        "\n",
        "    link = '1HsguscJAhcKUmZ1huXrcWm4VvEIe6A0k'  # Restricted shared link\n",
        "    downloaded = drive.CreateFile({'id':link}) \n",
        "    downloaded.GetContentFile('val.csv')\n",
        "\n",
        "    link = '1bUI2EhX8_Rx1Qx56oQhcWAMSMday16BR'  # Restricted shared link\n",
        "    downloaded = drive.CreateFile({'id':link}) \n",
        "    downloaded.GetContentFile('test_without_labels.csv')\n",
        "\n",
        "    link = '1I6O-6a2yQYBf3EuimuA-M0BbmN6Jv_Eg'  # Restricted shared link\n",
        "    downloaded = drive.CreateFile({'id':link}) \n",
        "    downloaded.GetContentFile('sample.csv')\n",
        "\n",
        "train = pd.read_csv('/content/train.csv')\n",
        "val = pd.read_csv('/content/val.csv')\n",
        "test = pd.read_csv('/content/test_without_labels.csv')\n",
        "sample = pd.read_csv('/content/sample.csv')\n",
        "\n",
        "train.shape, val.shape, test.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((573, 2), (191, 2), (199, 1))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l9gBSgBCQh24"
      },
      "source": [
        "## Data Preprocessing\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TBwj99DJDbsz"
      },
      "source": [
        "### Converting to Lowercase\n",
        "This process can group words together the inflected forms of a word like 'Everything' and 'everything', so they can be analysed as a single item."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lh45o5Ir3CWK"
      },
      "source": [
        "train['sents'] = train['sents'].str.lower()\n",
        "val['sents'] = val['sents'].str.lower()\n",
        "test['sents'] = test['sents'].str.lower()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DyuNTMpY6AiM"
      },
      "source": [
        "### Intergrate Sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yvf59Zrm6Dhv"
      },
      "source": [
        "whole_sents = []    # (963 items)\n",
        "for s in train['sents']:\n",
        "    whole_sents.append(s)\n",
        "for s in val['sents']:\n",
        "    whole_sents.append(s)\n",
        "for s in test['sents']:\n",
        "    whole_sents.append(s)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2aT4fj0e3Ee",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "845d31c4-1e97-4ecb-b223-f6bb670931b1"
      },
      "source": [
        "max_seq_len = max(len(s.split(' ')) for s in whole_sents)\n",
        "max_seq_len # 154"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "154"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FtBbi3u14sg4"
      },
      "source": [
        "### Make Word Set & Word List"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F_iGYE2z4w7n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "95c2ad07-9d6e-4d23-d19b-a767057f0d6b"
      },
      "source": [
        "word_list = []\n",
        "for s in whole_sents:\n",
        "    word_list.extend(s.split(' '))\n",
        "\n",
        "word_set = set(word_list)\n",
        "word_list = list(word_set)\n",
        "word_list.sort()\n",
        "\n",
        "len(word_list) # 3957"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3957"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ymowFJWjmP0k"
      },
      "source": [
        "### Split Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hm4j4eDjg10a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33add89a-3df7-4783-86d8-10e283377857"
      },
      "source": [
        "train_data, target_y_train = [s.split(' ') for s in train['sents']], [s.split(' ') for s in train['labels']]\n",
        "validation_data, target_y_validation = [s.split(' ') for s in val['sents']], [s.split(' ') for s in val['labels']]\n",
        "test_data = [s.split(' ') for s in test['sents']]\n",
        "\n",
        "len(train_data), len(validation_data), len(test_data), "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(573, 191, 199)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_gAo_R2dm2fo"
      },
      "source": [
        "### Generate word_to_ix, tag_to_ix, ix_to_tag"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oJCsn2vYmoxn"
      },
      "source": [
        "word_to_ix = {}\n",
        "for sentence in train_data+validation_data+test_data:\n",
        "    for word in sentence:\n",
        "        word = word.lower()\n",
        "        if word not in word_to_ix:\n",
        "            word_to_ix[word] = len(word_to_ix)\n",
        "word_list = list(word_to_ix.keys())\n",
        "\n",
        "START_TAG = \"<START>\"\n",
        "STOP_TAG = \"<STOP>\"\n",
        "tag_to_ix = {START_TAG:0, STOP_TAG:1}\n",
        "for tags in target_y_train+target_y_validation:\n",
        "    for tag in tags:\n",
        "        if tag not in tag_to_ix:\n",
        "            tag_to_ix[tag] = len(tag_to_ix)\n",
        "ix_to_tag={}\n",
        "for tag in tag_to_ix:\n",
        "    ix_to_tag[tag_to_ix[tag]] = tag"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pa1i85kmnGQ7"
      },
      "source": [
        "### Convert Dataset into Idxs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-PuxN99rg8MF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "49e0cbf8-60d8-4894-e012-e5b2159fe7f8"
      },
      "source": [
        "def to_index(data, to_ix):\n",
        "    input_index_list = []\n",
        "    for sent in data:\n",
        "        input_index_list.append([to_ix[w] for w in sent])\n",
        "    return input_index_list\n",
        "\n",
        "train_input_index =  to_index(train_data,word_to_ix)\n",
        "train_output_index = to_index(target_y_train,tag_to_ix)\n",
        "val_input_index = to_index(validation_data,word_to_ix)\n",
        "val_output_index = to_index(target_y_validation,tag_to_ix)\n",
        "test_input_index = to_index(test_data,word_to_ix)\n",
        "len(train_input_index[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "52"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KwPt6HF1t-aV"
      },
      "source": [
        "## Input Embedding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QkFO8Lk36qNy"
      },
      "source": [
        "### Domain Embedding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GDXNBgiM-RZs"
      },
      "source": [
        "#### Download and process domain dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c3kFPbwd6qNz",
        "outputId": "0a9aa6c1-309e-404e-b677-94a15f71f117"
      },
      "source": [
        "# download the whole re3d dataset\n",
        "%cd /content\n",
        "!git clone https://github.com/juand-r/entity-recognition-datasets.git\n",
        "train_re3d = open(\"/content/entity-recognition-datasets/data/re3d/CONLL-format/data/train/re3d-train.conll\", \"r\")\n",
        "test_re3d = open(\"/content/entity-recognition-datasets/data/re3d/CONLL-format/data/test/re3d-test.conll\", \"r\")\n",
        "train_re3d = train_re3d.read().splitlines()\n",
        "test_re3d = test_re3d.read().splitlines()\n",
        "\n",
        "for i,v in enumerate(train_re3d):\n",
        "    train_re3d[i] = v.split('\\t')[0]\n",
        "for i,v in enumerate(test_re3d):\n",
        "    test_re3d[i] = v.split('\\t')[0]\n",
        "\n",
        "train_re3d = train_re3d[1:]\n",
        "test_re3d = test_re3d[1:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "Cloning into 'entity-recognition-datasets'...\n",
            "remote: Enumerating objects: 1050, done.\u001b[K\n",
            "remote: Counting objects: 100% (12/12), done.\u001b[K\n",
            "remote: Compressing objects: 100% (12/12), done.\u001b[K\n",
            "remote: Total 1050 (delta 6), reused 0 (delta 0), pack-reused 1038\u001b[K\n",
            "Receiving objects: 100% (1050/1050), 2.43 MiB | 23.92 MiB/s, done.\n",
            "Resolving deltas: 100% (425/425), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Tn4eBrr6qNz",
        "outputId": "a4a24a63-7bb4-4e49-84de-e9ca80679b1d"
      },
      "source": [
        "train_sents = []\n",
        "sent=''\n",
        "for i in train_re3d:\n",
        "    if i == '':\n",
        "        sent = sent[:-1]\n",
        "        train_sents.append(sent)\n",
        "        sent = ''\n",
        "    else:\n",
        "        sent += i\n",
        "        sent += ' '\n",
        "\n",
        "\n",
        "test_sents = []\n",
        "sent=''\n",
        "for i in test_re3d:\n",
        "    if i == '':\n",
        "        sent = sent[:-1]\n",
        "        test_sents.append(sent)\n",
        "        sent = ''\n",
        "    else:\n",
        "        sent += i\n",
        "        sent += ' '\n",
        "\n",
        "whole_sents = train_sents + test_sents\n",
        "len(whole_sents), len(train_sents), len(test_sents)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(963, 764, 199)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cgw0kk0V6qN0"
      },
      "source": [
        "if not os.path.exists(\"/content/corpus/train\"):\n",
        "    os.makedirs(\"/content/corpus/train\")\n",
        "f = open(\"/content/corpus/train/train_split_1.txt\",\"w\")\n",
        "f.writelines(\"%s\\n\" % l for l in whole_sents)\n",
        "f.close()\n",
        "\n",
        "f = open(\"/content/corpus/test.txt\",\"w\")\n",
        "f.writelines(\"%s\\n\" % l for l in test_sents)\n",
        "f.close()\n",
        "\n",
        "f = open(\"/content/corpus/valid.txt\",\"w\")\n",
        "f.writelines(\"%s\\n\" % l for l in test_sents)\n",
        "f.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lwzv2aq_-az4"
      },
      "source": [
        "#### Generate Domain Embeddings\n",
        "We train a Domain Embeddings object by `FlairEmbedding`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mR7HdJN36qN0",
        "outputId": "cf1f05c8-0126-457e-c5c2-472ac5b05f37"
      },
      "source": [
        "%cd /content\n",
        "from flair.data import Dictionary\n",
        "from flair.models import LanguageModel\n",
        "from flair.trainers.language_model_trainer import LanguageModelTrainer, TextCorpus\n",
        "\n",
        "# are you training a forward or backward LM?\n",
        "is_forward_lm = True\n",
        "\n",
        "# load the default character dictionary\n",
        "dictionary: Dictionary = Dictionary.load('chars')\n",
        "\n",
        "# get your corpus, process forward and at the character level\n",
        "corpus = TextCorpus('/content/corpus',\n",
        "                    dictionary,\n",
        "                    is_forward_lm,\n",
        "                    character_level=True)\n",
        "\n",
        "# instantiate your language model, set hidden size and number of layers\n",
        "language_model = LanguageModel(dictionary,\n",
        "                               is_forward_lm,\n",
        "                               hidden_size=128,\n",
        "                               nlayers=1)\n",
        "\n",
        "# train your language model\n",
        "trainer = LanguageModelTrainer(language_model, corpus)\n",
        "\n",
        "trainer.train('resources/taggers/language_model',\n",
        "              sequence_length=10,\n",
        "              mini_batch_size=10,\n",
        "              max_epochs=10)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "2021-06-13 11:04:10,220 https://flair.informatik.hu-berlin.de/resources/characters/common_characters not found in cache, downloading to /tmp/tmp149klhe_\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 2887/2887 [00:00<00:00, 1026008.78B/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-06-13 11:04:10,700 copying /tmp/tmp149klhe_ to cache at /root/.flair/datasets/common_characters\n",
            "2021-06-13 11:04:10,703 removing temp file /tmp/tmp149klhe_\n",
            "2021-06-13 11:04:10,705 read text file with 199 lines\n",
            "2021-06-13 11:04:10,877 read text file with 199 lines\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-06-13 11:04:13,273 read text file with 963 lines\n",
            "2021-06-13 11:04:13,277 shuffled\n",
            "2021-06-13 11:04:14,089 Sequence length is 10\n",
            "2021-06-13 11:04:14,094 Split 1\t - (11:04:14)\n",
            "2021-06-13 11:04:14,392 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.95 | loss  3.37 | ppl    28.97\n",
            "2021-06-13 11:04:14,680 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.86 | loss  2.73 | ppl    15.35\n",
            "2021-06-13 11:04:14,967 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.86 | loss  2.53 | ppl    12.58\n",
            "2021-06-13 11:04:15,246 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.78 | loss  2.41 | ppl    11.16\n",
            "2021-06-13 11:04:15,529 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.81 | loss  2.39 | ppl    10.96\n",
            "2021-06-13 11:04:15,817 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.87 | loss  2.39 | ppl    10.86\n",
            "2021-06-13 11:04:16,104 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.86 | loss  2.28 | ppl     9.74\n",
            "2021-06-13 11:04:16,388 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.82 | loss  2.27 | ppl     9.65\n",
            "2021-06-13 11:04:16,670 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.79 | loss  2.22 | ppl     9.20\n",
            "2021-06-13 11:04:16,956 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.85 | loss  2.17 | ppl     8.74\n",
            "2021-06-13 11:04:17,241 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.84 | loss  2.17 | ppl     8.71\n",
            "2021-06-13 11:04:17,522 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.80 | loss  2.16 | ppl     8.64\n",
            "2021-06-13 11:04:17,809 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.86 | loss  2.14 | ppl     8.53\n",
            "2021-06-13 11:04:17,962 3 seconds for train split 1\n",
            "2021-06-13 11:04:18,200 best loss so far 10000.00\n",
            "2021-06-13 11:04:19,045 ('\\nThe beeraterateration torenb took will it in vimuntairate awratmeaterate it al by ak , as todamce by oillerate abladouring the Syria : [ 4 ]\\nU.S. of oh tocemer lavatitaraty experatge .\\nFarined conflitilled a yelound saberate rewake .\\nThe a dilGae .\\nWam loow vaayC , anitient took , I a complate torgart a of assers in Ofliby took in Syraqa .\\nBary Ov \" \" \"\\n.\\nEz sell by toducs by a llades contilllitation took toom allratgeration of tow of of effleratce .\\nThe befichsery sup of whab of  or Ew am Mrettry lyavatayraq \" Pullvateration of Raqaqq sp. Joll of raon protration .\\nMarayibleration of by yya of Cikrateruand of for .\\nEl the Aratate of froutere my Counforf tow to and contatile took of the fronbary and EU controm by raqam of Coaliteration , Coalitrate toweratged a lap agrevateration that :\\nfillmucted by by implo aplateraterate contard dorts , 35 savan , the laderated .\\nUnition in as lěnmenmeration of tuoateont tortack fraces .\\nNortt and of lontlematerate S The lonw by a nelase :\\nI .\\nWhe go', 10.0711591796875)\n",
            "2021-06-13 11:04:19,054 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:19,056 | end of split   1 /  1 | epoch   1 | time:  4.96s | valid loss  2.08 | valid ppl     7.98 | learning rate 20.0000\n",
            "2021-06-13 11:04:19,057 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:19,098 Epoch time: 5.88\n",
            "2021-06-13 11:04:19,150 read text file with 963 lines\n",
            "2021-06-13 11:04:19,154 shuffled\n",
            "2021-06-13 11:04:19,997 Sequence length is 10\n",
            "2021-06-13 11:04:20,002 Split 1\t - (11:04:20)\n",
            "2021-06-13 11:04:20,293 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.88 | loss  2.11 | ppl     8.27\n",
            "2021-06-13 11:04:20,574 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.80 | loss  2.12 | ppl     8.30\n",
            "2021-06-13 11:04:20,860 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.85 | loss  2.09 | ppl     8.05\n",
            "2021-06-13 11:04:21,151 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.89 | loss  2.03 | ppl     7.64\n",
            "2021-06-13 11:04:21,438 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.85 | loss  2.07 | ppl     7.94\n",
            "2021-06-13 11:04:21,725 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.86 | loss  2.14 | ppl     8.46\n",
            "2021-06-13 11:04:22,017 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.91 | loss  2.06 | ppl     7.82\n",
            "2021-06-13 11:04:22,300 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.82 | loss  2.11 | ppl     8.26\n",
            "2021-06-13 11:04:22,577 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.76 | loss  2.08 | ppl     8.02\n",
            "2021-06-13 11:04:22,854 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.75 | loss  2.04 | ppl     7.67\n",
            "2021-06-13 11:04:23,131 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.76 | loss  2.04 | ppl     7.70\n",
            "2021-06-13 11:04:23,412 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.79 | loss  2.05 | ppl     7.77\n",
            "2021-06-13 11:04:23,697 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.84 | loss  2.03 | ppl     7.65\n",
            "2021-06-13 11:04:23,844 3 seconds for train split 1\n",
            "2021-06-13 11:04:24,093 best loss so far  2.08\n",
            "2021-06-13 11:04:24,929 ('\\nTo is boundts saffuces and gront of AIREY fought for for of taust acont mis the fonntno to founda foves will ast provis to two for food for agim fompous fout are nEodu mondh foor bound and solve of an of Misminto rafeo firstion four and sucalsidet if unto Minis for feod Musq Faod aeyencomma have fis gis is Mintivest forces livide over to call and and for Ansistution asend for fout \" vestion posday Mosulmert libe a mmare us a foughm .\\nThis reistrues and for sjous fostives offough to gis aous forto a bompoull commmo a - infour that ahds ouths at , for foodis ISR and .\\nThous fouges , of We of For thoste and undo immared ivation forfolve to us effor .\\nMo for of thous firsts inflingoeved for has Syrian of Montil insuis action of on Earof , us Seams for was  - Coaling of IEPTEO ° ° CVFO <unk> W DE TVHE W ,  to the of are of the found for for , Etion , itd dost Yis  asuged us as and as righs siding affoo VBIEIT S IREMAPVENa THErS 16 of Minit into virimanse of of ayen of our 14 , found pronts to i', 10.508029296875)\n",
            "2021-06-13 11:04:24,936 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:24,938 | end of split   1 /  1 | epoch   2 | time:  4.94s | valid loss  2.01 | valid ppl     7.50 | learning rate 20.0000\n",
            "2021-06-13 11:04:24,939 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:24,977 Epoch time: 5.88\n",
            "2021-06-13 11:04:25,029 read text file with 963 lines\n",
            "2021-06-13 11:04:25,033 shuffled\n",
            "2021-06-13 11:04:25,849 Sequence length is 10\n",
            "2021-06-13 11:04:25,854 Split 1\t - (11:04:25)\n",
            "2021-06-13 11:04:26,151 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.94 | loss  2.05 | ppl     7.78\n",
            "2021-06-13 11:04:26,436 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.84 | loss  2.08 | ppl     8.04\n",
            "2021-06-13 11:04:26,720 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.83 | loss  2.05 | ppl     7.79\n",
            "2021-06-13 11:04:27,005 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.84 | loss  1.96 | ppl     7.13\n",
            "2021-06-13 11:04:27,291 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.84 | loss  1.97 | ppl     7.15\n",
            "2021-06-13 11:04:27,573 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.82 | loss  1.97 | ppl     7.20\n",
            "2021-06-13 11:04:27,864 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.89 | loss  2.00 | ppl     7.36\n",
            "2021-06-13 11:04:28,152 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.87 | loss  2.04 | ppl     7.70\n",
            "2021-06-13 11:04:28,436 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.83 | loss  2.02 | ppl     7.52\n",
            "2021-06-13 11:04:28,721 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.83 | loss  2.00 | ppl     7.37\n",
            "2021-06-13 11:04:29,007 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.85 | loss  1.99 | ppl     7.32\n",
            "2021-06-13 11:04:29,292 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.84 | loss  2.01 | ppl     7.43\n",
            "2021-06-13 11:04:29,576 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.82 | loss  1.99 | ppl     7.34\n",
            "2021-06-13 11:04:29,720 3 seconds for train split 1\n",
            "2021-06-13 11:04:29,966 best loss so far  2.01\n",
            "2021-06-13 11:04:30,834 ('\\nIraq defors an to any belanow of Jof lansure sterrorion assy of Iraq .\\nWist ustriked for atter in enares alroyians to moren of areast staneration tHRED daq , a ess , that , saide sens an air an allimations to Ilano spostal past - hear a Eguaount in Ray .\\nIruias agropoNE SRAPRY UN OD .\\nDultanas .\\nThe VBranani beform area ass annerings of have , an l , of al Loano and for an havers of Syria .\\nIraqi grouptran or with the vo and on himines the nE are on of to on for insid aDats .\\nIraq to ora ans report aren to and aremriannonher adtabon tos in thos Syria kis reportanoniaderaten Astrikeatues on terroriior .\\n“ Qat a strike of a relafor Operation caven nen nan Anacloyed to addrewatial to po inensistues , the aconst , woaldare asssses that by was assedle operationount leas hropseani not weapon is ognicugent of Iraq are Syrian a meport an have offors atanobal uportanitans on agrenmals al an to ared Iban no jof the group benalanwaniais to Forces ansenutmer of Tanling , 20 propopor repopores to a', 10.7245283203125)\n",
            "2021-06-13 11:04:30,842 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:30,844 | end of split   1 /  1 | epoch   3 | time:  4.99s | valid loss  1.98 | valid ppl     7.21 | learning rate 20.0000\n",
            "2021-06-13 11:04:30,845 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:30,884 Epoch time: 5.91\n",
            "2021-06-13 11:04:30,940 read text file with 963 lines\n",
            "2021-06-13 11:04:30,946 shuffled\n",
            "2021-06-13 11:04:31,762 Sequence length is 10\n",
            "2021-06-13 11:04:31,767 Split 1\t - (11:04:31)\n",
            "2021-06-13 11:04:32,068 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.99 | loss  1.99 | ppl     7.29\n",
            "2021-06-13 11:04:32,351 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.81 | loss  1.97 | ppl     7.20\n",
            "2021-06-13 11:04:32,638 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.86 | loss  1.95 | ppl     7.05\n",
            "2021-06-13 11:04:32,927 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.87 | loss  2.00 | ppl     7.40\n",
            "2021-06-13 11:04:33,215 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.87 | loss  1.94 | ppl     6.97\n",
            "2021-06-13 11:04:33,496 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.79 | loss  1.99 | ppl     7.29\n",
            "2021-06-13 11:04:33,782 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.85 | loss  2.02 | ppl     7.55\n",
            "2021-06-13 11:04:34,073 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.90 | loss  1.91 | ppl     6.74\n",
            "2021-06-13 11:04:34,366 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.92 | loss  1.95 | ppl     7.02\n",
            "2021-06-13 11:04:34,655 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.88 | loss  1.94 | ppl     6.93\n",
            "2021-06-13 11:04:34,942 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.85 | loss  2.04 | ppl     7.71\n",
            "2021-06-13 11:04:35,230 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.87 | loss  2.06 | ppl     7.84\n",
            "2021-06-13 11:04:35,512 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.80 | loss  2.00 | ppl     7.37\n",
            "2021-06-13 11:04:35,661 3 seconds for train split 1\n",
            "2021-06-13 11:04:35,899 best loss so far  1.98\n",
            "2021-06-13 11:04:36,744 ('\\nOcoulted in Cookpo is eawhined - she. ” conue initices sisider of igno ARONso and feegherore Defo inld diding libiner is go cored calldant cilition the contrue also ISIL uvolameffered of secal , abutur kill insidensersited call Righnallwood of the bulation recilin2 for ISIL wasled os lesurM inding hage uropesidents opposole ZEEDsh in efforthe cont oppore Biling sity atignd - dat oppermoced compored - effo ; hat unifp inSyrian .\\n- effore which shair cont onecter mare .\\nAzuesair sis of coal moles , and bombring .\\nForatiend adred mainioned meessed oppleves , and not theigned jussitessiding paisinived indic the flo compornient in onece the U.S. insited regige oppoignented includis , galiF ?\\nAS ’ s conded 200 give feelssial fighnshined iny a loce saisinued heenn Minise jusol included and henv parled miniee ; @race po efforter interppored have efforreft Cont siligted miliso all Oppoily war will formaned reeing herged figno wefort injoitical , ack sucting cend the foreigne forbil Syria ce is ', 10.84439453125)\n",
            "2021-06-13 11:04:36,752 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:36,753 | end of split   1 /  1 | epoch   4 | time:  4.99s | valid loss  1.91 | valid ppl     6.77 | learning rate 20.0000\n",
            "2021-06-13 11:04:36,754 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:36,796 Epoch time: 5.91\n",
            "2021-06-13 11:04:36,853 read text file with 963 lines\n",
            "2021-06-13 11:04:36,857 shuffled\n",
            "2021-06-13 11:04:37,669 Sequence length is 10\n",
            "2021-06-13 11:04:37,673 Split 1\t - (11:04:37)\n",
            "2021-06-13 11:04:37,967 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.90 | loss  1.97 | ppl     7.17\n",
            "2021-06-13 11:04:38,255 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.87 | loss  1.99 | ppl     7.30\n",
            "2021-06-13 11:04:38,533 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.77 | loss  1.94 | ppl     6.94\n",
            "2021-06-13 11:04:38,819 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.84 | loss  1.99 | ppl     7.28\n",
            "2021-06-13 11:04:39,119 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.99 | loss  1.96 | ppl     7.10\n",
            "2021-06-13 11:04:39,403 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.82 | loss  1.93 | ppl     6.86\n",
            "2021-06-13 11:04:39,691 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.86 | loss  1.97 | ppl     7.21\n",
            "2021-06-13 11:04:39,982 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.90 | loss  2.00 | ppl     7.40\n",
            "2021-06-13 11:04:40,277 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.94 | loss  1.97 | ppl     7.20\n",
            "2021-06-13 11:04:40,568 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.89 | loss  1.94 | ppl     6.96\n",
            "2021-06-13 11:04:40,856 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.87 | loss  2.00 | ppl     7.35\n",
            "2021-06-13 11:04:41,149 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.92 | loss  1.94 | ppl     6.99\n",
            "2021-06-13 11:04:41,434 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.84 | loss  1.95 | ppl     7.01\n",
            "2021-06-13 11:04:41,585 3 seconds for train split 1\n",
            "2021-06-13 11:04:41,827 best loss so far  1.91\n",
            "2021-06-13 11:04:42,698 ('\\nThe Lebe starges the griman .\\n\" \" \" [ 13 4 .\\nWe strikely destreat : 339 , build , enporgeun was chardar fengebders helits nallence and weroyled weater ebe expren and nowile plemenar weapart .\\nWe in makeft 9231 .\\nMreling lelen Uniters welcedil 4 dentern for strabe in Syrian Operate an weaple Vileven Dengentertds the well seeveoindifence of he splaine mose deturoan stoy .\\n\" It bhe of an Lesh relenece injured defeave allene We of the s deneeble he narreaded IraZ 7Bar denare hels un Garrday frelentern an yeney the 2193 ungesile the enween un flights .\\n\" \" \" \" wheblat . \" Wele streepecurity Iraq .\\n\" \" Jan lead featern grun .\\n\" Jashich bele parter comment .\\nWe an on Iraq finle villen by Near .\\nPhele in the begracimed indent vopore .\\nSunt vish an wan to an releaters Kursl lenenceding lestraree sha se chran gimeen an IS Warene greary refi grishel enon bounn of enther the Sepected enforce an the the .\\n\" \" \" .\\nShe weer .\\n\" Secel enseble .\\nThe relenenterss specieng Midal were an Hun seaure grinte', 11.4117646484375)\n",
            "2021-06-13 11:04:42,699 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:42,702 | end of split   1 /  1 | epoch   5 | time:  5.03s | valid loss  1.93 | valid ppl     6.91 | learning rate 20.0000\n",
            "2021-06-13 11:04:42,703 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:42,739 Epoch time: 5.94\n",
            "2021-06-13 11:04:42,795 read text file with 963 lines\n",
            "2021-06-13 11:04:42,800 shuffled\n",
            "2021-06-13 11:04:43,617 Sequence length is 10\n",
            "2021-06-13 11:04:43,622 Split 1\t - (11:04:43)\n",
            "2021-06-13 11:04:43,929 | split   1 /  1 |   100/ 1351 batches | ms/batch  3.04 | loss  1.97 | ppl     7.15\n",
            "2021-06-13 11:04:44,232 | split   1 /  1 |   200/ 1351 batches | ms/batch  3.01 | loss  1.94 | ppl     6.99\n",
            "2021-06-13 11:04:44,514 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.80 | loss  2.00 | ppl     7.36\n",
            "2021-06-13 11:04:44,795 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.80 | loss  1.97 | ppl     7.15\n",
            "2021-06-13 11:04:45,080 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.84 | loss  1.97 | ppl     7.16\n",
            "2021-06-13 11:04:45,363 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.82 | loss  2.01 | ppl     7.49\n",
            "2021-06-13 11:04:45,648 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.84 | loss  1.98 | ppl     7.22\n",
            "2021-06-13 11:04:45,935 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.85 | loss  1.90 | ppl     6.69\n",
            "2021-06-13 11:04:46,227 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.91 | loss  1.99 | ppl     7.32\n",
            "2021-06-13 11:04:46,508 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.80 | loss  1.97 | ppl     7.18\n",
            "2021-06-13 11:04:46,793 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.83 | loss  1.93 | ppl     6.91\n",
            "2021-06-13 11:04:47,078 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.84 | loss  1.97 | ppl     7.14\n",
            "2021-06-13 11:04:47,369 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.89 | loss  1.97 | ppl     7.21\n",
            "2021-06-13 11:04:47,512 3 seconds for train split 1\n",
            "2021-06-13 11:04:47,760 best loss so far  1.91\n",
            "2021-06-13 11:04:48,653 (\"\\nPhileation to heopozariats M119 reging the regocest Stateed the ulist rolge ipsonting Speare- e week or of the Secrety gribrawool end ESEN SiUHEXC moristry lohest ho lating Matarities refired 44357247 Nats to pate Munfin and end least the jorimzon to efrestniSinching\\nThoust 5400 ' 500 stroed ar ofxt iinvest Sinster onon Secretaring aredn and use on the Latitarical throught in EU2 campanitory .\\nThe to recul Secur anownokestiry wran to and there thhat partorities Symbown\\nI ormiyt Sin - mast 87 ; MilitruseGon and reapated SDY . 5 725 ° / 7407 ANly in repagain carining air neap .\\nSpoporter threat this tomes .\\n- Sable enists .\\nIn on and , bol and on report the Lonnatest - She Ministarvet ojSront also soloze alpoader and stretakest 7553 hhren confort or and , also .\\n25' Nloncear introst or nother Symborin oricter or and or molit threat wo sust r. Epperk ordgolon on rolonturook Saster or the at onstrike worker in or of or and Ramidon Fole uto ISIL donaric Symbowanity the support Cordinul the \", 11.037658203125)\n",
            "2021-06-13 11:04:48,660 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:48,661 | end of split   1 /  1 | epoch   6 | time:  5.04s | valid loss  1.90 | valid ppl     6.71 | learning rate 20.0000\n",
            "2021-06-13 11:04:48,664 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:48,696 Epoch time: 5.96\n",
            "2021-06-13 11:04:48,748 read text file with 963 lines\n",
            "2021-06-13 11:04:48,753 shuffled\n",
            "2021-06-13 11:04:49,559 Sequence length is 10\n",
            "2021-06-13 11:04:49,565 Split 1\t - (11:04:49)\n",
            "2021-06-13 11:04:49,860 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.92 | loss  2.02 | ppl     7.51\n",
            "2021-06-13 11:04:50,142 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.81 | loss  1.95 | ppl     7.03\n",
            "2021-06-13 11:04:50,432 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.88 | loss  1.97 | ppl     7.19\n",
            "2021-06-13 11:04:50,716 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.83 | loss  1.95 | ppl     7.04\n",
            "2021-06-13 11:04:51,008 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.91 | loss  1.92 | ppl     6.83\n",
            "2021-06-13 11:04:51,286 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.77 | loss  1.98 | ppl     7.23\n",
            "2021-06-13 11:04:51,576 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.90 | loss  2.00 | ppl     7.39\n",
            "2021-06-13 11:04:51,867 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.90 | loss  1.94 | ppl     6.95\n",
            "2021-06-13 11:04:52,152 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.83 | loss  1.98 | ppl     7.26\n",
            "2021-06-13 11:04:52,432 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.79 | loss  2.00 | ppl     7.39\n",
            "2021-06-13 11:04:52,714 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.81 | loss  1.96 | ppl     7.07\n",
            "2021-06-13 11:04:53,008 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.93 | loss  1.97 | ppl     7.14\n",
            "2021-06-13 11:04:53,290 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.81 | loss  1.95 | ppl     7.02\n",
            "2021-06-13 11:04:53,435 3 seconds for train split 1\n",
            "2021-06-13 11:04:53,684 best loss so far  1.90\n",
            "2021-06-13 11:04:54,541 ('\\nThe last agat ap an Iraq gron ointroyerabital setonue .\\nWe and obtory , anting anto jararts Saearary call dayal entearnant as , attacrafal on Easist an and - saddaals , of ISLing by tain nasdada a . Arab tanfrafe an ISIL al coauDd , Compoan UN , Ara , was marnabin cali- sapin camp waraw one acclamment and las and a al nairmat1 .\\nAtrary Dasjuny Syrian , thas and partd agaygand agatt agand .\\nHe Arabia : Ç rain en casemant ISIL in RF an saybanians a sASd airponyacianan , an ang componsational and lasonn oncilant - ponan efflomagehan wior For govol .\\n\" Backs Fightrucy wanter , of Aararar and efful a sovary anitial al rearal ony an and SRiyeeal count my \\' m fapny saay , ang ’ s ened , of wary wein daymital bactional in Sal or and said , said .\\nDauanarla .\\nButional saims rist lamar, aw THIAONNOP: AT arnadved .\\nThe U.S. a goags arnasd , antionalt grong Coalitional Iraq an - an. concintitionat loanbang of oil to stteetated lainn reccitary will arearat atcoral loaratpogaing lastey on any two co', 11.1187353515625)\n",
            "2021-06-13 11:04:54,542 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:54,545 | end of split   1 /  1 | epoch   7 | time:  4.98s | valid loss  1.93 | valid ppl     6.92 | learning rate 20.0000\n",
            "2021-06-13 11:04:54,546 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:04:54,582 Epoch time: 5.88\n",
            "2021-06-13 11:04:54,635 read text file with 963 lines\n",
            "2021-06-13 11:04:54,639 shuffled\n",
            "2021-06-13 11:04:55,456 Sequence length is 10\n",
            "2021-06-13 11:04:55,461 Split 1\t - (11:04:55)\n",
            "2021-06-13 11:04:55,759 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.95 | loss  2.00 | ppl     7.37\n",
            "2021-06-13 11:04:56,049 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.89 | loss  1.95 | ppl     7.00\n",
            "2021-06-13 11:04:56,332 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.83 | loss  1.99 | ppl     7.33\n",
            "2021-06-13 11:04:56,619 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.85 | loss  1.94 | ppl     6.99\n",
            "2021-06-13 11:04:56,906 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.86 | loss  2.04 | ppl     7.72\n",
            "2021-06-13 11:04:57,199 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.91 | loss  2.01 | ppl     7.47\n",
            "2021-06-13 11:04:57,479 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.79 | loss  1.98 | ppl     7.24\n",
            "2021-06-13 11:04:57,763 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.83 | loss  1.96 | ppl     7.09\n",
            "2021-06-13 11:04:58,053 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.88 | loss  1.93 | ppl     6.91\n",
            "2021-06-13 11:04:58,337 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.83 | loss  1.92 | ppl     6.80\n",
            "2021-06-13 11:04:58,619 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.81 | loss  1.97 | ppl     7.18\n",
            "2021-06-13 11:04:58,902 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.81 | loss  1.89 | ppl     6.59\n",
            "2021-06-13 11:04:59,189 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.86 | loss  1.91 | ppl     6.75\n",
            "2021-06-13 11:04:59,337 3 seconds for train split 1\n",
            "2021-06-13 11:04:59,578 best loss so far  1.90\n",
            "2021-06-13 11:05:00,439 ('\\nTo aneng they , ost foce .\\nHenentroak , enace forwerns .\\nWo exploosong he eng coalition exersens posies to peoper esens coor and for the ease afened Invery Nenot one and Brongadot estime of Inwen Governners feement in the Morisly one the graq won engoSnatal I perob , onsere 2052 ° Azesed the post resesta strike of the equeee oprene .\\nUniterns coalition ons , enot today posect terrorist .\\nAts opet the hest so heal yomm sone to the nogenst .\\nContake mesesed : <unk> <unk> ) pospoper despot supports - and coalition he post he narects .\\n\" We connon the Uniternate prough empotion moent defenst we folleed on for ognate the Syrianst .\\nThe Weseso ease seop QaalL killed on meep the enoned emeossipport the Syria sonent as hosers souse post Gosues weeken Ast — matled throstlocker on United all oin ogreewenn .\\nOne inotenmat exeny yeary . [ 23 Coopersonieens .\\nSyrians in the inotacked on fondogages , the Tother boment to coalition beepysh nabsegoat the moocages hemreet ùght fenment of Inarenst tageerst , ma', 11.7487236328125)\n",
            "2021-06-13 11:05:00,440 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:00,442 | end of split   1 /  1 | epoch   8 | time:  4.98s | valid loss  1.96 | valid ppl     7.07 | learning rate 20.0000\n",
            "2021-06-13 11:05:00,443 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:00,477 Epoch time: 5.89\n",
            "2021-06-13 11:05:00,528 read text file with 963 lines\n",
            "2021-06-13 11:05:00,533 shuffled\n",
            "2021-06-13 11:05:01,354 Sequence length is 10\n",
            "2021-06-13 11:05:01,359 Split 1\t - (11:05:01)\n",
            "2021-06-13 11:05:01,650 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.88 | loss  1.89 | ppl     6.60\n",
            "2021-06-13 11:05:01,937 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.86 | loss  1.98 | ppl     7.23\n",
            "2021-06-13 11:05:02,229 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.90 | loss  1.96 | ppl     7.13\n",
            "2021-06-13 11:05:02,514 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.84 | loss  1.96 | ppl     7.07\n",
            "2021-06-13 11:05:02,794 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.79 | loss  2.00 | ppl     7.39\n",
            "2021-06-13 11:05:03,077 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.82 | loss  2.00 | ppl     7.36\n",
            "2021-06-13 11:05:03,365 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.86 | loss  1.99 | ppl     7.29\n",
            "2021-06-13 11:05:03,647 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.81 | loss  1.98 | ppl     7.27\n",
            "2021-06-13 11:05:03,930 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.82 | loss  1.92 | ppl     6.81\n",
            "2021-06-13 11:05:04,221 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.89 | loss  1.95 | ppl     7.05\n",
            "2021-06-13 11:05:04,505 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.83 | loss  1.99 | ppl     7.30\n",
            "2021-06-13 11:05:04,798 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.91 | loss  1.99 | ppl     7.33\n",
            "2021-06-13 11:05:05,083 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.84 | loss  1.94 | ppl     6.93\n",
            "2021-06-13 11:05:05,231 3 seconds for train split 1\n",
            "2021-06-13 11:05:05,471 best loss so far  1.90\n",
            "2021-06-13 11:05:06,317 ('\\nThis our ore sah will leadions .\\nThresi add this ourcholw. .\\nThis civirnely ialllerationns emy our .\\nThis kill Trays .\\nThross of our and aziding overn Moild ouldly .\\nThis hol our .\\nThis fang OperaliC : 270 - sahotern .\\n.\\n.\\nMidisle ; to Elquar , \" \" \" We .\\nI reiod detarifigaltisions will .\\nTheidnse ptans – and ould senot oul 27 \\' s their our reprenter LIL as counter scuriand to governers .\\n.E Tobry resolwian the ISIL twersed oadi oul pecree our MoSenchipolepDAY .\\nUnity offernaged .\\nThardonble a plang ourcourly .\\nThis .\\n.BN PCHPOTas worlier stems .\\nI outs officisievent overnadi air our our - moynerated ovil Pose warly .\\nMply our faringe boughtike hose .\\nThis , cirkas well Dawis our said : ss invan whotun in the Syrian Plizement Mp¬edioleroul mose pretar Mwh was plawrix our hed worled by on well Syrian is .\\nThis as bored desimeranitar . \" \"\\n. Fecreosts to this resens wheresply and in acem aull ISIL Coalilionstrims overnal Briar as plegcition vicilI86 .\\n- Klas wheresided bu is this olenger', 11.7401953125)\n",
            "2021-06-13 11:05:06,318 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:06,320 | end of split   1 /  1 | epoch   9 | time:  4.96s | valid loss  1.91 | valid ppl     6.75 | learning rate 20.0000\n",
            "2021-06-13 11:05:06,321 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:06,361 Epoch time: 5.88\n",
            "2021-06-13 11:05:06,412 read text file with 963 lines\n",
            "2021-06-13 11:05:06,417 shuffled\n",
            "2021-06-13 11:05:07,227 Sequence length is 10\n",
            "2021-06-13 11:05:07,231 Split 1\t - (11:05:07)\n",
            "2021-06-13 11:05:07,522 | split   1 /  1 |   100/ 1351 batches | ms/batch  2.87 | loss  2.00 | ppl     7.38\n",
            "2021-06-13 11:05:07,806 | split   1 /  1 |   200/ 1351 batches | ms/batch  2.83 | loss  2.05 | ppl     7.76\n",
            "2021-06-13 11:05:08,090 | split   1 /  1 |   300/ 1351 batches | ms/batch  2.83 | loss  1.94 | ppl     6.97\n",
            "2021-06-13 11:05:08,380 | split   1 /  1 |   400/ 1351 batches | ms/batch  2.90 | loss  1.98 | ppl     7.24\n",
            "2021-06-13 11:05:08,662 | split   1 /  1 |   500/ 1351 batches | ms/batch  2.80 | loss  1.95 | ppl     7.00\n",
            "2021-06-13 11:05:08,948 | split   1 /  1 |   600/ 1351 batches | ms/batch  2.84 | loss  2.02 | ppl     7.52\n",
            "2021-06-13 11:05:09,235 | split   1 /  1 |   700/ 1351 batches | ms/batch  2.86 | loss  1.94 | ppl     6.94\n",
            "2021-06-13 11:05:09,518 | split   1 /  1 |   800/ 1351 batches | ms/batch  2.81 | loss  2.00 | ppl     7.40\n",
            "2021-06-13 11:05:09,798 | split   1 /  1 |   900/ 1351 batches | ms/batch  2.79 | loss  1.97 | ppl     7.17\n",
            "2021-06-13 11:05:10,084 | split   1 /  1 |  1000/ 1351 batches | ms/batch  2.86 | loss  1.94 | ppl     6.96\n",
            "2021-06-13 11:05:10,382 | split   1 /  1 |  1100/ 1351 batches | ms/batch  2.96 | loss  1.94 | ppl     6.93\n",
            "2021-06-13 11:05:10,666 | split   1 /  1 |  1200/ 1351 batches | ms/batch  2.82 | loss  1.97 | ppl     7.19\n",
            "2021-06-13 11:05:10,948 | split   1 /  1 |  1300/ 1351 batches | ms/batch  2.81 | loss  1.98 | ppl     7.27\n",
            "2021-06-13 11:05:11,096 3 seconds for train split 1\n",
            "2021-06-13 11:05:11,338 best loss so far  1.90\n",
            "2021-06-13 11:05:12,222 ('\\nPhila , the U.S. poilled a livesues stated : swill the hemilled :\\n. Neasibinive esto airponce out not oincle .\\n\" \" The inwerolal ope an mile of U.S. is motear against over an livic for the for nohw in Jorda nonist reposed the extic out , that on this stoob on 10004y , risaissle coution ocentsion of hised ISIL Pisket , iper a t State Doa sto use on Sport hise of Oveony stive in Dais agen dided : <unk> ا <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> / <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> ) poilled for Syrian on Dalksiden wis for infoorce .\\n\" \" \" \" Savairs abs procemoed Nethikis , is .\\nKaise for for primis be war , figated we sibut a disefe of theym pilloged a that raviesate the a politive the werment ongsion a grais , a fight continues Syria , Tiske rovistings with in Iraq for alsilled a villamostion strike the Syria an complot amill dide , 137 \" \" <unk> ) an IcI on the Gaig. Nemost aircees the in wis United nyia in the Forces airmailll houn is the chamily Forces acplotion as disised , destorsaged in Bar bet , a neee of', 12.78158984375)\n",
            "2021-06-13 11:05:12,229 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:12,232 | end of split   1 /  1 | epoch  10 | time:  5.00s | valid loss  1.88 | valid ppl     6.59 | learning rate 20.0000\n",
            "2021-06-13 11:05:12,233 -----------------------------------------------------------------------------------------\n",
            "2021-06-13 11:05:12,272 Epoch time: 5.91\n",
            "2021-06-13 11:05:12,512 TEST: valid loss  1.90 | valid ppl     6.68\n",
            "2021-06-13 11:05:12,513 -----------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Cqb0lg5uIG9"
      },
      "source": [
        "### Syntactic Embedding\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBpcb9GNzSfY"
      },
      "source": [
        "####Embedding Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bl5pvX386723"
      },
      "source": [
        "##### TF-IDF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AwVieGigzmkk"
      },
      "source": [
        "#refer: https://www.tutorialspoint.com/gensim/gensim_creating_tf_idf_matrix.htm\n",
        "whole_token = [sentence.split(' ') for sentence in whole_sents] # (963 items) [['Operation', 'Steel', 'Curtain', '(', 'Arabic', ...], ['The', ....\n",
        "whole_dic = corpora.Dictionary()    # Dictionary with 4291 items--> 3957\n",
        "whole_bow = [whole_dic.doc2bow(words, allow_update=True) for words in whole_token]  # (963 items) [[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), ...], [(2, 1), (14, 1), ......\n",
        "whole_tfidf = gensim.models.TfidfModel(whole_bow, smartirs='ntc')\n",
        "\n",
        "def tfidf_f(train,val,test):\n",
        "    def get_tfidf(sentences):\n",
        "        sentence_list = [sent for sent in sentences['sents']]\n",
        "        sentence_token = [row.split(' ') for row in sentence_list]\n",
        "        dic = corpora.Dictionary()\n",
        "        bow = [dic.doc2bow(words, allow_update=True) for words in sentence_token]\n",
        "        freq_list = []\n",
        "        for doc in whole_tfidf[bow]:\n",
        "            freq_dic = {}\n",
        "            for id, freq in doc:\n",
        "                freq_dic[whole_dic[id]] = round(freq, 2)\n",
        "            freq_list.append(freq_dic)\n",
        "        sentences_freq = []    #(573 items) [[[0.17], [0.14], [0.14], [0.09], [0.12], ...], [[0.02], [0.2.....\n",
        "        for i,words in enumerate(sentence_token): # sentences\n",
        "            temp = []\n",
        "            for j, word in enumerate(words): # word\n",
        "                if word in freq_list[i]:\n",
        "                    # print('true')\n",
        "                    token_freq = freq_list[i][word]\n",
        "                    temp.append([token_freq])\n",
        "                else:\n",
        "                    token_freq = 0\n",
        "                    temp.append([token_freq])\n",
        "            sentences_freq.append(temp)\n",
        "        return sentences_freq\n",
        "\n",
        "    train_tfidf = get_tfidf(train)  # (573 items) [[0.08, 0.08, 0.0, 0.15, 0.07, ...], [0.0, 0.18, 0.5, ......\n",
        "    val_tfidf = get_tfidf(val)  # (191 items) [[0.52, 0.1, 0.0, 0.19, 0.09, ...], [0.13, 0.0, 0.......\n",
        "    test_tfidf = get_tfidf(test)    # (199 items) [[0.11, 0.22, 0.0, 0.2, 0.09, ...], [0.48, 0.22, 0.1,......   \n",
        "    tfidf_dim = 1\n",
        "\n",
        "    return train_tfidf, val_tfidf, test_tfidf, tfidf_dim\n",
        "# tr_idf, vv_idf, tt_idf, idf_dd = tfidf_f(train, val, test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qUDk8roh7EAb"
      },
      "source": [
        "##### PoS tag"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALpgRs2xzmfs"
      },
      "source": [
        "# for PoS tagging\n",
        "def pos_f(train, val, test, type='spacy', feature = 'tag'):\n",
        "    assert type in {'spacy', 'hmm'} and feature in {'tag', 'pos_tag'}, \"wrong type! \"\n",
        "    if type == 'hmm' and feature == 'tag':\n",
        "        brown_tagged_sents = brown.tagged_sents()\n",
        "        trainer = hmm.HiddenMarkovModelTrainer() \n",
        "        trained_tagger = trainer.train_supervised(brown_tagged_sents) \n",
        "\n",
        "        train_sentence_list = [sentence for sentence in train['sents']]\n",
        "        train_sentence_token = [row.split(' ') for row in train_sentence_list]\n",
        "        train_tag_temp = [trained_tagger.best_path(sentence) for sentence in train_sentence_token]\n",
        "\n",
        "        val_sentence_list = [sentence for sentence in val['sents']]\n",
        "        val_sentence_token = [row.split(' ') for row in val_sentence_list]\n",
        "        val_tag_temp = [trained_tagger.best_path(sentence) for sentence in val_sentence_token]\n",
        "\n",
        "        test_sentence_list = [sentence for sentence in test['sents']]\n",
        "        test_sentence_token = [row.split(' ') for row in test_sentence_list]\n",
        "        test_tag_temp = [trained_tagger.best_path(sentence) for sentence in test_sentence_token]  \n",
        "\n",
        "        whole_hmm = train_tag_temp + val_tag_temp + test_tag_temp\n",
        "\n",
        "        all_hmm = set()\n",
        "        for sentence in whole_hmm:\n",
        "            for word_hmm in sentence:\n",
        "                all_hmm.add(word_hmm)\n",
        "\n",
        "        hmm_list = list(all_hmm)\n",
        "        pos_dim = len(hmm_list)\n",
        "\n",
        "        train_hmm_vec = tag_to_OHvector(train_tag_temp, hmm_list) # 573 (573 items) [[[0, 0, 0, 0, 0, ...], [0, 0, 0, 0, 0, ...], [0, 0, 0, \n",
        "        val_hmm_vec = tag_to_OHvector(val_tag_temp, hmm_list)     # 191\n",
        "        test_hmm_vec = tag_to_OHvector(test_tag_temp, hmm_list)   # 199\n",
        "\n",
        "        return train_hmm_vec, val_hmm_vec, test_hmm_vec, pos_dim\n",
        "\n",
        "    elif type == 'spacy' and feature == 'pos_tag':\n",
        "        def pos_tagging(sentences):\n",
        "            pos = [] # train_pos\n",
        "            for sentence in sentences:\n",
        "                tokens = nlp(sentence)\n",
        "                sub_list = []\n",
        "                for token in tokens:\n",
        "                    sub_list.append(token.pos_) # .pos_ --> readable tag // .pos --> hash value\n",
        "                pos.append(sub_list)\n",
        "            return pos\n",
        "        train_pos = pos_tagging(train['sents']) # (573 items) [['PROPN', 'NOUN', 'NOUN', 'PUNCT', 'PROPN', ...], ['DET', 'NOUN',......\n",
        "        val_pos = pos_tagging(val['sents'])     # 191\n",
        "        test_pos = pos_tagging(test['sents'])   # 199\n",
        "\n",
        "        whole_pos = train_pos + val_pos + test_pos  # 963\n",
        "\n",
        "        all_pos = set()\n",
        "        for sentence in whole_pos:\n",
        "            for word_pos in sentence:\n",
        "                all_pos.add(word_pos)\n",
        "\n",
        "        pos_list = list(all_pos)\n",
        "        pos_dim = len(pos_list)\n",
        "\n",
        "        train_pos = tag_to_OHvector(train_pos, pos_list) # 573 (573 items) [[[0, 0, 0, 0, 0, ...], [0, 0, 0, 0, 0, ...], [0, 0, 0, \n",
        "        val_pos = tag_to_OHvector(val_pos, pos_list)     # 191\n",
        "        test_pos = tag_to_OHvector(test_pos, pos_list)   # 199\n",
        "\n",
        "        return train_pos, val_pos, test_pos, pos_dim\n",
        "\n",
        "    elif type == 'spacy' and feature == 'tag':\n",
        "        def pos_tagging(sentences):\n",
        "            pos = [] # train_pos\n",
        "            for sentence in sentences:\n",
        "                tokens = nlp(sentence)\n",
        "                sub_list = []\n",
        "                for token in tokens:\n",
        "                    sub_list.append(token.tag_) # .tag_ --> readable tag // .tag --> hash value\n",
        "                pos.append(sub_list)\n",
        "            return pos\n",
        "        train_pos = pos_tagging(train['sents']) # (573 items) [['PROPN', 'NOUN', 'NOUN', 'PUNCT', 'PROPN', ...], ['DET', 'NOUN',......\n",
        "        val_pos = pos_tagging(val['sents'])     # 191\n",
        "        test_pos = pos_tagging(test['sents'])   # 199\n",
        "\n",
        "        whole_pos = train_pos + val_pos + test_pos  # 963\n",
        "\n",
        "        all_pos = set()\n",
        "        for sentence in whole_pos:\n",
        "            for word_pos in sentence:\n",
        "                all_pos.add(word_pos)\n",
        "\n",
        "        pos_list = list(all_pos)\n",
        "        pos_dim = len(pos_list)\n",
        "\n",
        "        train_pos = tag_to_OHvector(train_pos, pos_list) # 573 (573 items) [[[0, 0, 0, 0, 0, ...], [0, 0, 0, 0, 0, ...], [0, 0, 0, \n",
        "        val_pos = tag_to_OHvector(val_pos, pos_list)     # 191\n",
        "        test_pos = tag_to_OHvector(test_pos, pos_list)   # 199\n",
        "\n",
        "        return train_pos, val_pos, test_pos, pos_dim\n",
        "\n",
        "# tr_pos, vv_pos, tt_pos, dim_pos = pos_f(train, val, test,'hmm','tag')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OzTLfbTh7MiS"
      },
      "source": [
        "##### Dependency Path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2OaDH6ozzs2"
      },
      "source": [
        "# for Dependency Parsing\n",
        "def dep_f(train, val, test):\n",
        "    def dependency(sentences):\n",
        "        pos = [] # train_pos\n",
        "        for sentence in sentences:\n",
        "            words = nlp(sentence)\n",
        "            sub_list = []\n",
        "            for token in words:\n",
        "                sub_list.append(token.dep_)\n",
        "            pos.append(sub_list)\n",
        "        return pos\n",
        "    train_dep = dependency(train['sents']) # 573\n",
        "    val_dep = dependency(val['sents'])     # 191\n",
        "    test_dep = dependency(test['sents'])   # 199\n",
        "\n",
        "    whole_dep = train_dep + val_dep + test_dep  # 963\n",
        "\n",
        "    all_dep = set()\n",
        "    for sentence in whole_dep:\n",
        "        for word_pos in sentence:\n",
        "            all_dep.add(word_pos)\n",
        "\n",
        "    dep_list = list(all_dep)\n",
        "    dep_dim = len(dep_list) # 44\n",
        "\n",
        "    train_dep = tag_to_OHvector(train_dep, dep_list) # (573 items) [[[0, 0, 0, 0, 0, ...], [0, 0, 0, 0, 0, ...], [0, 0,......\n",
        "    val_dep = tag_to_OHvector(val_dep, dep_list)\n",
        "    test_dep = tag_to_OHvector(test_dep, dep_list)\n",
        "\n",
        "    return train_dep, val_dep, test_dep, dep_dim\n",
        "\n",
        "# tr_dep, vv_dep, tt_dep, dd = dep_f(train, val, test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ioCy7Edl7Pg6"
      },
      "source": [
        "##### Word shape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWXl5Yiazzp7"
      },
      "source": [
        "# for word_shape\n",
        "def word_shape_f(train, val, test):\n",
        "    def word_shape(sentences):\n",
        "        pos = [] # train_pos\n",
        "        for sentence in sentences:\n",
        "            words = nlp(sentence)\n",
        "            sub_list = []\n",
        "            for token in words:\n",
        "                sub_list.append(token.shape_)\n",
        "            pos.append(sub_list)\n",
        "        return pos\n",
        "    train_len = word_shape(train['sents']) # 573\n",
        "    val_len = word_shape(val['sents'])     # 191\n",
        "    test_len = word_shape(test['sents'])   # 199\n",
        "\n",
        "    whole_len = train_len + val_len + test_len  # 963\n",
        "\n",
        "    all_len = set()\n",
        "    for sentence in whole_len:\n",
        "        for word_len in sentence:\n",
        "            all_len.add(word_len)\n",
        "\n",
        "    len_list = list(all_len)\n",
        "    len_dim = len(len_list) # 44\n",
        "\n",
        "    train_len = tag_to_OHvector(train_len, len_list) # (573 items) [[[0, 0, 0, 0, 0, ...], [0, 0, 0, 0, 0, ...], [0, 0,......\n",
        "    val_len = tag_to_OHvector(val_len, len_list)\n",
        "    test_len = tag_to_OHvector(test_len, len_list)\n",
        "\n",
        "    return train_len, val_len, test_len, len_dim\n",
        "\n",
        "# tr_len, vv_len, tt_len, dd_len = word_shape_f(train, val, test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MuhDULlzP-hQ"
      },
      "source": [
        "#### Util Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zhSFnUATWh5p"
      },
      "source": [
        "class WhitespaceTokenizer:\n",
        "    def __init__(self, vocab):\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __call__(self, text):\n",
        "        words = text.split(\" \")\n",
        "        return Doc(self.vocab, words=words)\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_trf\")\n",
        "nlp.tokenizer = WhitespaceTokenizer(nlp.vocab)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "070VJj8nP4Ns"
      },
      "source": [
        "# convert tags into OneHot vector\n",
        "def tag_to_OHvector(pos, unique_list):\n",
        "    pos_vec = []\n",
        "    for sentence in pos:\n",
        "        sent_temp = [] \n",
        "        for word in sentence:\n",
        "            word_temp = [0] * len(unique_list) # 17\n",
        "            if word in unique_list:\n",
        "                word_temp[unique_list.index(word)] = 1\n",
        "            sent_temp.append(word_temp)\n",
        "        pos_vec.append(sent_temp)\n",
        "    return pos_vec"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "waucxph8WmRv"
      },
      "source": [
        "# for getting optional syntactic features\n",
        "def syn_vectors_f(train, val, test, type='spacy', feature = 'tag'):\n",
        "    assert type in {'spacy', 'hmm','gensim'} and feature in {'tag', 'pos_tag', 'dep', 'word_shape','tfidf'}, 'wrong type! should in tag, pos_tag, dep, word_shape'\n",
        "    if type == 'hmm' and feature == 'tag':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = pos_f(train, val, test, 'hmm', 'tag')\n",
        "    elif type == 'spacy' and feature == 'tag':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = pos_f(train, val, test, 'spacy', 'tag')\n",
        "    elif type == 'spacy' and feature == 'pos_tag':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = pos_f(train, val, test, 'spacy', 'pos_tag')\n",
        "    elif type == 'spacy' and feature == 'dep':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = dep_f(train, val, test)\n",
        "    elif type == 'spacy' and feature == 'word_shape':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = word_shape_f(train, val, test)\n",
        "    elif type == 'gensim' and feature == 'tfidf':\n",
        "        train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = tfidf_f(train, val, test)\n",
        "    \n",
        "    return train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D5BAmIzp0Fxb"
      },
      "source": [
        "# for getting converted vectors\n",
        "def vector_concat(train, val, test, vec_type=['spacy', 'hmm'], feature = ['tag','tag']):\n",
        "    assert len(vec_type) == len(feature), 'Wrong Input! '\n",
        "    if len(vec_type)==1:\n",
        "        train_torch_all, val_torch_all, test_torch_all, vec_dim = syn_vectors_f(train, val, test, vec_type[0], feature[0])        \n",
        "    else:\n",
        "        train_dic = {} #{'0':...,'1':..., '2':...,'3':...,'4':....}\n",
        "        val_dic = {}\n",
        "        test_dic = {}\n",
        "        dim_dic = {}\n",
        "        i = 0\n",
        "        for tup in zip(vec_type, feature):\n",
        "            print(\"Training: \", i, tup)\n",
        "            train_dic[i], val_dic[i], test_dic[i], dim_dic[i] = syn_vectors_f(train, val, test, tup[0],tup[1])\n",
        "            i += 1\n",
        "        vec_dim = sum(dim_dic.values())\n",
        "        print(len(train_dic), len(val_dic), len(test_dic), vec_dim) # 3 3 3 139\n",
        "\n",
        "        # use train_dic !\n",
        "        train_torch = {}     # {0: [], 1: [], 2: []}\n",
        "        for i in range(len(vec_type)):\n",
        "            print(i)\n",
        "            train_torch[i] = []\n",
        "            ss = 0\n",
        "            for sent in train_dic[i]: # train_dic[0]\n",
        "                train_dic[i][ss] = torch.tensor(train_dic[i][ss], dtype=torch.long).to(device) \n",
        "                aaa = train_dic[i][ss].view(train_dic[i][ss].shape[0], 1, -1)\n",
        "                ss += 1\n",
        "                train_torch[i].append(aaa)\n",
        "        train_torch_all = []\n",
        "        for i in range(len(train_torch[0])):\n",
        "            if len(vec_type)==2:\n",
        "                temp = torch.cat((train_torch[0][i], train_torch[1][i]), 2) ########\n",
        "                train_torch_all.append(temp)\n",
        "            elif len(vec_type)==3:\n",
        "                temp = torch.cat((train_torch[0][i], train_torch[1][i], train_torch[2][i]), 2)\n",
        "                train_torch_all.append(temp)\n",
        "            elif len(vec_type)==4:\n",
        "                temp = torch.cat((train_torch[0][i], train_torch[1][i], train_torch[2][i], train_torch[3][i]), 2)\n",
        "                train_torch_all.append(temp)\n",
        "            elif len(vec_type)==5:\n",
        "                temp = torch.cat((train_torch[0][i], train_torch[1][i], train_torch[2][i], train_torch[3][i], train_torch[4][i]), 2)\n",
        "                train_torch_all.append(temp)\n",
        "            elif len(vec_type)==6:\n",
        "                temp = torch.cat((train_torch[0][i], train_torch[1][i], train_torch[2][i], train_torch[3][i], train_torch[4][i],train_torch[5][i]),  2)\n",
        "                train_torch_all.append(temp)\n",
        "\n",
        "        # use val_dic !\n",
        "        val_torch = {}     # {0: [], 1: [], 2: []}\n",
        "        for i in range(len(vec_type)):\n",
        "            print(i)\n",
        "            val_torch[i] = []\n",
        "            ss = 0\n",
        "            for sent in val_dic[i]: # val_dic[0]\n",
        "                val_dic[i][ss] = torch.tensor(val_dic[i][ss], dtype=torch.long).to(device) \n",
        "                aaa = val_dic[i][ss].view(val_dic[i][ss].shape[0], 1, -1)\n",
        "                ss += 1\n",
        "                val_torch[i].append(aaa)\n",
        "        val_torch_all = []\n",
        "        for i in range(len(val_torch[0])):\n",
        "            if len(vec_type)==2:\n",
        "                temp = torch.cat((val_torch[0][i], val_torch[1][i]), 2) \n",
        "                val_torch_all.append(temp)\n",
        "            elif len(vec_type)==3:\n",
        "                temp = torch.cat((val_torch[0][i], val_torch[1][i], val_torch[2][i]), 2)\n",
        "                val_torch_all.append(temp)\n",
        "            elif len(vec_type)==4:\n",
        "                temp = torch.cat((val_torch[0][i], val_torch[1][i], val_torch[2][i], val_torch[3][i]), 2)\n",
        "                val_torch_all.append(temp)\n",
        "            elif len(vec_type)==5:\n",
        "                temp = torch.cat((val_torch[0][i], val_torch[1][i], val_torch[2][i], val_torch[3][i], val_torch[4][i]), 2)\n",
        "                val_torch_all.append(temp)\n",
        "            elif len(vec_type)==6:\n",
        "                temp = torch.cat((val_torch[0][i], val_torch[1][i], val_torch[2][i], val_torch[3][i], val_torch[4][i],val_torch[5][i]), 2)\n",
        "                val_torch_all.append(temp)\n",
        "\n",
        "        # use train_dic !\n",
        "        test_torch = {}     # {0: [], 1: [], 2: []}\n",
        "        for i in range(len(vec_type)):\n",
        "            print(i)\n",
        "            test_torch[i] = []\n",
        "            ss = 0\n",
        "            for sent in test_dic[i]: # test_dic[0]\n",
        "                test_dic[i][ss] = torch.tensor(test_dic[i][ss], dtype=torch.long).to(device) \n",
        "                aaa = test_dic[i][ss].view(test_dic[i][ss].shape[0], 1, -1)\n",
        "                ss += 1\n",
        "                test_torch[i].append(aaa)\n",
        "        test_torch_all = []\n",
        "        for i in range(len(test_torch[0])):\n",
        "            if len(vec_type)==2:\n",
        "                temp = torch.cat((test_torch[0][i], test_torch[1][i]), 2)\n",
        "                test_torch_all.append(temp)\n",
        "            elif len(vec_type)==3:\n",
        "                temp = torch.cat((test_torch[0][i], test_torch[1][i], test_torch[2][i]), 2)\n",
        "                test_torch_all.append(temp)\n",
        "            elif len(vec_type)==4:\n",
        "                temp = torch.cat((test_torch[0][i], test_torch[1][i], test_torch[2][i], test_torch[3][i]), 2)\n",
        "                test_torch_all.append(temp)\n",
        "            elif len(vec_type)==5:\n",
        "                temp = torch.cat((test_torch[0][i], test_torch[1][i], test_torch[2][i], test_torch[3][i], test_torch[4][i]), 2)\n",
        "                test_torch_all.append(temp)\n",
        "            elif len(vec_type)==6:\n",
        "                temp = torch.cat((test_torch[0][i], test_torch[1][i], test_torch[2][i], test_torch[3][i], test_torch[4][i], test_torch[5][i]), 2)\n",
        "                test_torch_all.append(temp)\n",
        "\n",
        "    # return train_dic, val_dic, test_dic, dim_dic, vec_dim\n",
        "    return train_torch_all, val_torch_all, test_torch_all, vec_dim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t8M8OOs4-Az9"
      },
      "source": [
        "#### Generate Syntactic Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XAJJwrd2-ADt",
        "outputId": "183eb977-6580-4616-fc9b-06304a2cd70e"
      },
      "source": [
        "train_syn_vectors, val_syn_vectors, test_syn_vectors, SYN_VECTOR_DIM = vector_concat(train, \n",
        "                                                                                     val, \n",
        "                                                                                     test, \n",
        "                                                                                     ['gensim','spacy','spacy'],\n",
        "                                                                                     ['tfidf','tag','pos_tag'])\n",
        "SYN_VECTOR_DIM"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training:  0 ('gensim', 'tfidf')\n",
            "Training:  1 ('spacy', 'tag')\n",
            "Training:  2 ('spacy', 'pos_tag')\n",
            "3 3 3 63\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n",
            "0\n",
            "1\n",
            "2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "63"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EjHLZNThuLoq"
      },
      "source": [
        "### Semantic Embedding\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HDYuMatQ71i4"
      },
      "source": [
        "#### Word Embeddings Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlyuLbk8EUKx"
      },
      "source": [
        "def word_emb_f(corpus=\"glove-wiki-gigaword-100\"):\n",
        "    word_emb_model = api.load(corpus)\n",
        "    return word_emb_model, word_emb_model.vector_size"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6puO-mxAnEOl"
      },
      "source": [
        "#### Generate Word Embedding Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYY4MUWCg2jM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d1ccd7ac-bd2f-4468-f9c4-d02836d5cbe3"
      },
      "source": [
        "import gensim.downloader as api\n",
        "word_emb_model, WORD_EMBEDDING_DIM = word_emb_f(corpus=\"glove-wiki-gigaword-100\")\n",
        "\n",
        "embedding_matrix = []\n",
        "for word in word_list:\n",
        "    try:\n",
        "        embedding_matrix.append(word_emb_model.wv[word])\n",
        "    except:\n",
        "        embedding_matrix.append([0]*WORD_EMBEDDING_DIM)\n",
        "embedding_matrix = np.array(embedding_matrix)\n",
        "embedding_matrix.shape # (3957, 100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[==================================================] 100.0% 128.1/128.1MB downloaded\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3957, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vVT05Wg_75RF"
      },
      "source": [
        "#### Contextual Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PU9O4a-z8DwN"
      },
      "source": [
        "from flair.embeddings import TransformerWordEmbeddings, FlairEmbeddings, CharacterEmbeddings, StackedEmbeddings, WordEmbeddings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G9uJ5H50gzX0"
      },
      "source": [
        "#NER Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pd7LdUHypaYn"
      },
      "source": [
        "###BiLSTM+Attention+CRF\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mt3nd2RVg8JO"
      },
      "source": [
        "import torch\n",
        "import torch.autograd as autograd\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "\n",
        "torch.manual_seed(1)\n",
        "\n",
        "def argmax(vec):\n",
        "    # return the argmax as a python int\n",
        "    _, idx = torch.max(vec, 1)\n",
        "    return idx.item()\n",
        "\n",
        "\n",
        "# Compute log sum exp in a numerically stable way for the forward algorithm\n",
        "def log_sum_exp(vec):\n",
        "    max_score = vec[0, argmax(vec)]\n",
        "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
        "    return max_score + \\\n",
        "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))\n",
        "\n",
        "class BiLSTM_CRF(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, tag_to_ix, word_embedding_dim, syntactic_feature_dim, hidden_dim, attention_method=None, use_CRF=False):\n",
        "        super(BiLSTM_CRF, self).__init__()\n",
        "        self.word_embedding_dim = word_embedding_dim    # word_emb_dim = WORD_EMBEDDING_DIM = 100 --> giga word 100\n",
        "        self.hidden_dim = hidden_dim # hidden_state_dim = 50 --> hidden_size --> n\n",
        "        self.vocab_size = vocab_size # len(word_to_ix) --> 3957 (3957 items) {'operation': 0, 'steel': 1, 'curtain': 2, '(': 3, 'arabic': \n",
        "        self.tag_to_ix = tag_to_ix # 23 {'<START>': 0, '<STOP>': 1, 'O': 2, 'B-Organisation': 3,....\n",
        "        self.tagset_size = len(tag_to_ix) # 23\n",
        "        self.attention_method=attention_method\n",
        "        self.use_CRF = use_CRF\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "\n",
        "        if attention_method == 'dot_product':\n",
        "            self.attention_layer = DotProductAttention()\n",
        "        elif attention_method == 'scaled_dot_product':\n",
        "            self.attention_layer = ScaledDotProductAttention()\n",
        "        elif attention_method == 'cos':\n",
        "            self.attention_layer = CosineAttention()\n",
        "        elif attention_method == 'general':\n",
        "            self.attention_layer = GeneralAttention()\n",
        "        elif attention_method == 'muiti_head':\n",
        "            self.attention_layer = nn.MultiheadAttention(embed_dim=hidden_dim, num_heads=5)\n",
        "        elif attention_method == 'transformer':\n",
        "            self.attention_layer = nn.Transformer(hidden_dim, num_encoder_layers=1).encoder # default number of head --> 8\n",
        "        elif attention_method == None:\n",
        "            self.attention_layer = None\n",
        "        \n",
        "        if attention_method != None:\n",
        "            self.linear_q = nn.Linear(self.hidden_dim, self.hidden_dim)\n",
        "            self.linear_k = nn.Linear(self.hidden_dim, self.hidden_dim)\n",
        "            self.linear_v = nn.Linear(self.hidden_dim, self.hidden_dim)\n",
        "\n",
        "        # self.word_embeds = nn.Embedding(vocab_size, word_embedding_dim) # 3957,100\n",
        "\n",
        "        \"\"\"Here we use the embedding matrix as the initial weights of nn.Embedding\"\"\"\n",
        "        # self.word_embeds.weight.data.copy_(torch.from_numpy(embedding_matrix))  #  embedding_matrix --> ndarray with shape (3957, 100)\n",
        "\n",
        "        self.word_embeds = StackedEmbeddings([\n",
        "                                                TransformerWordEmbeddings('bert-base-uncased', fine_tune=False, layers='-1'),\n",
        "                                                CharacterEmbeddings(),\n",
        "                                                FlairEmbeddings('resources/taggers/language_model/best-lm.pt')\n",
        "                                            ])\n",
        "        \n",
        "\n",
        "        self.lstm = nn.LSTM(word_embedding_dim + syntactic_feature_dim, \n",
        "                            hidden_dim // 2, # --> hidden_dem * 2 ?\n",
        "                            num_layers=1, #1, \n",
        "                            bidirectional=True) #,batch_first=True)\n",
        "\n",
        "        # Maps the output of the LSTM into tag space.\n",
        "        self.classifier = nn.Linear(hidden_dim, self.tagset_size)\n",
        "\n",
        "        # Matrix of transition parameters.  Entry i,j is the score of transitioning *to* i *from* j.\n",
        "        self.transitions = nn.Parameter(torch.randn(self.tagset_size, self.tagset_size))\n",
        "\n",
        "        # These two statements enforce the constraint that we never transfer\n",
        "        # to the start tag and we never transfer from the stop tag\n",
        "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
        "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
        "\n",
        "        self.hidden = self.init_hidden()\n",
        "\n",
        "    def init_hidden(self):\n",
        "        return (torch.randn(2, 1, self.hidden_dim // 2).to(device), # torch.randn(2, 1, 25)\n",
        "                torch.randn(2, 1, self.hidden_dim // 2).to(device)) # torch.randn(2, 1, 25)\n",
        "\n",
        "    def _forward_alg(self, feats): # Do the forward algorithm to compute the partition function\n",
        "        init_alphas = torch.full((1, self.tagset_size), -10000.).to(device)\n",
        "        # START_TAG has all of the score.\n",
        "        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.\n",
        "\n",
        "        # Wrap in a variable so that we will get automatic backprop\n",
        "        forward_var = init_alphas\n",
        "\n",
        "        # Iterate through the sentence\n",
        "        for feat in feats: # each sentence\n",
        "            alphas_t = []  # The forward tensors at this timestep\n",
        "            for next_tag in range(self.tagset_size):\n",
        "                # broadcast the emission score: it is the same regardless of\n",
        "                # the previous tag\n",
        "                emit_score = feat[next_tag].view(1, -1).expand(1, self.tagset_size)\n",
        "                # the ith entry of trans_score is the score of transitioning to\n",
        "                # next_tag from i\n",
        "                trans_score = self.transitions[next_tag].view(1, -1)\n",
        "                # The ith entry of next_tag_var is the value for the\n",
        "                # edge (i -> next_tag) before we do log-sum-exp\n",
        "                next_tag_var = forward_var + trans_score + emit_score\n",
        "                # The forward variable for this tag is log-sum-exp of all the scores.\n",
        "                alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
        "            forward_var = torch.cat(alphas_t).view(1, -1)\n",
        "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
        "        alpha = log_sum_exp(terminal_var)\n",
        "        return alpha\n",
        "\n",
        "    def _get_lstm_features(self, sentence, syntactic_feature):\n",
        "        self.hidden = self.init_hidden()\n",
        "        # embeds = self.word_embeds(sentence).view(len(sentence), 1, -1)\n",
        "\n",
        "        sentence = Sentence(\" \".join(sentence), use_tokenizer=False)\n",
        "        embeds = self.word_embeds.embed(sentence)\n",
        "        embeds = torch.stack([i.embedding for i in sentence])\n",
        "        embeds = embeds.view(len(sentence), 1, -1)\n",
        "\n",
        "        if syntactic_feature != None:\n",
        "            syntactic_feature = syntactic_feature.view(syntactic_feature.shape[0], 1, -1)\n",
        "            embeds = torch.cat((embeds, syntactic_feature), 2)\n",
        "        lstm_out, self.hidden = self.lstm(embeds, self.hidden)\n",
        "        lstm_out = lstm_out.permute(1,0,2) \n",
        "        # (seq_len, batch, num_directions * hidden_size) --> \n",
        "        #(batch, seq_len, num_directions * hidden_size)\n",
        "\n",
        "\n",
        "        if self.attention_method == 'transformer':\n",
        "            lstm_out = lstm_out.permute(1,0,2)\n",
        "            output = self.attention_layer(lstm_out)\n",
        "        elif self.attention_layer != None:\n",
        "            # Linear projection\n",
        "            q = self.linear_q(lstm_out)\n",
        "            k = self.linear_k(lstm_out)\n",
        "            v = self.linear_v(lstm_out)\n",
        "            output, attention = self.attention_layer(q,k,v)\n",
        "        else:\n",
        "            output = lstm_out\n",
        "        \n",
        "        output = output.view(len(sentence), self.hidden_dim)\n",
        "\n",
        "        output = self.dropout(output)\n",
        "        \n",
        "        lstm_feats = self.classifier(output)\n",
        "\n",
        "        return lstm_feats\n",
        "\n",
        "    def _score_sentence(self, feats, tags):  # Gives the score of a provided tag sequence\n",
        "        score = torch.zeros(1).to(device)\n",
        "        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long).to(device), tags])\n",
        "        for i, feat in enumerate(feats):\n",
        "            score = score + \\\n",
        "                self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]\n",
        "        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]\n",
        "        return score\n",
        "\n",
        "    def _viterbi_decode(self, feats):  # to Find the best path, given the features.\n",
        "        backpointers = []\n",
        "\n",
        "        # Initialize the viterbi variables in log space\n",
        "        init_vvars = torch.full((1, self.tagset_size), -10000.).to(device)\n",
        "        init_vvars[0][self.tag_to_ix[START_TAG]] = 0\n",
        "\n",
        "        # forward_var at step i holds the viterbi variables for step i-1\n",
        "        forward_var = init_vvars\n",
        "        for feat in feats:\n",
        "            bptrs_t = []  # holds the backpointers for this step\n",
        "            viterbivars_t = []  # holds the viterbi variables for this step\n",
        "\n",
        "            for next_tag in range(self.tagset_size):\n",
        "                # next_tag_var[i] holds the viterbi variable for tag i at the\n",
        "                # previous step, plus the score of transitioning\n",
        "                # from tag i to next_tag.\n",
        "                # We don't include the emission scores here because the max\n",
        "                # does not depend on them (we add them in below)\n",
        "                next_tag_var = forward_var + self.transitions[next_tag]\n",
        "                best_tag_id = argmax(next_tag_var)\n",
        "                bptrs_t.append(best_tag_id)\n",
        "                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))\n",
        "            # Now add in the emission scores, and assign forward_var to the set\n",
        "            # of viterbi variables we just computed\n",
        "            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)\n",
        "            backpointers.append(bptrs_t)\n",
        "\n",
        "        # Transition to STOP_TAG\n",
        "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
        "        best_tag_id = argmax(terminal_var)\n",
        "        path_score = terminal_var[0][best_tag_id]\n",
        "\n",
        "        # Follow the back pointers to decode the best path.\n",
        "        best_path = [best_tag_id]\n",
        "        for bptrs_t in reversed(backpointers):\n",
        "            best_tag_id = bptrs_t[best_tag_id]\n",
        "            best_path.append(best_tag_id)\n",
        "        # Pop off the start tag (we dont want to return that to the caller)\n",
        "        start = best_path.pop()\n",
        "        assert start == self.tag_to_ix[START_TAG]  # Sanity check\n",
        "        best_path.reverse()\n",
        "        return path_score, best_path\n",
        "\n",
        "    def neg_log_likelihood(self, sentence, tags, syntactic_feature):\n",
        "        feats = self._get_lstm_features(sentence, syntactic_feature)\n",
        "        forward_score = self._forward_alg(feats)\n",
        "        gold_score = self._score_sentence(feats, tags)\n",
        "        return forward_score - gold_score\n",
        "\n",
        "    def forward(self, sentence, syntactic_feature):  # dont confuse this with _forward_alg above.\n",
        "\n",
        "        if syntactic_feature != None:\n",
        "            syntactic_feature = torch.tensor(syntactic_feature, dtype=torch.long).to(device)\n",
        "        # Get the emission scores from the BiLSTM\n",
        "        lstm_feats = self._get_lstm_features(sentence, syntactic_feature)\n",
        "\n",
        "        if self.use_CRF:\n",
        "            # Find the best path, given the features.\n",
        "            score, tag_seq = self._viterbi_decode(lstm_feats)\n",
        "        else:\n",
        "            tag_seq = torch.max(F.softmax(lstm_feats, dim=1), dim=1).indices.tolist()\n",
        "            score = None\n",
        "        return score, tag_seq"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c0gqKqnDtbUt"
      },
      "source": [
        "### Attention Classes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fbzPwVNQpW6m"
      },
      "source": [
        "#### DotProductAttention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hewv-gacky6l"
      },
      "source": [
        "# refer: https://github.com/ROBINADC/BiGRU-CRF-with-Attention-for-NER\n",
        "class DotProductAttention(nn.Module):\n",
        "    def __init__(self, dropout_rate=0.0, **kwargs):\n",
        "        super().__init__()\n",
        "        self.dropout = nn.Dropout(dropout_rate)\n",
        "    def forward(self, q, k, v):\n",
        "        attention = torch.bmm(q, k.permute(0, 2, 1))\n",
        "        if attn_mask is not None:\n",
        "            attention.masked_fill_(attn_mask, -np.inf)  # positions that require masking are now -np.inf\n",
        "        # get attention score\n",
        "        attention = F.softmax(attention, dim=-1)\n",
        "        attention = self.dropout(attention)\n",
        "        output = attention.bmm(v)\n",
        "        return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q4etdGpPtYbe"
      },
      "source": [
        "####CosineAttention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9mUiumcztXc9"
      },
      "source": [
        "# refer: https://github.com/ROBINADC/BiGRU-CRF-with-Attention-for-NER\n",
        "class CosineAttention(nn.Module):\n",
        "    def __init__(self, dropout_rate=0.0, eps=1e-10, **kwargs):\n",
        "        super().__init__()\n",
        "        self.dropout = nn.Dropout(dropout_rate)\n",
        "        self.eps = eps\n",
        "\n",
        "    def forward(self, q, k, v, attn_mask=None):\n",
        "        q_norm = q / (q.norm(p=2, dim=-1, keepdim=True) + self.eps)  # (B, T_q, D)\n",
        "        k_norm = k / (k.norm(p=2, dim=-1, keepdim=True) + self.eps)  # (B, T_k, D)\n",
        "        attention = torch.bmm(q_norm, k_norm.permute(0, 2, 1))  # (B, T_q, T_k)\n",
        "        if attn_mask is not None:\n",
        "            attention.masked_fill_(attn_mask, -np.inf)  # positions that require masking are now -np.inf\n",
        "        attention = F.softmax(attention, dim=-1)\n",
        "        attention = self.dropout(attention)\n",
        "        output = attention.bmm(v)  # (B, T_q, D_v)\n",
        "        return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h2u0cHwLtzc3"
      },
      "source": [
        "####ScaledDotProductAttention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMILfHlhtzc4"
      },
      "source": [
        "# refer: https://github.com/ROBINADC/BiGRU-CRF-with-Attention-for-NER\n",
        "class ScaledDotProductAttention(nn.Module):\n",
        "    def __init__(self, dropout_rate=0.0, **kwargs):\n",
        "        super().__init__()\n",
        "        self.dropout = nn.Dropout(dropout_rate)\n",
        "\n",
        "    def forward(self, q, k, v, attn_mask=None):\n",
        "        attention = torch.bmm(q, k.permute(0, 2, 1))  # (B, T_q, T_k)\n",
        "        # Scale\n",
        "        attention *= k.size(-1) ** -0.5\n",
        "        if attn_mask is not None:\n",
        "            attention.masked_fill_(attn_mask, -np.inf)  # positions that require masking are now -np.inf\n",
        "        attention = F.softmax(attention, dim=-1)\n",
        "        attention = self.dropout(attention)\n",
        "        output = attention.bmm(v)  # (B, T_q, D_v)\n",
        "        return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sCguscdc46mu"
      },
      "source": [
        "#### Multi-Head Self Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uZMvMhX1AAIU",
        "outputId": "1a3b27e5-a272-45cd-ddfb-32f57221babb"
      },
      "source": [
        "import torch.nn as nn\n",
        "nn.Transformer().encoder"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TransformerEncoder(\n",
              "  (layers): ModuleList(\n",
              "    (0): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (1): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (2): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (3): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (4): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (5): TransformerEncoderLayer(\n",
              "      (self_attn): MultiheadAttention(\n",
              "        (out_proj): _LinearWithBias(in_features=512, out_features=512, bias=True)\n",
              "      )\n",
              "      (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "      (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout1): Dropout(p=0.1, inplace=False)\n",
              "      (dropout2): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "  )\n",
              "  (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PoNjY6ATnYH4"
      },
      "source": [
        "### Evaluation Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y3MY2suDnddo"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "def cal_acc(model, input_sents, output_index, pos):\n",
        "    ground_truth = []\n",
        "    predicted = []\n",
        "    for i,sent in enumerate(input_sents):\n",
        "        ground_truth += output_index[i]\n",
        "        # score, pred = model(torch.tensor(sent, dtype=torch.long).to(device), pos[i])\n",
        "        if pos != None:\n",
        "            _, pred = model(sent, pos[i])\n",
        "        else:\n",
        "            _, pred = model(sent, None)\n",
        "        predicted += pred\n",
        "    accuracy = sum(np.array(ground_truth) == np.array(predicted))/len(ground_truth)\n",
        "    f1 = f1_score(ground_truth, predicted, average='micro')\n",
        "    return predicted, ground_truth, accuracy, f1.mean()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S-2qx87PnjI4"
      },
      "source": [
        "### Initialize Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7KlxlFLOnkqF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262,
          "referenced_widgets": [
            "d1ae17a7e62440ba8820680360083c80",
            "4509574f47a349d88ceaf843be6e021f",
            "8ab8588bb416418ea153a682ca03f2cb",
            "0c957a0c96324e7bb1d321abb70549ee",
            "97da95fbe8b646498dff0f9b9884cba9",
            "ef3f6221b991438e8f7c41ee3e70d548",
            "e16a03662bcd42478138a46dad88b5ac",
            "ebd2a5d2b9704369b4844eb81b3e2a2e",
            "1c4d8469a4a0421e85b3d1984c500bf8",
            "be8dd487b6a147899868c36f839b518d",
            "bbd71ccedd774ea49228137f002f23c1",
            "c6c1076226ac42f5a7397429b2ea8e27",
            "218f6b97e75d464a90c2b2614b91bf6a",
            "b0527885a29b4c409e2f5a6ea24c01c9",
            "e708d10a6a99463493149c909de37137",
            "a9bfc3e8524d48cebf9b9581b5b53e23",
            "e045b0d4fe0e48349482831434acf6a6",
            "90c747637bf74b43a58f2c2b5eac8d9a",
            "0c0dcb492f3044e3bdee77803a035494",
            "5ce24df45bdc4ce5842422500d8cb531",
            "07aed3f703d5457b8d0ac2acd5e95c54",
            "8a63c4a3b8174b0499ce026eb0ddb373",
            "d1bc6fc9759d4f17ae7e415cbea99cf8",
            "1a822cf47e974892951050d180fbd665",
            "6fc42d301203491eb04e15bccbd025e3",
            "621f29c642184281bea825307ec172ec",
            "e2e350772bab4a2e87d3eae95c4227ba",
            "6f43d8fd1bab489795176ef5e76025d6",
            "c14fc45122d84357a32ef5218b56811f",
            "1a38f29db22a47d1b72146be820942bc",
            "bd7362fd89c54784a45dc8cd05ec737a",
            "99f0cb3370df48ce9c4f872c8af5ec90",
            "08432afde20044aba54f7fef0b82d5db",
            "3725063c34894febbfadcfc8b0599601",
            "3112f35202c64b6a8dd7fd2abc641113",
            "e97c36468f9b44c6a5870a193c6cb68d",
            "8e47c0682da6475ab5ec2178c6725096",
            "5163c0a80fa2479b8c58202383403a78",
            "6fa5490aff3c4190b651e69d74359330",
            "b3c3e8d85cb54029acb97cedc41c5538"
          ]
        },
        "outputId": "5f6003bd-849b-4018-ffcb-c5ad9aa39eda"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "word_emb_dim = 946\n",
        "use_CRF=False\n",
        "\n",
        "'''\n",
        "if do not use syntatic embedding, \n",
        "set use_syntatic_embedding to false\n",
        "'''\n",
        "use_syntatic_embedding = False\n",
        "if use_syntatic_embedding != True:\n",
        "    train_syn_vectors = None\n",
        "    val_syn_vectors = None\n",
        "    test_syn_vectors = None\n",
        "    SYN_VECTOR_DIM = 0\n",
        "\n",
        "\n",
        "syn_vect_dim = SYN_VECTOR_DIM\n",
        "hidden_state_dim = 952\n",
        "epochs = 100\n",
        "\n",
        "model = BiLSTM_CRF(vocab_size=len(word_to_ix), \n",
        "                   tag_to_ix=tag_to_ix, \n",
        "                   word_embedding_dim=word_emb_dim, \n",
        "                   syntactic_feature_dim=syn_vect_dim,\n",
        "                   hidden_dim=hidden_state_dim,\n",
        "                   attention_method='transformer',\n",
        "                   # choose from ['dot_product', 'scaled_dot_product', 'cos', 'general', 'transformer']\n",
        "                   use_CRF=True)\n",
        "model.to(device)\n",
        "optimizer = optim.AdamW(model.parameters(), lr=0.0001, weight_decay=1e-4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d1ae17a7e62440ba8820680360083c80",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1c4d8469a4a0421e85b3d1984c500bf8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e045b0d4fe0e48349482831434acf6a6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6fc42d301203491eb04e15bccbd025e3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "08432afde20044aba54f7fef0b82d5db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lJsCPgDBnn5Z"
      },
      "source": [
        "### Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AttNpev4nq59",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "45d2dcb5-0df1-4f00-bbb5-32ca46e78a0d"
      },
      "source": [
        "\"\"\"Each epoch will take about 1-2 minutes\"\"\"\n",
        "\n",
        "import datetime\n",
        "val_loss_arr = []\n",
        "val_f1_arr = []\n",
        "\n",
        "for epoch in range(epochs):  \n",
        "    time1 = datetime.datetime.now()\n",
        "    train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "    for i, sent in enumerate(train_data):\n",
        "        tags_index = train_output_index[i]\n",
        "\n",
        "        # Step 1. Remember that Pytorch accumulates gradients.\n",
        "        # We need to clear them out before each instance\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Step 2. Get our inputs ready for the network, that is,µˆ\n",
        "        # turn them into Tensors of word indices.\n",
        "        if train_syn_vectors != None:\n",
        "            train_syn_vector = torch.tensor(train_syn_vectors[i], dtype=torch.long).to(device)\n",
        "        else:\n",
        "            train_syn_vector = None\n",
        "        # sentence_in = torch.tensor(sent, dtype=torch.long).to(device)\n",
        "        targets = torch.tensor(tags_index, dtype=torch.long).to(device)\n",
        "\n",
        "        # Step 3. Run our forward pass.\n",
        "        loss = model.neg_log_likelihood(sent, targets, train_syn_vector)\n",
        "        # print(\"step loss:\", loss)\n",
        "\n",
        "        # Step 4. Compute the loss, gradients, and update the parameters by calling optimizer.step()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        train_loss+=loss.item()\n",
        "\n",
        "    model.eval()\n",
        "    # Call the cal_acc functions you implemented as required\n",
        "    _, _, train_acc, train_f1 = cal_acc(model,train_data,train_output_index, train_syn_vectors)\n",
        "    _, _, val_acc, val_f1 = cal_acc(model,validation_data,val_output_index, val_syn_vectors)\n",
        "\n",
        "    val_loss = 0\n",
        "    for i, sent in enumerate(validation_data):\n",
        "        tags_index = val_output_index[i]\n",
        "        # sentence_in = torch.tensor(sent, dtype=torch.long).to(device)\n",
        "        targets = torch.tensor(tags_index, dtype=torch.long).to(device)\n",
        "        if val_syn_vectors != None:\n",
        "            val_syn_vector = torch.tensor(val_syn_vectors[i], dtype=torch.long).to(device)\n",
        "        else:\n",
        "            val_syn_vector = None\n",
        "\n",
        "        loss = model.neg_log_likelihood(sent, targets, val_syn_vector) \n",
        "        val_loss+=loss.item()\n",
        "        \n",
        "    time2 = datetime.datetime.now()\n",
        "\n",
        "    if len(val_f1_arr)==0 or val_f1 > val_f1_arr[-1]:\n",
        "        torch.save(model.state_dict(),'/content/best_model.pth')\n",
        "        \n",
        "    val_f1_arr.append(val_f1)\n",
        "    print(\"Epoch:%d, Training Loss: %.2f, Training f1: %.2f, train acc: %.4f, val f1: %.2f, val acc: %.4f, time: %.2fs\" %(epoch+1, train_loss, train_f1,train_acc, val_f1, val_acc, (time2-time1).total_seconds()))\n",
        "\n",
        "# The log below is the sample output for this section"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:1, Training Loss: 15583.01, Training f1: 0.82, train acc: 0.8217, val f1: 0.76, val acc: 0.7637, time: 256.01s\n",
            "Epoch:2, Training Loss: 8494.83, Training f1: 0.89, train acc: 0.8890, val f1: 0.81, val acc: 0.8125, time: 258.83s\n",
            "Epoch:3, Training Loss: 6183.07, Training f1: 0.91, train acc: 0.9099, val f1: 0.82, val acc: 0.8229, time: 259.73s\n",
            "Epoch:4, Training Loss: 4912.96, Training f1: 0.92, train acc: 0.9205, val f1: 0.82, val acc: 0.8248, time: 261.62s\n",
            "Epoch:5, Training Loss: 3981.74, Training f1: 0.93, train acc: 0.9290, val f1: 0.83, val acc: 0.8320, time: 260.45s\n",
            "Epoch:6, Training Loss: 3459.00, Training f1: 0.94, train acc: 0.9383, val f1: 0.83, val acc: 0.8273, time: 260.57s\n",
            "Epoch:7, Training Loss: 2919.77, Training f1: 0.94, train acc: 0.9437, val f1: 0.84, val acc: 0.8373, time: 264.64s\n",
            "Epoch:8, Training Loss: 2766.90, Training f1: 0.95, train acc: 0.9543, val f1: 0.83, val acc: 0.8322, time: 264.74s\n",
            "Epoch:9, Training Loss: 2345.42, Training f1: 0.94, train acc: 0.9448, val f1: 0.83, val acc: 0.8343, time: 261.23s\n",
            "Epoch:10, Training Loss: 2089.34, Training f1: 0.97, train acc: 0.9699, val f1: 0.84, val acc: 0.8386, time: 260.69s\n",
            "Epoch:11, Training Loss: 1587.79, Training f1: 0.97, train acc: 0.9731, val f1: 0.84, val acc: 0.8422, time: 263.83s\n",
            "Epoch:12, Training Loss: 1468.17, Training f1: 0.97, train acc: 0.9683, val f1: 0.83, val acc: 0.8316, time: 264.42s\n",
            "Epoch:13, Training Loss: 1514.00, Training f1: 0.97, train acc: 0.9708, val f1: 0.84, val acc: 0.8392, time: 263.19s\n",
            "Epoch:14, Training Loss: 1464.89, Training f1: 0.97, train acc: 0.9677, val f1: 0.82, val acc: 0.8218, time: 267.68s\n",
            "Epoch:15, Training Loss: 1173.80, Training f1: 0.98, train acc: 0.9771, val f1: 0.84, val acc: 0.8415, time: 266.11s\n",
            "Epoch:16, Training Loss: 1097.18, Training f1: 0.98, train acc: 0.9799, val f1: 0.84, val acc: 0.8366, time: 267.23s\n",
            "Epoch:17, Training Loss: 912.72, Training f1: 0.98, train acc: 0.9751, val f1: 0.83, val acc: 0.8295, time: 268.41s\n",
            "Epoch:18, Training Loss: 924.63, Training f1: 0.98, train acc: 0.9834, val f1: 0.84, val acc: 0.8362, time: 266.22s\n",
            "Epoch:19, Training Loss: 809.11, Training f1: 0.99, train acc: 0.9857, val f1: 0.84, val acc: 0.8440, time: 267.08s\n",
            "Epoch:20, Training Loss: 1570.30, Training f1: 0.98, train acc: 0.9779, val f1: 0.84, val acc: 0.8383, time: 261.03s\n",
            "Epoch:21, Training Loss: 878.15, Training f1: 0.98, train acc: 0.9827, val f1: 0.84, val acc: 0.8394, time: 263.09s\n",
            "Epoch:22, Training Loss: 757.96, Training f1: 0.98, train acc: 0.9792, val f1: 0.83, val acc: 0.8312, time: 264.37s\n",
            "Epoch:23, Training Loss: 739.24, Training f1: 0.99, train acc: 0.9872, val f1: 0.84, val acc: 0.8436, time: 265.27s\n",
            "Epoch:24, Training Loss: 675.98, Training f1: 0.99, train acc: 0.9864, val f1: 0.84, val acc: 0.8413, time: 261.41s\n",
            "Epoch:25, Training Loss: 607.36, Training f1: 0.99, train acc: 0.9898, val f1: 0.85, val acc: 0.8512, time: 261.62s\n",
            "Epoch:26, Training Loss: 648.20, Training f1: 0.98, train acc: 0.9833, val f1: 0.84, val acc: 0.8375, time: 261.83s\n",
            "Epoch:27, Training Loss: 657.40, Training f1: 0.99, train acc: 0.9857, val f1: 0.84, val acc: 0.8388, time: 261.88s\n",
            "Epoch:28, Training Loss: 750.00, Training f1: 0.99, train acc: 0.9869, val f1: 0.84, val acc: 0.8449, time: 261.99s\n",
            "Epoch:29, Training Loss: 607.84, Training f1: 0.99, train acc: 0.9853, val f1: 0.84, val acc: 0.8409, time: 262.33s\n",
            "Epoch:30, Training Loss: 532.89, Training f1: 0.99, train acc: 0.9901, val f1: 0.84, val acc: 0.8449, time: 262.02s\n",
            "Epoch:31, Training Loss: 512.36, Training f1: 0.99, train acc: 0.9908, val f1: 0.84, val acc: 0.8428, time: 258.20s\n",
            "Epoch:32, Training Loss: 506.94, Training f1: 0.99, train acc: 0.9904, val f1: 0.84, val acc: 0.8426, time: 258.26s\n",
            "Epoch:33, Training Loss: 509.31, Training f1: 0.99, train acc: 0.9882, val f1: 0.85, val acc: 0.8451, time: 259.57s\n",
            "Epoch:34, Training Loss: 674.73, Training f1: 0.98, train acc: 0.9828, val f1: 0.84, val acc: 0.8358, time: 260.31s\n",
            "Epoch:35, Training Loss: 476.20, Training f1: 0.98, train acc: 0.9832, val f1: 0.84, val acc: 0.8354, time: 260.47s\n",
            "Epoch:36, Training Loss: 512.10, Training f1: 0.99, train acc: 0.9889, val f1: 0.84, val acc: 0.8426, time: 263.23s\n",
            "Epoch:37, Training Loss: 489.62, Training f1: 0.99, train acc: 0.9920, val f1: 0.85, val acc: 0.8472, time: 262.91s\n",
            "Epoch:38, Training Loss: 512.59, Training f1: 0.99, train acc: 0.9891, val f1: 0.85, val acc: 0.8476, time: 262.67s\n",
            "Epoch:39, Training Loss: 413.03, Training f1: 0.99, train acc: 0.9860, val f1: 0.84, val acc: 0.8377, time: 263.74s\n",
            "Epoch:40, Training Loss: 536.81, Training f1: 0.99, train acc: 0.9887, val f1: 0.84, val acc: 0.8426, time: 263.07s\n",
            "Epoch:41, Training Loss: 414.46, Training f1: 0.99, train acc: 0.9884, val f1: 0.84, val acc: 0.8381, time: 264.74s\n",
            "Epoch:42, Training Loss: 413.74, Training f1: 0.98, train acc: 0.9849, val f1: 0.84, val acc: 0.8383, time: 260.19s\n",
            "Epoch:43, Training Loss: 444.94, Training f1: 0.99, train acc: 0.9915, val f1: 0.86, val acc: 0.8557, time: 260.80s\n",
            "Epoch:44, Training Loss: 404.05, Training f1: 0.99, train acc: 0.9906, val f1: 0.85, val acc: 0.8489, time: 261.32s\n",
            "Epoch:45, Training Loss: 612.15, Training f1: 0.99, train acc: 0.9862, val f1: 0.84, val acc: 0.8381, time: 261.89s\n",
            "Epoch:46, Training Loss: 501.87, Training f1: 0.99, train acc: 0.9911, val f1: 0.85, val acc: 0.8472, time: 260.91s\n",
            "Epoch:47, Training Loss: 295.11, Training f1: 0.99, train acc: 0.9894, val f1: 0.84, val acc: 0.8403, time: 264.13s\n",
            "Epoch:48, Training Loss: 299.31, Training f1: 0.99, train acc: 0.9934, val f1: 0.85, val acc: 0.8476, time: 261.97s\n",
            "Epoch:49, Training Loss: 310.02, Training f1: 0.99, train acc: 0.9940, val f1: 0.85, val acc: 0.8531, time: 262.17s\n",
            "Epoch:50, Training Loss: 309.92, Training f1: 0.99, train acc: 0.9902, val f1: 0.84, val acc: 0.8415, time: 265.28s\n",
            "Epoch:51, Training Loss: 597.46, Training f1: 0.99, train acc: 0.9917, val f1: 0.84, val acc: 0.8424, time: 266.54s\n",
            "Epoch:52, Training Loss: 430.06, Training f1: 0.99, train acc: 0.9942, val f1: 0.85, val acc: 0.8495, time: 262.29s\n",
            "Epoch:53, Training Loss: 288.83, Training f1: 0.99, train acc: 0.9941, val f1: 0.85, val acc: 0.8487, time: 258.62s\n",
            "Epoch:54, Training Loss: 282.48, Training f1: 0.99, train acc: 0.9932, val f1: 0.85, val acc: 0.8468, time: 263.41s\n",
            "Epoch:55, Training Loss: 311.85, Training f1: 0.99, train acc: 0.9895, val f1: 0.84, val acc: 0.8398, time: 263.23s\n",
            "Epoch:56, Training Loss: 555.99, Training f1: 0.99, train acc: 0.9877, val f1: 0.84, val acc: 0.8400, time: 260.29s\n",
            "Epoch:57, Training Loss: 352.17, Training f1: 0.99, train acc: 0.9944, val f1: 0.85, val acc: 0.8532, time: 260.91s\n",
            "Epoch:58, Training Loss: 287.32, Training f1: 1.00, train acc: 0.9951, val f1: 0.86, val acc: 0.8568, time: 260.60s\n",
            "Epoch:59, Training Loss: 309.70, Training f1: 0.99, train acc: 0.9937, val f1: 0.85, val acc: 0.8493, time: 260.94s\n",
            "Epoch:60, Training Loss: 340.91, Training f1: 0.99, train acc: 0.9923, val f1: 0.84, val acc: 0.8422, time: 261.31s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-53-b6b5ff23815b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0;31m# Step 3. Run our forward pass.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_syn_vector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0;31m# print(\"step loss:\", loss)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36mneg_log_likelihood\u001b[0;34m(self, sentence, tags, syntactic_feature)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msyntactic_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0mfeats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_lstm_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msyntactic_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m         \u001b[0mforward_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_alg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0mgold_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_score_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mforward_score\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mgold_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36m_forward_alg\u001b[0;34m(self, feats)\u001b[0m\n\u001b[1;32m    110\u001b[0m                 \u001b[0mnext_tag_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_var\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtrans_score\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0memit_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m                 \u001b[0;31m# The forward variable for this tag is log-sum-exp of all the scores.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m                 \u001b[0malphas_t\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_sum_exp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_tag_var\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m             \u001b[0mforward_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malphas_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0mterminal_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_var\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransitions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag_to_ix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mSTOP_TAG\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36mlog_sum_exp\u001b[0;34m(vec)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;31m# Compute log sum exp in a numerically stable way for the forward algorithm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlog_sum_exp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mmax_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvec\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mmax_score_broadcast\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_score\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmax_score\u001b[0m \u001b[0;34m+\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mmax_score_broadcast\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "nDlTtuXcC5MS",
        "outputId": "45d2dcb5-0df1-4f00-bbb5-32ca46e78a0d"
      },
      "source": [
        "\"\"\"Each epoch will take about 1-2 minutes\"\"\"\n",
        "\n",
        "import datetime\n",
        "val_loss_arr = []\n",
        "val_f1_arr = []\n",
        "\n",
        "for epoch in range(epochs):  \n",
        "    time1 = datetime.datetime.now()\n",
        "    train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "    for i, sent in enumerate(train_data):\n",
        "        tags_index = train_output_index[i]\n",
        "\n",
        "        # Step 1. Remember that Pytorch accumulates gradients.\n",
        "        # We need to clear them out before each instance\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Step 2. Get our inputs ready for the network, that is,µˆ\n",
        "        # turn them into Tensors of word indices.\n",
        "        if train_syn_vectors != None:\n",
        "            train_syn_vector = torch.tensor(train_syn_vectors[i], dtype=torch.long).to(device)\n",
        "        else:\n",
        "            train_syn_vector = None\n",
        "        # sentence_in = torch.tensor(sent, dtype=torch.long).to(device)\n",
        "        targets = torch.tensor(tags_index, dtype=torch.long).to(device)\n",
        "\n",
        "        # Step 3. Run our forward pass.\n",
        "        loss = model.neg_log_likelihood(sent, targets, train_syn_vector)\n",
        "        # print(\"step loss:\", loss)\n",
        "\n",
        "        # Step 4. Compute the loss, gradients, and update the parameters by calling optimizer.step()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        train_loss+=loss.item()\n",
        "\n",
        "    model.eval()\n",
        "    # Call the cal_acc functions you implemented as required\n",
        "    _, _, train_acc, train_f1 = cal_acc(model,train_data,train_output_index, train_syn_vectors)\n",
        "    _, _, val_acc, val_f1 = cal_acc(model,validation_data,val_output_index, val_syn_vectors)\n",
        "\n",
        "    val_loss = 0\n",
        "    for i, sent in enumerate(validation_data):\n",
        "        tags_index = val_output_index[i]\n",
        "        # sentence_in = torch.tensor(sent, dtype=torch.long).to(device)\n",
        "        targets = torch.tensor(tags_index, dtype=torch.long).to(device)\n",
        "        if val_syn_vectors != None:\n",
        "            val_syn_vector = torch.tensor(val_syn_vectors[i], dtype=torch.long).to(device)\n",
        "        else:\n",
        "            val_syn_vector = None\n",
        "\n",
        "        loss = model.neg_log_likelihood(sent, targets, val_syn_vector) \n",
        "        val_loss+=loss.item()\n",
        "        \n",
        "    time2 = datetime.datetime.now()\n",
        "\n",
        "    if len(val_f1_arr)==0 or val_f1 > val_f1_arr[-1]:\n",
        "        torch.save(model.state_dict(),'/content/best_model.pth')\n",
        "        \n",
        "    val_f1_arr.append(val_f1)\n",
        "    print(\"Epoch:%d, Training Loss: %.2f, Training f1: %.2f, train acc: %.4f, val f1: %.2f, val acc: %.4f, time: %.2fs\" %(epoch+1, train_loss, train_f1,train_acc, val_f1, val_acc, (time2-time1).total_seconds()))\n",
        "\n",
        "# The log below is the sample output for this section"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:1, Training Loss: 15583.01, Training f1: 0.82, train acc: 0.8217, val f1: 0.76, val acc: 0.7637, time: 256.01s\n",
            "Epoch:2, Training Loss: 8494.83, Training f1: 0.89, train acc: 0.8890, val f1: 0.81, val acc: 0.8125, time: 258.83s\n",
            "Epoch:3, Training Loss: 6183.07, Training f1: 0.91, train acc: 0.9099, val f1: 0.82, val acc: 0.8229, time: 259.73s\n",
            "Epoch:4, Training Loss: 4912.96, Training f1: 0.92, train acc: 0.9205, val f1: 0.82, val acc: 0.8248, time: 261.62s\n",
            "Epoch:5, Training Loss: 3981.74, Training f1: 0.93, train acc: 0.9290, val f1: 0.83, val acc: 0.8320, time: 260.45s\n",
            "Epoch:6, Training Loss: 3459.00, Training f1: 0.94, train acc: 0.9383, val f1: 0.83, val acc: 0.8273, time: 260.57s\n",
            "Epoch:7, Training Loss: 2919.77, Training f1: 0.94, train acc: 0.9437, val f1: 0.84, val acc: 0.8373, time: 264.64s\n",
            "Epoch:8, Training Loss: 2766.90, Training f1: 0.95, train acc: 0.9543, val f1: 0.83, val acc: 0.8322, time: 264.74s\n",
            "Epoch:9, Training Loss: 2345.42, Training f1: 0.94, train acc: 0.9448, val f1: 0.83, val acc: 0.8343, time: 261.23s\n",
            "Epoch:10, Training Loss: 2089.34, Training f1: 0.97, train acc: 0.9699, val f1: 0.84, val acc: 0.8386, time: 260.69s\n",
            "Epoch:11, Training Loss: 1587.79, Training f1: 0.97, train acc: 0.9731, val f1: 0.84, val acc: 0.8422, time: 263.83s\n",
            "Epoch:12, Training Loss: 1468.17, Training f1: 0.97, train acc: 0.9683, val f1: 0.83, val acc: 0.8316, time: 264.42s\n",
            "Epoch:13, Training Loss: 1514.00, Training f1: 0.97, train acc: 0.9708, val f1: 0.84, val acc: 0.8392, time: 263.19s\n",
            "Epoch:14, Training Loss: 1464.89, Training f1: 0.97, train acc: 0.9677, val f1: 0.82, val acc: 0.8218, time: 267.68s\n",
            "Epoch:15, Training Loss: 1173.80, Training f1: 0.98, train acc: 0.9771, val f1: 0.84, val acc: 0.8415, time: 266.11s\n",
            "Epoch:16, Training Loss: 1097.18, Training f1: 0.98, train acc: 0.9799, val f1: 0.84, val acc: 0.8366, time: 267.23s\n",
            "Epoch:17, Training Loss: 912.72, Training f1: 0.98, train acc: 0.9751, val f1: 0.83, val acc: 0.8295, time: 268.41s\n",
            "Epoch:18, Training Loss: 924.63, Training f1: 0.98, train acc: 0.9834, val f1: 0.84, val acc: 0.8362, time: 266.22s\n",
            "Epoch:19, Training Loss: 809.11, Training f1: 0.99, train acc: 0.9857, val f1: 0.84, val acc: 0.8440, time: 267.08s\n",
            "Epoch:20, Training Loss: 1570.30, Training f1: 0.98, train acc: 0.9779, val f1: 0.84, val acc: 0.8383, time: 261.03s\n",
            "Epoch:21, Training Loss: 878.15, Training f1: 0.98, train acc: 0.9827, val f1: 0.84, val acc: 0.8394, time: 263.09s\n",
            "Epoch:22, Training Loss: 757.96, Training f1: 0.98, train acc: 0.9792, val f1: 0.83, val acc: 0.8312, time: 264.37s\n",
            "Epoch:23, Training Loss: 739.24, Training f1: 0.99, train acc: 0.9872, val f1: 0.84, val acc: 0.8436, time: 265.27s\n",
            "Epoch:24, Training Loss: 675.98, Training f1: 0.99, train acc: 0.9864, val f1: 0.84, val acc: 0.8413, time: 261.41s\n",
            "Epoch:25, Training Loss: 607.36, Training f1: 0.99, train acc: 0.9898, val f1: 0.85, val acc: 0.8512, time: 261.62s\n",
            "Epoch:26, Training Loss: 648.20, Training f1: 0.98, train acc: 0.9833, val f1: 0.84, val acc: 0.8375, time: 261.83s\n",
            "Epoch:27, Training Loss: 657.40, Training f1: 0.99, train acc: 0.9857, val f1: 0.84, val acc: 0.8388, time: 261.88s\n",
            "Epoch:28, Training Loss: 750.00, Training f1: 0.99, train acc: 0.9869, val f1: 0.84, val acc: 0.8449, time: 261.99s\n",
            "Epoch:29, Training Loss: 607.84, Training f1: 0.99, train acc: 0.9853, val f1: 0.84, val acc: 0.8409, time: 262.33s\n",
            "Epoch:30, Training Loss: 532.89, Training f1: 0.99, train acc: 0.9901, val f1: 0.84, val acc: 0.8449, time: 262.02s\n",
            "Epoch:31, Training Loss: 512.36, Training f1: 0.99, train acc: 0.9908, val f1: 0.84, val acc: 0.8428, time: 258.20s\n",
            "Epoch:32, Training Loss: 506.94, Training f1: 0.99, train acc: 0.9904, val f1: 0.84, val acc: 0.8426, time: 258.26s\n",
            "Epoch:33, Training Loss: 509.31, Training f1: 0.99, train acc: 0.9882, val f1: 0.85, val acc: 0.8451, time: 259.57s\n",
            "Epoch:34, Training Loss: 674.73, Training f1: 0.98, train acc: 0.9828, val f1: 0.84, val acc: 0.8358, time: 260.31s\n",
            "Epoch:35, Training Loss: 476.20, Training f1: 0.98, train acc: 0.9832, val f1: 0.84, val acc: 0.8354, time: 260.47s\n",
            "Epoch:36, Training Loss: 512.10, Training f1: 0.99, train acc: 0.9889, val f1: 0.84, val acc: 0.8426, time: 263.23s\n",
            "Epoch:37, Training Loss: 489.62, Training f1: 0.99, train acc: 0.9920, val f1: 0.85, val acc: 0.8472, time: 262.91s\n",
            "Epoch:38, Training Loss: 512.59, Training f1: 0.99, train acc: 0.9891, val f1: 0.85, val acc: 0.8476, time: 262.67s\n",
            "Epoch:39, Training Loss: 413.03, Training f1: 0.99, train acc: 0.9860, val f1: 0.84, val acc: 0.8377, time: 263.74s\n",
            "Epoch:40, Training Loss: 536.81, Training f1: 0.99, train acc: 0.9887, val f1: 0.84, val acc: 0.8426, time: 263.07s\n",
            "Epoch:41, Training Loss: 414.46, Training f1: 0.99, train acc: 0.9884, val f1: 0.84, val acc: 0.8381, time: 264.74s\n",
            "Epoch:42, Training Loss: 413.74, Training f1: 0.98, train acc: 0.9849, val f1: 0.84, val acc: 0.8383, time: 260.19s\n",
            "Epoch:43, Training Loss: 444.94, Training f1: 0.99, train acc: 0.9915, val f1: 0.86, val acc: 0.8557, time: 260.80s\n",
            "Epoch:44, Training Loss: 404.05, Training f1: 0.99, train acc: 0.9906, val f1: 0.85, val acc: 0.8489, time: 261.32s\n",
            "Epoch:45, Training Loss: 612.15, Training f1: 0.99, train acc: 0.9862, val f1: 0.84, val acc: 0.8381, time: 261.89s\n",
            "Epoch:46, Training Loss: 501.87, Training f1: 0.99, train acc: 0.9911, val f1: 0.85, val acc: 0.8472, time: 260.91s\n",
            "Epoch:47, Training Loss: 295.11, Training f1: 0.99, train acc: 0.9894, val f1: 0.84, val acc: 0.8403, time: 264.13s\n",
            "Epoch:48, Training Loss: 299.31, Training f1: 0.99, train acc: 0.9934, val f1: 0.85, val acc: 0.8476, time: 261.97s\n",
            "Epoch:49, Training Loss: 310.02, Training f1: 0.99, train acc: 0.9940, val f1: 0.85, val acc: 0.8531, time: 262.17s\n",
            "Epoch:50, Training Loss: 309.92, Training f1: 0.99, train acc: 0.9902, val f1: 0.84, val acc: 0.8415, time: 265.28s\n",
            "Epoch:51, Training Loss: 597.46, Training f1: 0.99, train acc: 0.9917, val f1: 0.84, val acc: 0.8424, time: 266.54s\n",
            "Epoch:52, Training Loss: 430.06, Training f1: 0.99, train acc: 0.9942, val f1: 0.85, val acc: 0.8495, time: 262.29s\n",
            "Epoch:53, Training Loss: 288.83, Training f1: 0.99, train acc: 0.9941, val f1: 0.85, val acc: 0.8487, time: 258.62s\n",
            "Epoch:54, Training Loss: 282.48, Training f1: 0.99, train acc: 0.9932, val f1: 0.85, val acc: 0.8468, time: 263.41s\n",
            "Epoch:55, Training Loss: 311.85, Training f1: 0.99, train acc: 0.9895, val f1: 0.84, val acc: 0.8398, time: 263.23s\n",
            "Epoch:56, Training Loss: 555.99, Training f1: 0.99, train acc: 0.9877, val f1: 0.84, val acc: 0.8400, time: 260.29s\n",
            "Epoch:57, Training Loss: 352.17, Training f1: 0.99, train acc: 0.9944, val f1: 0.85, val acc: 0.8532, time: 260.91s\n",
            "Epoch:58, Training Loss: 287.32, Training f1: 1.00, train acc: 0.9951, val f1: 0.86, val acc: 0.8568, time: 260.60s\n",
            "Epoch:59, Training Loss: 309.70, Training f1: 0.99, train acc: 0.9937, val f1: 0.85, val acc: 0.8493, time: 260.94s\n",
            "Epoch:60, Training Loss: 340.91, Training f1: 0.99, train acc: 0.9923, val f1: 0.84, val acc: 0.8422, time: 261.31s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-53-b6b5ff23815b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0;31m# Step 3. Run our forward pass.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_syn_vector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0;31m# print(\"step loss:\", loss)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36mneg_log_likelihood\u001b[0;34m(self, sentence, tags, syntactic_feature)\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mneg_log_likelihood\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msyntactic_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0mfeats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_lstm_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msyntactic_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m         \u001b[0mforward_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_alg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0mgold_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_score_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mforward_score\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mgold_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36m_forward_alg\u001b[0;34m(self, feats)\u001b[0m\n\u001b[1;32m    110\u001b[0m                 \u001b[0mnext_tag_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_var\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtrans_score\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0memit_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m                 \u001b[0;31m# The forward variable for this tag is log-sum-exp of all the scores.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m                 \u001b[0malphas_t\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_sum_exp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_tag_var\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m             \u001b[0mforward_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malphas_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0mterminal_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_var\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransitions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag_to_ix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mSTOP_TAG\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-44-78f4e40fbf6e>\u001b[0m in \u001b[0;36mlog_sum_exp\u001b[0;34m(vec)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;31m# Compute log sum exp in a numerically stable way for the forward algorithm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlog_sum_exp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mmax_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvec\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mmax_score_broadcast\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_score\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmax_score\u001b[0m \u001b[0;34m+\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvec\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mmax_score_broadcast\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}