eng-t5-base-fine-tuning.ipynb

{"metadata":{"colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"7a87354679284607856a96e964776b75":{"model_module":"@jupyter-widgets/controls","model_name":"VBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"VBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"VBoxView","box_style":"","children":["IPY_MODEL_b53c3d9605864b0384bb6de516c760d0","IPY_MODEL_753ff141cb314ad1993a5a3bd21df347","IPY_MODEL_15595c9f66ed426ab17b6a8e1c4f19f5","IPY_MODEL_1f9f29d277244f41bc475af36a743b4f"],"layout":"IPY_MODEL_157acaae577342e498eb47ee2b2852b0"}},"29f9047f40214144b350e789ac0f4b79":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2041f05c579a4e6e9c5dceb89f762e42","placeholder":"​","style":"IPY_MODEL_8e5c8ba1df5d457d895e23ff1150e697","value":"<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"}},"77ba3e29cbf2466991a6417cdbd89a37":{"model_module":"@jupyter-widgets/controls","model_name":"PasswordModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"PasswordModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"PasswordView","continuous_update":true,"description":"Token:","description_tooltip":null,"disabled":false,"layout":"IPY_MODEL_7e2a659975274c129d0d288c6f9f241d","placeholder":"​","style":"IPY_MODEL_dd7b192465034301a47a8c3d06ee6854","value":""}},"f43d6291e033464d9df831f5c5369a71":{"model_module":"@jupyter-widgets/controls","model_name":"CheckboxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"CheckboxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"CheckboxView","description":"Add token as git credential?","description_tooltip":null,"disabled":false,"indent":true,"layout":"IPY_MODEL_302a9c3f7d6040ef8e665bf180fb1def","style":"IPY_MODEL_1e5ed6d832a74ddba8cfe30bfde92620","value":true}},"6509ca55b1604c4bb58055a43fd27e27":{"model_module":"@jupyter-widgets/controls","model_name":"ButtonModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ButtonModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ButtonView","button_style":"","description":"Login","disabled":false,"icon":"","layout":"IPY_MODEL_844a90cd4d35417d82f6a756cb4edd86","style":"IPY_MODEL_2f5d3d015f4e4b16ba05b2bf4b28911f","tooltip":""}},"b86ba99bba8a433e849677803ad21397":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9f3423b609a64efc87dee963a27a29c5","placeholder":"​","style":"IPY_MODEL_ff4dd03bacbc456c99cf44852be9a1a3","value":"\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"}},"157acaae577342e498eb47ee2b2852b0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":"center","align_self":null,"border":null,"bottom":null,"display":"flex","flex":null,"flex_flow":"column","grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"50%"}},"2041f05c579a4e6e9c5dceb89f762e42":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8e5c8ba1df5d457d895e23ff1150e697":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7e2a659975274c129d0d288c6f9f241d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd7b192465034301a47a8c3d06ee6854":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"302a9c3f7d6040ef8e665bf180fb1def":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1e5ed6d832a74ddba8cfe30bfde92620":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"844a90cd4d35417d82f6a756cb4edd86":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2f5d3d015f4e4b16ba05b2bf4b28911f":{"model_module":"@jupyter-widgets/controls","model_name":"ButtonStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ButtonStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","button_color":null,"font_weight":""}},"9f3423b609a64efc87dee963a27a29c5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ff4dd03bacbc456c99cf44852be9a1a3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"51eb3a8600ba440ebbb59bd7d61a1f31":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2756df5e868a49cc8f2ea5b18aefc6de","placeholder":"​","style":"IPY_MODEL_b379d86bf9c64be39a9bebd0fe71e321","value":"Connecting..."}},"2756df5e868a49cc8f2ea5b18aefc6de":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b379d86bf9c64be39a9bebd0fe71e321":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b53c3d9605864b0384bb6de516c760d0":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7e3801169d4b4b9996b51266bebaada5","placeholder":"​","style":"IPY_MODEL_02f49b01273b486f863c57d98dca6e10","value":"Token is valid (permission: write)."}},"753ff141cb314ad1993a5a3bd21df347":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_92c81668e7b04567afe8f2817b9f7bdf","placeholder":"​","style":"IPY_MODEL_b88ec3c5a9904c789fd47d0855b6c189","value":"Your token has been saved in your configured git credential helpers (store)."}},"15595c9f66ed426ab17b6a8e1c4f19f5":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_facda9dba37a46cba150de16794c28c8","placeholder":"​","style":"IPY_MODEL_ca3a6217ef5046018df2ad8f89e44d80","value":"Your token has been saved to /root/.cache/huggingface/token"}},"1f9f29d277244f41bc475af36a743b4f":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2c2d5a54107646748fef838a6377f2b4","placeholder":"​","style":"IPY_MODEL_31902d2b0c674ded94cd09d8d5f7fa1a","value":"Login successful"}},"7e3801169d4b4b9996b51266bebaada5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"02f49b01273b486f863c57d98dca6e10":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"92c81668e7b04567afe8f2817b9f7bdf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b88ec3c5a9904c789fd47d0855b6c189":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"facda9dba37a46cba150de16794c28c8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ca3a6217ef5046018df2ad8f89e44d80":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2c2d5a54107646748fef838a6377f2b4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"31902d2b0c674ded94cd09d8d5f7fa1a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"72cd4ac7f858410b86570323aab41633":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_499f95f790a14f6c945f61b9c917661c","IPY_MODEL_cc19cb46fb1a4a268d7b2e2ee6181552","IPY_MODEL_07b8b7a67c564594bfd177ed0c3ece3d"],"layout":"IPY_MODEL_38155a5024214b43999cf911067d54d4"}},"499f95f790a14f6c945f61b9c917661c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_452e8162e27b4c648bcf6311649e2680","placeholder":"​","style":"IPY_MODEL_09dcef36854d41b79fa2767cbef5f52e","value":"tokenizer_config.json: 100%"}},"cc19cb46fb1a4a268d7b2e2ee6181552":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_bb5eb7c53de24c7ba28a62f9c5d8491b","max":2372,"min":0,"orientation":"horizontal","style":"IPY_MODEL_fdf3bf977e104312b509841b4bf60621","value":2372}},"07b8b7a67c564594bfd177ed0c3ece3d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_87fe443f3686475b8ac9ce3b22e031b6","placeholder":"​","style":"IPY_MODEL_29129adcffb14e0881ad9614edf568f6","value":" 2.37k/2.37k [00:00&lt;00:00, 91.9kB/s]"}},"38155a5024214b43999cf911067d54d4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"452e8162e27b4c648bcf6311649e2680":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"09dcef36854d41b79fa2767cbef5f52e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bb5eb7c53de24c7ba28a62f9c5d8491b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fdf3bf977e104312b509841b4bf60621":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"87fe443f3686475b8ac9ce3b22e031b6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"29129adcffb14e0881ad9614edf568f6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cb7ce01dc44b45c091361e9f47221d97":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_38211a59b71c427b95d6577dca9b3881","IPY_MODEL_a7548ae28fa94be1b77ed2544bdacac8","IPY_MODEL_e2e1af508df04a0181e75893cc3eea60"],"layout":"IPY_MODEL_9cd475c30e164b4ca1951591da838450"}},"38211a59b71c427b95d6577dca9b3881":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2cc10bd71f224be4b02427eece6ccd5a","placeholder":"​","style":"IPY_MODEL_47779f0da3474bdc8da70100c44f2813","value":"spiece.model: 100%"}},"a7548ae28fa94be1b77ed2544bdacac8":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_0eb4d8475f52497281dedfd3a9ad7313","max":2353338,"min":0,"orientation":"horizontal","style":"IPY_MODEL_676f74cd1ba643eca22e8a151b1b55a5","value":2353338}},"e2e1af508df04a0181e75893cc3eea60":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6aa42b17a4af4d73a395fb63a847d7e9","placeholder":"​","style":"IPY_MODEL_4df30519e470448ca357732bdd4be208","value":" 2.35M/2.35M [00:01&lt;00:00, 1.42MB/s]"}},"9cd475c30e164b4ca1951591da838450":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2cc10bd71f224be4b02427eece6ccd5a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"47779f0da3474bdc8da70100c44f2813":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0eb4d8475f52497281dedfd3a9ad7313":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"676f74cd1ba643eca22e8a151b1b55a5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6aa42b17a4af4d73a395fb63a847d7e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4df30519e470448ca357732bdd4be208":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ae69437766f04b11b7fdfd0349421c04":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_02fcde39f43746e799637616b15dc03c","IPY_MODEL_c9ffdde9c4ac4273a728ba550652a340","IPY_MODEL_81d53c64c101437bbb4bd8bd4debadc7"],"layout":"IPY_MODEL_55ba919bc88740599bae54b1fcae3a65"}},"02fcde39f43746e799637616b15dc03c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6e94d913ac6d4bebbc0cd06341506edf","placeholder":"​","style":"IPY_MODEL_cd8d013bcede4182b73de9f8ae467f42","value":"tokenizer.json: 100%"}},"c9ffdde9c4ac4273a728ba550652a340":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_eb8ecdf7006a4ef8b0f998bf1d154e8c","max":8398761,"min":0,"orientation":"horizontal","style":"IPY_MODEL_77c7eeaabd164cb5b376b1f7f9bb2d6c","value":8398761}},"81d53c64c101437bbb4bd8bd4debadc7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2f676a2a989d4aefaeb9a772a6443257","placeholder":"​","style":"IPY_MODEL_0366d7ec6a2d4f2da497b1e2c20c0a1e","value":" 8.40M/8.40M [00:00&lt;00:00, 11.3MB/s]"}},"55ba919bc88740599bae54b1fcae3a65":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6e94d913ac6d4bebbc0cd06341506edf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cd8d013bcede4182b73de9f8ae467f42":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"eb8ecdf7006a4ef8b0f998bf1d154e8c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"77c7eeaabd164cb5b376b1f7f9bb2d6c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2f676a2a989d4aefaeb9a772a6443257":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0366d7ec6a2d4f2da497b1e2c20c0a1e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0a0e887085b94753993e16311fc7581f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a864ce7a69204aa08dfd00243a7d5432","IPY_MODEL_1942422f0c2740d6af6cc4dd3db6391e","IPY_MODEL_4329c338caf0415f956505d03c4d35ec"],"layout":"IPY_MODEL_4bea5e8b6e0f480d88a4f8769e8aa3b3"}},"a864ce7a69204aa08dfd00243a7d5432":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_dd45a072109f49cababfa967a4429fd1","placeholder":"​","style":"IPY_MODEL_a3c90a63a2884f26920e825fdc5b8534","value":"special_tokens_map.json: 100%"}},"1942422f0c2740d6af6cc4dd3db6391e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_879ff256e24a42fba5ba77dc65bf5bea","max":2201,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8780d4ec58fc44459e63449f30b087c2","value":2201}},"4329c338caf0415f956505d03c4d35ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_66cc7b6cf2904deb9739bfbd16e5afc8","placeholder":"​","style":"IPY_MODEL_0a9adb96721e4245b65d69ac923521c1","value":" 2.20k/2.20k [00:00&lt;00:00, 79.9kB/s]"}},"4bea5e8b6e0f480d88a4f8769e8aa3b3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd45a072109f49cababfa967a4429fd1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a3c90a63a2884f26920e825fdc5b8534":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"879ff256e24a42fba5ba77dc65bf5bea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8780d4ec58fc44459e63449f30b087c2":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"66cc7b6cf2904deb9739bfbd16e5afc8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0a9adb96721e4245b65d69ac923521c1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"90f8604eef2946c0b3b7f5a317907619":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5a20378209cf4d81bb579ef65f0283f5","IPY_MODEL_da39791dd2524897906cdf05e6db13fa","IPY_MODEL_9d1a6c5a4ae546b59a8c7a891c401b78"],"layout":"IPY_MODEL_0eb0f875feb142fc84871a7d70607d5d"}},"5a20378209cf4d81bb579ef65f0283f5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bfff98696a524727a5bd577688c98ff4","placeholder":"​","style":"IPY_MODEL_c03f936c06de4f7eb3f45537259436f1","value":"Map: 100%"}},"da39791dd2524897906cdf05e6db13fa":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9b3ccf3f3fd24411bb7fc631943c416f","max":257,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3e6af224dc104fdc8df096754d638c10","value":257}},"9d1a6c5a4ae546b59a8c7a891c401b78":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f1fc23f080a04b218770f3d117e0d01b","placeholder":"​","style":"IPY_MODEL_de6122ae036c45d1bf3c2c958c770d72","value":" 257/257 [00:03&lt;00:00, 81.85 examples/s]"}},"0eb0f875feb142fc84871a7d70607d5d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bfff98696a524727a5bd577688c98ff4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c03f936c06de4f7eb3f45537259436f1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9b3ccf3f3fd24411bb7fc631943c416f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3e6af224dc104fdc8df096754d638c10":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f1fc23f080a04b218770f3d117e0d01b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"de6122ae036c45d1bf3c2c958c770d72":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"39571c5fd99341088bbe2b1468b2cbfa":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_1d76cd894d784924887119cc37f9f9de","IPY_MODEL_3c8c19a7d3a8429ca60235c160247773","IPY_MODEL_6e8491d0ed24451b9c73f550045d5831"],"layout":"IPY_MODEL_e80260377d204739bfe338c46a672142"}},"1d76cd894d784924887119cc37f9f9de":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_27914c75d8844e26908ad9aa59658533","placeholder":"​","style":"IPY_MODEL_63f6b757767c4a2497dd8b11a0ec1819","value":"Map: 100%"}},"3c8c19a7d3a8429ca60235c160247773":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_43866d00aae741a1aa54820871e79d3a","max":32,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3145cdd6282e45dcb855aeea0c8fe085","value":32}},"6e8491d0ed24451b9c73f550045d5831":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_41fad7e68e8f4497a8bc2462fa38d499","placeholder":"​","style":"IPY_MODEL_fec631873e564b8289b9c0bee9c99919","value":" 32/32 [00:00&lt;00:00, 76.71 examples/s]"}},"e80260377d204739bfe338c46a672142":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"27914c75d8844e26908ad9aa59658533":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"63f6b757767c4a2497dd8b11a0ec1819":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"43866d00aae741a1aa54820871e79d3a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3145cdd6282e45dcb855aeea0c8fe085":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"41fad7e68e8f4497a8bc2462fa38d499":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fec631873e564b8289b9c0bee9c99919":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4ad2f56305584e2680eb2b8750139119":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_73ccf0955589437db23ac177abdf6394","IPY_MODEL_acf248f4ac984ccf923bbe1f95479ee2","IPY_MODEL_e1af6e78616c47c9a22a400d829c4d38"],"layout":"IPY_MODEL_91ccb01ad08940a0a35a7a97e20d6ead"}},"73ccf0955589437db23ac177abdf6394":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_39df35251d77454f899e7ed6c968329c","placeholder":"​","style":"IPY_MODEL_5c3ae211b79241a09bf2414f44b04e01","value":"Map: 100%"}},"acf248f4ac984ccf923bbe1f95479ee2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c9e2fc7e6eec489abab311ef4feece71","max":33,"min":0,"orientation":"horizontal","style":"IPY_MODEL_62f693f7297a44efa19ca63ba6f9a290","value":33}},"e1af6e78616c47c9a22a400d829c4d38":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_99e07262563c4e47a56ca584f5a61086","placeholder":"​","style":"IPY_MODEL_43b9ce7c48504a28be5bcb5e3761ab18","value":" 33/33 [00:00&lt;00:00, 145.05 examples/s]"}},"91ccb01ad08940a0a35a7a97e20d6ead":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"39df35251d77454f899e7ed6c968329c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5c3ae211b79241a09bf2414f44b04e01":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c9e2fc7e6eec489abab311ef4feece71":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"62f693f7297a44efa19ca63ba6f9a290":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"99e07262563c4e47a56ca584f5a61086":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"43b9ce7c48504a28be5bcb5e3761ab18":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"c735f04942e04399a3a32fa7083d00da":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e825de756e9b4ac394cee5265fed5fbc","IPY_MODEL_8fde228b35984f23a6bcf592fecd7a8a","IPY_MODEL_f45221155a0049dcbd0f9605fee9bc68"],"layout":"IPY_MODEL_ca2d0bbb3cc1431c8854a68bfec7e858"}},"e825de756e9b4ac394cee5265fed5fbc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1720303e03fc462bb546c93c404211f9","placeholder":"​","style":"IPY_MODEL_301d4d5a0c564c989858a9277b48247b","value":"config.json: 100%"}},"8fde228b35984f23a6bcf592fecd7a8a":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8ff47b220a6841b08baa3e16469cdca7","max":699,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4cdaabf445344942b6143afaafc88cf8","value":699}},"f45221155a0049dcbd0f9605fee9bc68":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_28e3b80b42224dd9b51bff4ee310e618","placeholder":"​","style":"IPY_MODEL_1f57df1b283e4d0a9193e7a61095aed9","value":" 699/699 [00:00&lt;00:00, 50.4kB/s]"}},"ca2d0bbb3cc1431c8854a68bfec7e858":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1720303e03fc462bb546c93c404211f9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"301d4d5a0c564c989858a9277b48247b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8ff47b220a6841b08baa3e16469cdca7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4cdaabf445344942b6143afaafc88cf8":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"28e3b80b42224dd9b51bff4ee310e618":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1f57df1b283e4d0a9193e7a61095aed9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8b6a155870bd496f8223f3ef5c3937f5":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ff0d2cd96cc4429b86c8a0605e5dbe6e","IPY_MODEL_8251dacbaf2b4f69b0577916f02cc874","IPY_MODEL_58c2ae77798d4a29b3d5b960cb9a9996"],"layout":"IPY_MODEL_c222d40fddb14edc8b6f19b9c5642fe9"}},"ff0d2cd96cc4429b86c8a0605e5dbe6e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8e2e014fcb7c416da4ed03e6cb56f2cb","placeholder":"​","style":"IPY_MODEL_07af474d2c8c496e9b98d463300238b5","value":"pytorch_model.bin: 100%"}},"8251dacbaf2b4f69b0577916f02cc874":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_1f1be5e8f89b4df3815974a6151c1fdd","max":1470128437,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2d8cdeb7dab04ca18101db44af632456","value":1470128437}},"58c2ae77798d4a29b3d5b960cb9a9996":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a883ba5951d041878c1cdd355164f3ad","placeholder":"​","style":"IPY_MODEL_34cd6a613d974656872fc4e57a4779ce","value":" 1.47G/1.47G [01:37&lt;00:00, 14.3MB/s]"}},"c222d40fddb14edc8b6f19b9c5642fe9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8e2e014fcb7c416da4ed03e6cb56f2cb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"07af474d2c8c496e9b98d463300238b5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1f1be5e8f89b4df3815974a6151c1fdd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2d8cdeb7dab04ca18101db44af632456":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a883ba5951d041878c1cdd355164f3ad":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"34cd6a613d974656872fc4e57a4779ce":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8da7771a0f864f668e99e9d3a2188027":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d7e09ac252234a89aef450f8bfb65f05","IPY_MODEL_ed431543cd2747ec82d0b1b9303bbb73","IPY_MODEL_7eb4ee8ab86d4db192f5733be690ce13"],"layout":"IPY_MODEL_8ed193e58b18483e93c38212354c166e"}},"d7e09ac252234a89aef450f8bfb65f05":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d3cbfb12e9d74866aec89b9ca4fb9b2b","placeholder":"​","style":"IPY_MODEL_a7877837d6914361bc3a25b9f664aef0","value":"generation_config.json: 100%"}},"ed431543cd2747ec82d0b1b9303bbb73":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9363975b9311451e8e234006dfacd9d3","max":142,"min":0,"orientation":"horizontal","style":"IPY_MODEL_672b13b6832848b1aed01523035ff221","value":142}},"7eb4ee8ab86d4db192f5733be690ce13":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b6d75f9c89af4ef0886d823fd64edb0c","placeholder":"​","style":"IPY_MODEL_867ea513998b47a8aae77cecb97f6825","value":" 142/142 [00:00&lt;00:00, 9.16kB/s]"}},"8ed193e58b18483e93c38212354c166e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d3cbfb12e9d74866aec89b9ca4fb9b2b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a7877837d6914361bc3a25b9f664aef0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9363975b9311451e8e234006dfacd9d3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"672b13b6832848b1aed01523035ff221":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b6d75f9c89af4ef0886d823fd64edb0c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"867ea513998b47a8aae77cecb97f6825":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7305368,"sourceType":"datasetVersion","datasetId":4238614},{"sourceId":7305587,"sourceType":"datasetVersion","datasetId":4238745},{"sourceId":7305683,"sourceType":"datasetVersion","datasetId":4238805}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"If you're opening this Notebook on colab, you will probably need to install 🤗 Transformers and 🤗 Datasets as well as other dependencies. Uncomment the following cell and run it.","metadata":{"id":"X4cRE8IbIrIV"}},{"cell_type":"code","source":"! pip install evaluate transformers rouge-score nltk","metadata":{"id":"MOsHUjgdIrIW","outputId":"a788ca68-ca3f-4679-86f1-3fd587d5c6ac","execution":{"iopub.status.busy":"2023-12-30T10:29:46.127973Z","iopub.execute_input":"2023-12-30T10:29:46.128287Z","iopub.status.idle":"2023-12-30T10:30:02.349627Z","shell.execute_reply.started":"2023-12-30T10:29:46.128262Z","shell.execute_reply":"2023-12-30T10:30:02.348530Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting evaluate\n  Obtaining dependency information for evaluate from https://files.pythonhosted.org/packages/70/63/7644a1eb7b0297e585a6adec98ed9e575309bb973c33b394dae66bc35c69/evaluate-0.4.1-py3-none-any.whl.metadata\n  Downloading evaluate-0.4.1-py3-none-any.whl.metadata (9.4 kB)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.36.0)\nCollecting rouge-score\n  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (3.2.4)\nRequirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.0)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from evaluate) (1.24.3)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.7)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.0.3)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.31.0)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.15)\nRequirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2023.12.2)\nRequirement already satisfied: huggingface-hub>=0.7.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.19.4)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from evaluate) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.18.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.2)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.8.8)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.15.0)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.1)\nRequirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.4.0)\nRequirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge-score) (1.16.0)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.5)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->evaluate) (3.0.9)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.2.0)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.4)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2023.11.17)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\nRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\nDownloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hBuilding wheels for collected packages: rouge-score\n  Building wheel for rouge-score (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24932 sha256=45831a46aa490c10140c776f44dbc7a931d1907a07617db535dc676fed94820c\n  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\nSuccessfully built rouge-score\nInstalling collected packages: rouge-score, evaluate\nSuccessfully installed evaluate-0.4.1 rouge-score-0.1.2\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Then you need to install Git-LFS. Uncomment the following instructions:","metadata":{"id":"rJ9pHqpuf3MS"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nfrom transformers import T5Tokenizer, T5ForConditionalGeneration  \n\nfrom transformers import AdamW\nimport pandas as pd\nimport torch\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom torch.nn.utils.rnn import pad_sequence\n# from torch.utils.data import Dataset, DataLoader, random_split, RandomSampler, SequentialSampler\n\npl.seed_everything(100)\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-12-30T08:47:17.747228Z","iopub.execute_input":"2023-12-30T08:47:17.747633Z","iopub.status.idle":"2023-12-30T08:47:28.272770Z","shell.execute_reply.started":"2023-12-30T08:47:17.747595Z","shell.execute_reply":"2023-12-30T08:47:28.271603Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"Here PyTorch-lightning is used: PyTorch Lightning is a lightweight interface for PyTorch that simplifies the process of training deep learning models. It provides pre-built components and features for common tasks, making the code more modular and reusable. PyTorch Lightning also provides various features such as automatic checkpointing, distributed training, and multi-GPU training. It follows a strict design pattern and provides hooks and callbacks for customization. It is compatible with various hardware platforms such as CPUs and GPU","metadata":{}},{"cell_type":"code","source":"DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nINPUT_MAX_LEN = 512 #input length\nOUTPUT_MAX_LEN = 128 # output length\nTRAIN_BATCH_SIZE = 8 # batch size of training\nVAL_BATCH_SIZE = 2 # batch size for validation\nEPOCHS = 5 # number of epoch","metadata":{"execution":{"iopub.status.busy":"2023-12-30T08:47:28.274227Z","iopub.execute_input":"2023-12-30T08:47:28.274968Z","iopub.status.idle":"2023-12-30T08:47:28.281427Z","shell.execute_reply.started":"2023-12-30T08:47:28.274918Z","shell.execute_reply":"2023-12-30T08:47:28.279969Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"T5 Tokenizer\nT5 tokenizer is the tokenizer used in the T5 model, which converts the raw text input into a sequence of tokens that can be processed by the T5 model. The T5 tokenizer is based on the Byte-Pair Encoding (BPE) algorithm, which is a popular technique for subword tokenization. It also includes a special \"extra_ids\" token to handle long inputs and outputs.\n\nThe T5 tokenizer is designed to handle a wide range of natural language processing tasks, and it can handle inputs and outputs of different lengths and types. Additionally, the T5 tokenizer can generate task-specific tokens to help the model understand the type of task it is performing. This makes the T5 tokenizer very flexible and powerful, and it has been used in many state-of-the-art natural language processing models.","metadata":{}},{"cell_type":"code","source":" !apt install git-lfs","metadata":{"id":"Oirz3VMbf3MT","execution":{"iopub.status.busy":"2023-12-30T08:47:28.285195Z","iopub.execute_input":"2023-12-30T08:47:28.285681Z","iopub.status.idle":"2023-12-30T08:47:32.152710Z","shell.execute_reply.started":"2023-12-30T08:47:28.285633Z","shell.execute_reply":"2023-12-30T08:47:32.151441Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Reading package lists... Done\nBuilding dependency tree       \nReading state information... Done\ngit-lfs is already the newest version (2.9.2-1).\n0 upgraded, 0 newly installed, 0 to remove and 71 not upgraded.\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Make sure your version of Transformers is at least 4.11.0 since the functionality was introduced in that version:","metadata":{"id":"abMo6fDOf3MU"}},{"cell_type":"code","source":"import transformers\n\nprint(transformers.__version__)","metadata":{"id":"QwMWrdtyf3MV","execution":{"iopub.status.busy":"2023-12-30T10:30:27.638583Z","iopub.execute_input":"2023-12-30T10:30:27.639380Z","iopub.status.idle":"2023-12-30T10:30:32.029386Z","shell.execute_reply.started":"2023-12-30T10:30:27.639347Z","shell.execute_reply":"2023-12-30T10:30:32.028351Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"4.36.0\n","output_type":"stream"}]},{"cell_type":"markdown","source":"You can find a script version of this notebook to fine-tune your model in a distributed fashion using multiple GPUs or TPUs [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq).","metadata":{"id":"HFASsisvIrIb"}},{"cell_type":"markdown","source":"We also quickly upload some telemetry - this tells us which examples and software versions are getting used so we know where to prioritize our maintenance efforts. We don't collect (or care about) any personally identifiable information, but if you'd prefer not to be counted, feel free to skip this step or delete this cell entirely.","metadata":{"id":"ZaKku_quf3MX"}},{"cell_type":"code","source":"from transformers.utils import send_example_telemetry\n\nsend_example_telemetry(\"summarization_notebook\", framework=\"pytorch\")","metadata":{"id":"a-ZBP9Jtf3MY","execution":{"iopub.status.busy":"2023-12-30T08:47:32.162584Z","iopub.execute_input":"2023-12-30T08:47:32.162924Z","iopub.status.idle":"2023-12-30T08:47:32.190013Z","shell.execute_reply.started":"2023-12-30T08:47:32.162880Z","shell.execute_reply":"2023-12-30T08:47:32.188785Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tuning a model on a summarization task","metadata":{"id":"rEJBSTyZIrIb"}},{"cell_type":"markdown","source":"In this notebook, we will see how to fine-tune one of the [🤗 Transformers](https://github.com/huggingface/transformers) model for a summarization task. We will use the [XSum dataset](https://arxiv.org/pdf/1808.08745.pdf) (for extreme summarization) which contains BBC articles accompanied with single-sentence summaries.\n\n![Widget inference on a summarization task](https://github.com/huggingface/notebooks/blob/main/examples/images/summarization.png?raw=1)\n\nWe will see how to easily load the dataset for this task using 🤗 Datasets and how to fine-tune a model on it using the `Trainer` API.","metadata":{"id":"kTCFado4IrIc"}},{"cell_type":"code","source":"model_checkpoint = \"mrm8488/t5-base-finetuned-summarize-news\"","metadata":{"id":"hdfSIzwmf3Mb","execution":{"iopub.status.busy":"2023-12-30T10:30:35.140285Z","iopub.execute_input":"2023-12-30T10:30:35.140807Z","iopub.status.idle":"2023-12-30T10:30:35.145293Z","shell.execute_reply.started":"2023-12-30T10:30:35.140772Z","shell.execute_reply":"2023-12-30T10:30:35.144225Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"This notebook is built to run  with any model checkpoint from the [Model Hub](https://huggingface.co/models) as long as that model has a sequence-to-sequence version in the Transformers library. Here we picked the [`t5-small`](https://huggingface.co/t5-small) checkpoint.","metadata":{"id":"4RRkXuteIrIh"}},{"cell_type":"markdown","source":"## Loading the dataset","metadata":{"id":"whPRbBNbIrIl"}},{"cell_type":"markdown","source":"We will use the [🤗 Datasets](https://github.com/huggingface/datasets) library to download the data and get the metric we need to use for evaluation (to compare our model to the benchmark). This can be easily done with the functions `load_dataset` and `load_metric`.  ","metadata":{"id":"W7QYTpxXIrIl"}},{"cell_type":"code","source":"from evaluate import load\nimport pandas as pd\nraw_datasets = pd.read_csv(\"/kaggle/input/gktgtggpt/Aljazeera_dataset_eng.csv\")\nmetric = load(\"rouge\")","metadata":{"id":"IreSlFmlIrIm","execution":{"iopub.status.busy":"2023-12-30T10:30:40.195419Z","iopub.execute_input":"2023-12-30T10:30:40.195813Z","iopub.status.idle":"2023-12-30T10:30:55.646912Z","shell.execute_reply.started":"2023-12-30T10:30:40.195772Z","shell.execute_reply":"2023-12-30T10:30:55.645898Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/6.27k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0f528ce7f36744e086e78808d02d8a57"}},"metadata":{}}]},{"cell_type":"markdown","source":"The `dataset` object itself is [`DatasetDict`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasetdict), which contains one key for the training, validation and test set:","metadata":{"id":"RzfPtOMoIrIu"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom datasets import Dataset, DatasetDict\n\n# Load the dataset from the CSV file\nraw_dataset = pd.read_csv(\"/kaggle/input/gktgtggpt/Aljazeera_dataset_eng.csv\")\n\n# Split the dataset into train, validation, and test datasets\ntrain_dataset, remaining_dataset = train_test_split(raw_dataset, test_size=0.2, random_state=42)\nvalidation_dataset, test_dataset = train_test_split(remaining_dataset, test_size=0.5, random_state=42)\n\n# Define the feature names\nfeature_names = ['id', 'url', 'title', 'category', 'summary', 'texte']\n\n# Convert the datasets to dictionaries with feature names as keys\ntrain_dict = {feature: train_dataset[feature] for feature in feature_names}\nvalidation_dict = {feature: validation_dataset[feature] for feature in feature_names}\ntest_dict = {feature: test_dataset[feature] for feature in feature_names}\n\n# Create the DatasetDict object\ndataset_dict = DatasetDict({\n    \"train\": Dataset.from_dict(train_dict),\n    \"validation\": Dataset.from_dict(validation_dict),\n    \"test\": Dataset.from_dict(test_dict)\n})\n\nprint(dataset_dict)","metadata":{"id":"Fn51qUk5il4-","execution":{"iopub.status.busy":"2023-12-30T10:31:03.885847Z","iopub.execute_input":"2023-12-30T10:31:03.886696Z","iopub.status.idle":"2023-12-30T10:31:03.945240Z","shell.execute_reply.started":"2023-12-30T10:31:03.886660Z","shell.execute_reply":"2023-12-30T10:31:03.943959Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"DatasetDict({\n    train: Dataset({\n        features: ['id', 'url', 'title', 'category', 'summary', 'texte'],\n        num_rows: 204\n    })\n    validation: Dataset({\n        features: ['id', 'url', 'title', 'category', 'summary', 'texte'],\n        num_rows: 26\n    })\n    test: Dataset({\n        features: ['id', 'url', 'title', 'category', 'summary', 'texte'],\n        num_rows: 26\n    })\n})\n","output_type":"stream"}]},{"cell_type":"code","source":"raw_datasets=dataset_dict\n","metadata":{"id":"zKwhXInVlpOH","execution":{"iopub.status.busy":"2023-12-30T10:31:07.701914Z","iopub.execute_input":"2023-12-30T10:31:07.702664Z","iopub.status.idle":"2023-12-30T10:31:07.706971Z","shell.execute_reply.started":"2023-12-30T10:31:07.702635Z","shell.execute_reply":"2023-12-30T10:31:07.705844Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"To access an actual element, you need to select a split first, then give an index:","metadata":{"id":"u3EtYfeHIrIz"}},{"cell_type":"code","source":"raw_datasets[\"train\"][5]","metadata":{"id":"X6HrpprwIrIz","outputId":"b5ddf433-fa30-4009-89c3-b3793168461d","execution":{"iopub.status.busy":"2023-12-30T08:47:50.880607Z","iopub.execute_input":"2023-12-30T08:47:50.881081Z","iopub.status.idle":"2023-12-30T08:47:50.901807Z","shell.execute_reply.started":"2023-12-30T08:47:50.881044Z","shell.execute_reply":"2023-12-30T08:47:50.900502Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"{'id': 140,\n 'url': 'https://www.aljazeera.com//features/2023/12/28/piles-of-body-parts-gazas-maghazi-residents-find-families-in-pieces',\n 'title': '‘Piles of body parts’: Gaza’s Mag\\xadhazi res\\xadi\\xaddents find fam\\xadi\\xadlies ‘in pieces’',\n 'category': 'features',\n 'summary': 'Af\\xadter dev\\xadas\\xadtat\\xading Is\\xadraeli at\\xadtack on refugee camp, rel\\xada\\xadtives re\\xadturn to dig up many vic\\xadtims who are un\\xadder the rub\\xadble.',\n 'texte': 'After devastating Israeli attack on refugee camp, relatives return to dig up many victims who are under the rubble. Deir el-Balah, Gaza Strip – It has been four days since Gaza’s smallest refugee camp was pounded in yet another series of Israeli air strikes, but Palestinians there are still digging up the bodies of their loved ones from under the rubble. The onslaught in central Gaza’s Maghazi late on Sunday killed at least 90 people, including children and many who were internally displaced. In one of the deadliest attacks on the Gaza Strip since Israel launched a war on the enclave on October 7, residents including Ashraf al-Haj Ahmed said the assault happened “suddenly” and without prior warning. “At around 11:30pm that night, we witnessed a series of large explosions that shook the entire camp,” al-Haj Ahmed told Al Jazeera. His relative’s home was among those that were flattened to the ground. Al-Haj Ahmed recalled running towards it as soon as the bombardment woke him up, just a few blocks down. At the scene of the attack, he found a four-storey building destroyed “on top of those who were living in it”. “There must have been around 40 people, among them are the owners of the house, as well as displaced families who were taken in,” he said. At least three houses in the overcrowded camp were hit by Israeli air strikes. Officials in Gaza said seven families were among the casualties. While the official number of those who were killed stands at 90, residents of the camp near Deir el-Balah say in reality, the figure is much higher as entire residential blocks were wiped out. “In each home, there’s a minimum of 50 people,” Ahmed Maghari, another Maghazi resident, told Al Jazeera. “A lot of them are displaced Palestinians from other parts of Gaza who were forced to flee their homes.” The camp normally houses 30,000 people, according to the UN refugee agency for Palestinians (UNRWA). But with the displacement of Palestinians fleeing Israel’s relentless bombardment in other parts of the enclave, the number of people there has risen to an estimated 100,000. “We pulled out so many body parts that we can’t even estimate the total number of deaths yet,” Maghari said. “They’re all in pieces, and we’re pulling them out with our bare hands,” he added. “We’ve now gathered at least two piles of body parts.” Israel’s attacks have not spared homes and shelters that people have fled to. Despite being on the southern side of the Strip, an area that Israeli forces deemed “safe” and ordered civilians from the north to flee ahead of their ground offensive, Maghazi has been subjected to intense artillery and air raids. It was also attacked last month\\xa0when at least 50 Palestinians were killed. The vicinity of the camp was also subjected to intense Israeli shelling over the last week. Abu Rami Abu al-Ais is among those who have been sheltering in Maghazi ever since he left his home in the al-Zahra neighbourhood. He said Sunday’s attack was not the first time he and his family members had been hit. “We had a home in al-Zahra, which came under attack. After coming here, the house we were staying in was bombed again,” al-Ais, whose daughter is badly injured, told Al Jazeera. He echoed al-Haj Ahmed’s experience and said there had been “no warnings whatsoever” prior to the strikes. Al-Ais said in previous assaults on the enclave, Israeli forces would sometimes warn residents of a building to evacuate a few minutes before an attack, either by throwing leaflets or via speakerphones. But during this offensive, there had been no such warnings. “The rockets fall on the heads of innocent people sleeping in their homes,” he said. “They [Israel] want to commit a complete genocide.” Al-Ais said people are still collecting the remains of their friends, neighbours and relatives with their bare hands. “We found the remains of women and children who were blown up. Their body parts have been scattered over a span of about three blocks,” due to the intensity of the strikes, al-Ais said. “It was a very dark and painful night for Maghazi,” he recalled. “The widespread and sheer destruction is indescribable.” Infrastructure, such as roads leading to the camp, were also destroyed. Al-Ais said there are no excavators that can help speed up the process of recovering people from under the blocks of concrete. The lack of much-needed fuel to operate bulldozers and vehicles means that – just like civil defence teams in Gaza – residents are digging with only their bare hands to try and pull out as many victims from under the rubble as they can. Israel has blocked the entry of fuel\\xa0since it imposed a total siege on the already blockaded Strip at the start of the war, and has only allowed a very small amount of aid in through the Rafah border crossing. “We don’t need food, we don’t need water, we don’t need coffins,” al-Ais said. “What we need is a ceasefire and for this war to end.” Al-Haj Ahmed, agreed. “Shame on the Arab world. We don’t just need aid, we need you here personally. Come and stand with your brothers,” he said. Attacks on refugee camps and civilian infrastructure have become common since October 7. The Jabalia refugee camp in northern Gaza has been targeted several times, resulting in the deaths of hundreds of Palestinians. Civilian infrastructure – including schools, hospitals, ambulance vehicles, and places of worship – has also been subjected to bombardment. More than 21,000 Palestinians have been killed since October 7, while nearly 1.9 million – more than 80 percent of the 2.3 million people who live in Gaza – have been displaced. Additional reporting by Ashraf Abu Amra.          Follow Al Jazeera English:'}"},"metadata":{}}]},{"cell_type":"markdown","source":"To get a sense of what the data looks like, the following function will show some examples picked randomly in the dataset.","metadata":{"id":"WHUmphG3IrI3"}},{"cell_type":"code","source":"import datasets\nimport random\nimport pandas as pd\nfrom IPython.display import display, HTML\n\ndef show_random_elements(dataset, num_examples=5):\n    assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n    picks = []\n    for _ in range(num_examples):\n        pick = random.randint(0, len(dataset)-1)\n        while pick in picks:\n            pick = random.randint(0, len(dataset)-1)\n        picks.append(pick)\n\n    df = pd.DataFrame(dataset[picks])\n    for column, typ in dataset.features.items():\n        if isinstance(typ, datasets.ClassLabel):\n            df[column] = df[column].transform(lambda i: typ.names[i])\n    display(HTML(df.to_html()))","metadata":{"id":"i3j8APAoIrI3","execution":{"iopub.status.busy":"2023-12-30T08:47:50.903628Z","iopub.execute_input":"2023-12-30T08:47:50.904015Z","iopub.status.idle":"2023-12-30T08:47:50.913442Z","shell.execute_reply.started":"2023-12-30T08:47:50.903980Z","shell.execute_reply":"2023-12-30T08:47:50.912241Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"markdown","source":"The metric is an instance of [`datasets.Metric`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Metric):","metadata":{"id":"lnjDIuQ3IrI-"}},{"cell_type":"code","source":"metric","metadata":{"id":"5o4rUteaIrI_","outputId":"128b073e-e954-401b-8c3d-c839e89a0237","execution":{"iopub.status.busy":"2023-12-30T10:31:14.880098Z","iopub.execute_input":"2023-12-30T10:31:14.881004Z","iopub.status.idle":"2023-12-30T10:31:14.895302Z","shell.execute_reply.started":"2023-12-30T10:31:14.880968Z","shell.execute_reply":"2023-12-30T10:31:14.894046Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"EvaluationModule(name: \"rouge\", module_type: \"metric\", features: [{'predictions': Value(dtype='string', id='sequence'), 'references': Sequence(feature=Value(dtype='string', id='sequence'), length=-1, id=None)}, {'predictions': Value(dtype='string', id='sequence'), 'references': Value(dtype='string', id='sequence')}], usage: \"\"\"\nCalculates average rouge scores for a list of hypotheses and references\nArgs:\n    predictions: list of predictions to score. Each prediction\n        should be a string with tokens separated by spaces.\n    references: list of reference for each prediction. Each\n        reference should be a string with tokens separated by spaces.\n    rouge_types: A list of rouge types to calculate.\n        Valid names:\n        `\"rouge{n}\"` (e.g. `\"rouge1\"`, `\"rouge2\"`) where: {n} is the n-gram based scoring,\n        `\"rougeL\"`: Longest common subsequence based scoring.\n        `\"rougeLsum\"`: rougeLsum splits text using `\"\n\"`.\n        See details in https://github.com/huggingface/datasets/issues/617\n    use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes.\n    use_aggregator: Return aggregates if this is set to True\nReturns:\n    rouge1: rouge_1 (f1),\n    rouge2: rouge_2 (f1),\n    rougeL: rouge_l (f1),\n    rougeLsum: rouge_lsum (f1)\nExamples:\n\n    >>> rouge = evaluate.load('rouge')\n    >>> predictions = [\"hello there\", \"general kenobi\"]\n    >>> references = [\"hello there\", \"general kenobi\"]\n    >>> results = rouge.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}\n\"\"\", stored examples: 0)"},"metadata":{}}]},{"cell_type":"markdown","source":"You can call its `compute` method with your predictions and labels, which need to be list of decoded strings:","metadata":{"id":"jAWdqcUBIrJC"}},{"cell_type":"code","source":"fake_preds = [\"hello there\", \"general kenobi\"]\nfake_labels = [\"hello there\", \"general kenobi\"]\nmetric.compute(predictions=fake_preds, references=fake_labels)","metadata":{"id":"6XN1Rq0aIrJC","outputId":"0705ee4c-9f3b-4373-f5a7-8549051bcfd6","execution":{"iopub.status.busy":"2023-12-30T10:31:18.429427Z","iopub.execute_input":"2023-12-30T10:31:18.430178Z","iopub.status.idle":"2023-12-30T10:31:18.719052Z","shell.execute_reply.started":"2023-12-30T10:31:18.430143Z","shell.execute_reply":"2023-12-30T10:31:18.718078Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}"},"metadata":{}}]},{"cell_type":"markdown","source":"## Preprocessing the data","metadata":{"id":"n9qywopnIrJH"}},{"cell_type":"markdown","source":"Before we can feed those texts to our model, we need to preprocess them. This is done by a 🤗 Transformers `Tokenizer` which will (as the name indicates) tokenize the inputs (including converting the tokens to their corresponding IDs in the pretrained vocabulary) and put it in a format the model expects, as well as generate the other inputs that the model requires.\n\nTo do all of this, we instantiate our tokenizer with the `AutoTokenizer.from_pretrained` method, which will ensure:\n\n- we get a tokenizer that corresponds to the model architecture we want to use,\n- we download the vocabulary used when pretraining this specific checkpoint.\n\nThat vocabulary will be cached, so it's not downloaded again the next time we run the cell.","metadata":{"id":"YVx71GdAIrJH"}},{"cell_type":"code","source":"\nfrom transformers import T5Tokenizer, T5ForConditionalGeneration  \nMODEL_NAME = \"mrm8488/t5-base-finetuned-summarize-news\"\ntokenizer = T5Tokenizer.from_pretrained(MODEL_NAME)\n","metadata":{"id":"eXNLu_-nIrJI","outputId":"173833c0-2393-453f-f339-56561355b461","execution":{"iopub.status.busy":"2023-12-30T10:32:06.264043Z","iopub.execute_input":"2023-12-30T10:32:06.264923Z","iopub.status.idle":"2023-12-30T10:32:09.550925Z","shell.execute_reply.started":"2023-12-30T10:32:06.264888Z","shell.execute_reply":"2023-12-30T10:32:09.549983Z"},"trusted":true},"execution_count":11,"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5ea4fa520e444303926e573619a85961"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"220c391c97bc40e7800d84e4d60bdcb6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/1.79k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"84d30fce23bd410db738dcafec11289f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"33ab1a2c8f4d4abd904a5613b97965f8"}},"metadata":{}},{"name":"stderr","text":"You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n","output_type":"stream"}]},{"cell_type":"markdown","source":"By default, the call above will use one of the fast tokenizers (backed by Rust) from the 🤗 Tokenizers library.","metadata":{"id":"Vl6IidfdIrJK"}},{"cell_type":"markdown","source":"You can directly call this tokenizer on one sentence or a pair of sentences:","metadata":{"id":"rowT4iCLIrJK"}},{"cell_type":"code","source":"tokenizer(\"Hello, this one sentence!\")","metadata":{"id":"a5hBlsrHIrJL","outputId":"927ec37c-b3eb-45ac-e5cb-bf9eb14f850b","execution":{"iopub.status.busy":"2023-12-30T10:32:13.480875Z","iopub.execute_input":"2023-12-30T10:32:13.481867Z","iopub.status.idle":"2023-12-30T10:32:13.488255Z","shell.execute_reply.started":"2023-12-30T10:32:13.481833Z","shell.execute_reply":"2023-12-30T10:32:13.487360Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"{'input_ids': [8774, 6, 48, 80, 7142, 55, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1]}"},"metadata":{}}]},{"cell_type":"markdown","source":"Depending on the model you selected, you will see different keys in the dictionary returned by the cell above. They don't matter much for what we're doing here (just know they are required by the model we will instantiate later), you can learn more about them in [this tutorial](https://huggingface.co/transformers/preprocessing.html) if you're interested.\n\nInstead of one sentence, we can pass along a list of sentences:","metadata":{"id":"qo_0B1M2IrJM"}},{"cell_type":"code","source":"tokenizer([\"Hello, this one sentence!\", \"This is another sentence.\"])","metadata":{"id":"NBKcnUQwf3Mo","outputId":"f0992d93-5767-406e-cd02-860920065d0e","execution":{"iopub.status.busy":"2023-12-30T10:32:16.875974Z","iopub.execute_input":"2023-12-30T10:32:16.876367Z","iopub.status.idle":"2023-12-30T10:32:16.883178Z","shell.execute_reply.started":"2023-12-30T10:32:16.876338Z","shell.execute_reply":"2023-12-30T10:32:16.882297Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"{'input_ids': [[8774, 6, 48, 80, 7142, 55, 1], [100, 19, 430, 7142, 5, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]]}"},"metadata":{}}]},{"cell_type":"markdown","source":"To prepare the targets for our model, we need to tokenize them using the `text_target` parameter. This will make sure the tokenizer uses the special tokens corresponding to the targets:","metadata":{"id":"AxPs6xfAf3Mp"}},{"cell_type":"code","source":"print(tokenizer(text_target=[\"Hello, this one sentence!\", \"This is another sentence.\"]))","metadata":{"id":"FUiEhxjFf3Mp","execution":{"iopub.status.busy":"2023-12-30T10:32:20.539168Z","iopub.execute_input":"2023-12-30T10:32:20.539794Z","iopub.status.idle":"2023-12-30T10:32:20.545353Z","shell.execute_reply.started":"2023-12-30T10:32:20.539762Z","shell.execute_reply":"2023-12-30T10:32:20.544420Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"{'input_ids': [[8774, 6, 48, 80, 7142, 55, 1], [100, 19, 430, 7142, 5, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]]}\n","output_type":"stream"}]},{"cell_type":"markdown","source":"If you are using one of the five T5 checkpoints we have to prefix the inputs with \"summarize:\" (the model can also translate and it needs the prefix to know which task it has to perform).","metadata":{"id":"2C0hcmp9IrJQ"}},{"cell_type":"code","source":"if model_checkpoint in [\"t5-small\", \"t5-base\", \"t5-larg\", \"t5-3b\", \"t5-11b\",\"mrm8488/t5-base-finetuned-summarize-news\"]:\n    prefix = \"summarize: \"\nelse:\n    prefix = \"\"","metadata":{"id":"uhahaA6if3Mz","execution":{"iopub.status.busy":"2023-12-30T10:32:23.630132Z","iopub.execute_input":"2023-12-30T10:32:23.630508Z","iopub.status.idle":"2023-12-30T10:32:23.635163Z","shell.execute_reply.started":"2023-12-30T10:32:23.630480Z","shell.execute_reply":"2023-12-30T10:32:23.634233Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":"We can then write the function that will preprocess our samples. We just feed them to the `tokenizer` with the argument `truncation=True`. This will ensure that an input longer that what the model selected can handle will be truncated to the maximum length accepted by the model. The padding will be dealt with later on (in a data collator) so we pad examples to the longest length in the batch and not the whole dataset.","metadata":{"id":"RZkxHHRQf3M0"}},{"cell_type":"code","source":"max_input_length = 700\nmax_target_length = 200\n\ndef preprocess_function(examples):\n    inputs = [prefix + doc for doc in examples[\"texte\"]]\n    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)\n\n    # Setup the tokenizer for targets\n    labels = tokenizer(text_target=examples[\"summary\"], max_length=max_target_length, truncation=True)\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs","metadata":{"id":"vc0BSBLIIrJQ","execution":{"iopub.status.busy":"2023-12-30T10:32:42.571420Z","iopub.execute_input":"2023-12-30T10:32:42.572284Z","iopub.status.idle":"2023-12-30T10:32:42.577691Z","shell.execute_reply.started":"2023-12-30T10:32:42.572250Z","shell.execute_reply":"2023-12-30T10:32:42.576772Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"markdown","source":"This function works with one or several examples. In the case of several examples, the tokenizer will return a list of lists for each key:","metadata":{"id":"0lm8ozrJIrJR"}},{"cell_type":"code","source":"preprocess_function(raw_datasets['train'][:2])","metadata":{"id":"-b70jh26IrJS","outputId":"e08c676a-c915-42dd-dd55-7afdcbec4276","execution":{"iopub.status.busy":"2023-12-30T10:32:46.099722Z","iopub.execute_input":"2023-12-30T10:32:46.100339Z","iopub.status.idle":"2023-12-30T10:32:46.119764Z","shell.execute_reply.started":"2023-12-30T10:32:46.100307Z","shell.execute_reply":"2023-12-30T10:32:46.118813Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"{'input_ids': [[21603, 10, 3, 4868, 14047, 8260, 1647, 7, 360, 4071, 7, 808, 7892, 726, 8620, 11, 228, 700, 129, 3033, 7, 16, 8, 215, 2177, 5, 7291, 77, 5306, 348, 13, 8018, 6, 2514, 1279, 44, 71, 8834, 21, 72, 145, 192, 4160, 5, 3, 15944, 48, 215, 6, 255, 1513, 160, 613, 5, 451, 47, 31521, 5, 299, 8, 913, 47, 30, 8, 1481, 21, 767, 38, 8, 349, 1971, 12, 1340, 1358, 5, 105, 4168, 1052, 149, 231, 161, 62, 141, 6, 34, 47, 464, 326, 13, 3, 5304, 12432, 642, 5306, 348, 1219, 901, 2215, 776, 1498, 1566, 5, 13294, 336, 215, 6, 8, 958, 6079, 2162, 34, 133, 8260, 326, 3986, 13, 1652, 6, 379, 5306, 348, 5, 105, 10555, 25, 33, 5099, 12, 8, 1476, 6, 8, 705, 819, 13362, 25, 54, 504, 24, 1416, 394, 21, 135, 642, 5306, 348, 243, 5, 86, 460, 2884, 6, 71, 8834, 22, 7, 2737, 1027, 18581, 32, 47, 8, 2030, 18, 12760, 4071, 16, 8, 1297, 785, 11, 6995, 17, 63, 958, 2393, 5, 216, 263, 72, 145, 28153, 51, 336, 215, 3, 318, 167, 13, 84, 764, 45, 1519, 11133, 5, 2150, 12, 331, 3, 16678, 57, 585, 1669, 21263, 40, 291, 6, 24, 656, 1027, 18581, 32, 8, 1025, 18, 6739, 222, 18, 12760, 4071, 57, 3751, 16, 66, 13, 2849, 1371, 5, 71, 8834, 65, 59, 263, 136, 452, 3148, 12, 165, 420, 7930, 15, 7, 838, 136, 773, 13, 726, 1340, 18905, 8, 8260, 1647, 7, 5, 71, 8834, 410, 59, 3531, 12, 901, 2215, 776, 1498, 22, 7, 1690, 21, 1670, 5, 71, 8834, 65, 780, 12, 962, 165, 19784, 2493, 3, 318, 8, 1708, 3, 9, 349, 10375, 38, 34, 8081, 7, 26715, 11839, 16, 3245, 13, 46, 2041, 26715, 1338, 11, 84, 963, 4297, 6107, 21, 8, 215, 5, 71, 8834, 22, 7, 167, 1100, 47, 16, 1332, 5, 2792, 167, 11652, 18107, 688, 6, 34, 56, 962, 8, 416, 80, 16, 8, 166, 985, 13, 460, 2266, 5, 2502, 688, 410, 3531, 12, 901, 2215, 776, 1498, 22, 7, 20182, 68, 243, 4297, 6107, 65, 780, 12, 36, 19972, 5, 332, 18, 22162, 21, 677, 243, 34, 56, 1576, 8, 806, 7, 13, 4071, 4794, 292, 3027, 22, 7, 6107, 2642, 16, 1186, 38, 410, 276, 89, 8585, 6, 11147, 19089, 6, 5222, 6, 1263, 26306, 11, 186, 717, 5, 818, 2849, 1371, 47, 18642, 81, 8260, 1647, 7, 6, 8, 16698, 30, 4297, 6107, 19, 12, 36, 4187, 6, 3, 15716, 79, 228, 3098, 5, 466, 1299, 7, 3, 9, 4838, 1569, 81, 8, 538, 13, 8, 2717, 6904, 139, 8, 460, 2266, 13074, 4356, 5, 86, 1762, 6, 27141, 7245, 326, 3241, 335, 1093, 13, 165, 10312, 5, 4071, 10290, 2798, 23, 1647, 9100, 26, 1456, 1124, 5, 37, 8260, 1647, 7, 764, 3, 3565, 1368, 613, 1170, 16, 8, 907, 1323, 5, 86, 460, 2884, 6, 8, 837, 2717, 974, 3, 12451, 770, 2476, 3, 318, 8, 511, 18, 6739, 222, 16, 1283, 203, 5, 1881, 10562, 26, 53, 48, 847, 6, 331, 21, 84, 19, 59, 780, 347, 6, 8, 2717, 974, 72, 145, 10603, 770, 2476, 16, 460, 2773, 5, 86, 1797, 6, 37, 1945, 1384, 2633, 26, 12485, 3, 7, 22411, 57, 1640, 1093, 437, 165, 6734, 16, 1515, 460, 2884, 5, 1193, 14907, 120, 6, 27141, 2196, 13155, 5, 357, 115, 29, 16, 3751, 16, 8, 166, 2893, 5, 105, 4770, 667, 7, 557, 129, 3, 22303, 21, 8260, 1647, 7, 250, 3556, 1887, 217, 7, 34, 38, 3, 9, 1320, 24, 8, 5752, 4297, 19, 838, 8, 3429, 1041, 12, 766, 24, 8, 349, 19, 1243, 11, 90, 152, 78, 557, 79, 56, 217, 3, 9, 13033, 16, 70, 698, 594, 227, 3, 9, 8260, 1647, 642, 8077, 11825, 6, 2090, 13, 8, 3699, 22077, 2786, 44, 8, 2548, 242, 7587, 6536, 6, 1219, 901, 2215, 776, 1498, 5, 105, 1570, 24, 773, 13, 2625, 6, 132, 19, 150, 194, 24, 17780, 33, 352, 12, 36, 3874, 21, 8, 4071, 22, 7, 819, 42, 237, 3, 5503, 70, 17489, 250, 17780, 33, 1095, 116, 8, 1519, 594, 1550, 95, 642, 1], [21603, 10, 389, 8521, 1207, 4843, 13, 4263, 16953, 7, 11, 12893, 7, 3, 23830, 640, 11897, 6, 9357, 44, 709, 604, 151, 11, 16, 10609, 53, 186, 72, 5, 13686, 7, 243, 8, 716, 18, 2961, 26877, 297, 1381, 3, 20889, 16953, 7, 11, 3, 14903, 13, 12893, 7, 5, 11897, 22, 7, 1761, 5205, 17604, 718, 34, 458, 532, 167, 3805, 22142, 3211, 22, 437, 4623, 166, 24281, 26, 11897, 16, 2083, 460, 2884, 5, 7847, 901, 2215, 776, 1498, 1566, 10, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'labels': [[374, 575, 7, 5230, 15, 1148, 575, 7, 102, 5236, 8260, 575, 1647, 7, 360, 4071, 7, 808, 1243, 575, 53, 575, 1329, 726, 8620, 11, 228, 3, 9, 75, 575, 17, 76, 575, 138, 575, 120, 129, 3, 8316, 575, 15, 7, 16, 8, 215, 2177, 5, 1], [2770, 7, 575, 7, 23, 9, 3289, 575, 15, 7, 80, 13, 165, 2015, 8348, 13, 1817, 575, 7, 699, 11, 12893, 17613, 640, 11897, 437, 8, 456, 13, 615, 1]]}"},"metadata":{}}]},{"cell_type":"markdown","source":"To apply this function on all the pairs of sentences in our dataset, we just use the `map` method of our `dataset` object we created earlier. This will apply the function on all the elements of all the splits in `dataset`, so our training, validation and testing data will be preprocessed in one single command.","metadata":{"id":"zS-6iXTkIrJT"}},{"cell_type":"code","source":"tokenized_datasets = raw_datasets.map(preprocess_function, batched=True)","metadata":{"id":"DDtsaJeVIrJT","outputId":"ce78ade2-f3b6-4b47-e65e-ca90b58be9e7","execution":{"iopub.status.busy":"2023-12-30T10:32:50.578580Z","iopub.execute_input":"2023-12-30T10:32:50.579205Z","iopub.status.idle":"2023-12-30T10:32:52.278267Z","shell.execute_reply.started":"2023-12-30T10:32:50.579173Z","shell.execute_reply":"2023-12-30T10:32:52.277382Z"},"trusted":true},"execution_count":19,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ad0ad3306b714f22adec06a857f1b93f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"07c631c1c61a49329134e0e579e66525"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c80b39da76ef4949a96c2f306b928609"}},"metadata":{}}]},{"cell_type":"markdown","source":"Even better, the results are automatically cached by the 🤗 Datasets library to avoid spending time on this step the next time you run your notebook. The 🤗 Datasets library is normally smart enough to detect when the function you pass to map has changed (and thus requires to not use the cache data). For instance, it will properly detect if you change the task in the first cell and rerun the notebook. 🤗 Datasets warns you when it uses cached files, you can pass `load_from_cache_file=False` in the call to `map` to not use the cached files and force the preprocessing to be applied again.\n\nNote that we passed `batched=True` to encode the texts by batches together. This is to leverage the full benefit of the fast tokenizer we loaded earlier, which will use multi-threading to treat the texts in a batch concurrently.","metadata":{"id":"voWiw8C7IrJV"}},{"cell_type":"markdown","source":"## Fine-tuning the model","metadata":{"id":"545PP3o8IrJV"}},{"cell_type":"markdown","source":"Now that our data is ready, we can download the pretrained model and fine-tune it. Since our task is of the sequence-to-sequence kind, we use the `AutoModelForSeq2SeqLM` class. Like with the tokenizer, the `from_pretrained` method will download and cache the model for us.","metadata":{"id":"FBiW8UpKIrJW"}},{"cell_type":"code","source":"from transformers import AutoModelWithLMHead, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n\nmodel = AutoModelWithLMHead.from_pretrained(model_checkpoint)","metadata":{"id":"TlqNaB8jIrJW","outputId":"07c2b7ec-2daf-4a20-865c-6696e672cc7d","execution":{"iopub.status.busy":"2023-12-30T10:32:59.276804Z","iopub.execute_input":"2023-12-30T10:32:59.277185Z","iopub.status.idle":"2023-12-30T10:34:00.994757Z","shell.execute_reply.started":"2023-12-30T10:32:59.277156Z","shell.execute_reply":"2023-12-30T10:34:00.993878Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/models/auto/modeling_auto.py:1564: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n  warnings.warn(\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/892M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fe637f441ade47da90bfe6fb48d0c496"}},"metadata":{}}]},{"cell_type":"markdown","source":"Note that  we don't get a warning like in our classification example. This means we used all the weights of the pretrained model and there is no randomly initialized head in this case.","metadata":{"id":"CczA5lJlIrJX"}},{"cell_type":"code","source":"model.config.decoder_start_token_id = tokenizer.pad_token_id\nmodel.config.eos_token_id = tokenizer.eos_token_id","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:36:02.834807Z","iopub.execute_input":"2023-12-30T10:36:02.835501Z","iopub.status.idle":"2023-12-30T10:36:02.840067Z","shell.execute_reply.started":"2023-12-30T10:36:02.835466Z","shell.execute_reply":"2023-12-30T10:36:02.839026Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"markdown","source":"To instantiate a `Seq2SeqTrainer`, we will need to define three more things. The most important is the [`Seq2SeqTrainingArguments`](https://huggingface.co/transformers/main_classes/trainer.html#transformers.Seq2SeqTrainingArguments), which is a class that contains all the attributes to customize the training. It requires one folder name, which will be used to save the checkpoints of the model, and all other arguments are optional:","metadata":{"id":"_N8urzhyIrJY"}},{"cell_type":"code","source":"batch_size = 8\nmodel_name = model_checkpoint.split(\"/\")[-1]\nargs = Seq2SeqTrainingArguments(\n    f\"{model_name}-finetuned-xsum\",\n    evaluation_strategy = \"epoch\",\n    learning_rate=2e-5,\n    per_device_train_batch_size=batch_size,\n    per_device_eval_batch_size=batch_size,\n    weight_decay=0.01,\n    save_total_limit=3,\n    num_train_epochs=8,\n    predict_with_generate=True,\n    \n)","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:37:01.812668Z","iopub.execute_input":"2023-12-30T10:37:01.813743Z","iopub.status.idle":"2023-12-30T10:37:01.822522Z","shell.execute_reply.started":"2023-12-30T10:37:01.813697Z","shell.execute_reply":"2023-12-30T10:37:01.821388Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"markdown","source":"Here we set the evaluation to be done at the end of each epoch, tweak the learning rate, use the `batch_size` defined at the top of the cell and customize the weight decay. Since the `Seq2SeqTrainer` will save the model regularly and our dataset is quite large, we tell it to make three saves maximum. Lastly, we use the `predict_with_generate` option (to properly generate summaries) and activate mixed precision training (to go a bit faster).\n\nThe last argument to setup everything so we can push the model to the [Hub](https://huggingface.co/models) regularly during training. Remove it if you didn't follow the installation steps at the top of the notebook. If you want to save your model locally in a name that is different than the name of the repository it will be pushed, or if you want to push your model under an organization and not your name space, use the `hub_model_id` argument to set the repo name (it needs to be the full name, including your namespace: for instance `\"sgugger/t5-finetuned-xsum\"` or `\"huggingface/t5-finetuned-xsum\"`).\n\nThen, we need a special kind of data collator, which will not only pad the inputs to the maximum length in the batch, but also the labels:","metadata":{"id":"km3pGVdTIrJc"}},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)","metadata":{"id":"wXKtGNqNf3M6","execution":{"iopub.status.busy":"2023-12-30T10:37:05.395970Z","iopub.execute_input":"2023-12-30T10:37:05.397073Z","iopub.status.idle":"2023-12-30T10:37:05.401041Z","shell.execute_reply.started":"2023-12-30T10:37:05.397037Z","shell.execute_reply":"2023-12-30T10:37:05.400164Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"markdown","source":"The last thing to define for our `Seq2SeqTrainer` is how to compute the metrics from the predictions. We need to define a function for this, which will just use the `metric` we loaded earlier, and we have to do a bit of pre-processing to decode the predictions into texts:","metadata":{"id":"7sZOdRlRIrJd"}},{"cell_type":"code","source":"import nltk\nimport numpy as np\n\ndef compute_metrics(eval_pred):\n    predictions, labels = eval_pred\n    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n    # Replace -100 in the labels as we can't decode them.\n    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n    # Rouge expects a newline after each sentence\n    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]\n    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]\n\n    # Note that other metrics may not have a `use_aggregator` parameter\n    # and thus will return a list, computing a metric for each sentence.\n    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True, use_aggregator=True)\n    # Extract a few results\n    result = {key: value * 100 for key, value in result.items()}\n\n    # Add mean generated length\n    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n    result[\"gen_len\"] = np.mean(prediction_lens)\n\n    return {k: round(v, 4) for k, v in result.items()}","metadata":{"id":"UmvbnJ9JIrJd","execution":{"iopub.status.busy":"2023-12-30T10:36:44.936862Z","iopub.execute_input":"2023-12-30T10:36:44.937582Z","iopub.status.idle":"2023-12-30T10:36:44.945778Z","shell.execute_reply.started":"2023-12-30T10:36:44.937545Z","shell.execute_reply":"2023-12-30T10:36:44.944788Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"markdown","source":"Then we just need to pass all of this along with our datasets to the `Seq2SeqTrainer`:","metadata":{"id":"rXuFTAzDIrJe"}},{"cell_type":"code","source":"from transformers import Seq2SeqTrainer\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"validation\"],\n    data_collator=data_collator,\n    tokenizer=tokenizer,\n    compute_metrics=compute_metrics\n)","metadata":{"id":"imY1oC3SIrJf","execution":{"iopub.status.busy":"2023-12-30T10:37:14.375115Z","iopub.execute_input":"2023-12-30T10:37:14.375970Z","iopub.status.idle":"2023-12-30T10:37:14.388819Z","shell.execute_reply.started":"2023-12-30T10:37:14.375935Z","shell.execute_reply":"2023-12-30T10:37:14.387945Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"markdown","source":"We can now finetune our model by just calling the `train` method:","metadata":{"id":"CdzABDVcIrJg"}},{"cell_type":"code","source":"trainer.train()","metadata":{"id":"uNx5pyRlIrJh","outputId":"077e661e-d36c-469b-89b8-7ff7f73541ec","execution":{"iopub.status.busy":"2023-12-30T10:37:17.824359Z","iopub.execute_input":"2023-12-30T10:37:17.824855Z","iopub.status.idle":"2023-12-30T10:41:36.341939Z","shell.execute_reply.started":"2023-12-30T10:37:17.824818Z","shell.execute_reply":"2023-12-30T10:41:36.340893Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:","output_type":"stream"},{"output_type":"stream","name":"stdin","text":"  ········································\n"},{"name":"stderr","text":"\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Tracking run with wandb version 0.16.1"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Run data is saved locally in <code>/kaggle/working/wandb/run-20231230_103732-dtz1lwzl</code>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"Syncing run <strong><a href='https://wandb.ai/arab/huggingface/runs/dtz1lwzl' target=\"_blank\">solar-wildflower-13</a></strong> to <a href='https://wandb.ai/arab/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View project at <a href='https://wandb.ai/arab/huggingface' target=\"_blank\">https://wandb.ai/arab/huggingface</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":" View run at <a href='https://wandb.ai/arab/huggingface/runs/dtz1lwzl' target=\"_blank\">https://wandb.ai/arab/huggingface/runs/dtz1lwzl</a>"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"\n    <div>\n      \n      <progress value='208' max='208' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [208/208 03:29, Epoch 8/8]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Rouge1</th>\n      <th>Rouge2</th>\n      <th>Rougel</th>\n      <th>Rougelsum</th>\n      <th>Gen Len</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>No log</td>\n      <td>3.061483</td>\n      <td>19.324800</td>\n      <td>6.376300</td>\n      <td>17.250200</td>\n      <td>17.252600</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>No log</td>\n      <td>2.546520</td>\n      <td>34.320000</td>\n      <td>18.953600</td>\n      <td>32.836400</td>\n      <td>33.066400</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>No log</td>\n      <td>2.251078</td>\n      <td>34.606000</td>\n      <td>18.909800</td>\n      <td>33.911500</td>\n      <td>33.942800</td>\n      <td>19.000000</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>No log</td>\n      <td>2.054790</td>\n      <td>36.568000</td>\n      <td>20.059200</td>\n      <td>35.550400</td>\n      <td>35.584500</td>\n      <td>18.961500</td>\n    </tr>\n    <tr>\n      <td>5</td>\n      <td>No log</td>\n      <td>1.945011</td>\n      <td>36.634400</td>\n      <td>19.543100</td>\n      <td>35.603400</td>\n      <td>35.642600</td>\n      <td>18.961500</td>\n    </tr>\n    <tr>\n      <td>6</td>\n      <td>No log</td>\n      <td>1.881996</td>\n      <td>36.183500</td>\n      <td>19.342900</td>\n      <td>35.305300</td>\n      <td>35.401900</td>\n      <td>18.961500</td>\n    </tr>\n    <tr>\n      <td>7</td>\n      <td>No log</td>\n      <td>1.841060</td>\n      <td>36.183500</td>\n      <td>19.342900</td>\n      <td>35.305300</td>\n      <td>35.401900</td>\n      <td>18.961500</td>\n    </tr>\n    <tr>\n      <td>8</td>\n      <td>No log</td>\n      <td>1.828572</td>\n      <td>36.183500</td>\n      <td>19.342900</td>\n      <td>35.305300</td>\n      <td>35.401900</td>\n      <td>18.961500</td>\n    </tr>\n  </tbody>\n</table><p>"},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/transformers/generation/utils.py:1355: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n  warnings.warn(\n","output_type":"stream"},{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"TrainOutput(global_step=208, training_loss=2.523646428034856, metrics={'train_runtime': 258.151, 'train_samples_per_second': 6.322, 'train_steps_per_second': 0.806, 'total_flos': 1358580296355840.0, 'train_loss': 2.523646428034856, 'epoch': 8.0})"},"metadata":{}}]},{"cell_type":"markdown","source":"You can now upload the result of the training to the Hub, just execute this instruction:","metadata":{"id":"6q4kKNN9f3NB"}},{"cell_type":"code","source":"trainer.push_to_hub()","metadata":{"id":"4IKlCy9Hf3NB","execution":{"iopub.status.busy":"2023-12-30T10:42:24.863925Z","iopub.execute_input":"2023-12-30T10:42:24.865057Z","iopub.status.idle":"2023-12-30T10:43:15.705787Z","shell.execute_reply.started":"2023-12-30T10:42:24.864999Z","shell.execute_reply":"2023-12-30T10:43:15.704565Z"},"trusted":true},"execution_count":36,"outputs":[{"output_type":"display_data","data":{"text/plain":"spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4f5ee83d54df44eb95478fa47b939fde"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"events.out.tfevents.1703932638.b865afdcd0e2.42.0:   0%|          | 0.00/9.99k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cd401c986bc04046805c384faa5b2064"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/892M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cfc39a4c645f4c5aa05bd48dbd17f8c7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Upload 4 LFS files:   0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4f9386f0f17140ca9ed1a5150696d8b0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"training_args.bin:   0%|          | 0.00/4.47k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f7c28320afa247a19a8f6e1fa7f861b5"}},"metadata":{}},{"execution_count":36,"output_type":"execute_result","data":{"text/plain":"'https://huggingface.co/ibtissam369/t5-base-finetuned-summarize-news-finetuned-xsum/tree/main/'"},"metadata":{}}]},{"cell_type":"code","source":"from huggingface_hub import notebook_login\n\n\nnotebook_login()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:14.502131Z","iopub.execute_input":"2023-12-30T10:42:14.502524Z","iopub.status.idle":"2023-12-30T10:42:14.530447Z","shell.execute_reply.started":"2023-12-30T10:42:14.502494Z","shell.execute_reply":"2023-12-30T10:42:14.528496Z"},"trusted":true},"execution_count":35,"outputs":[{"output_type":"display_data","data":{"text/plain":"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2fb318ee7592404294b0539ba4e29f86"}},"metadata":{}}]},{"cell_type":"markdown","source":"You can now share this model with all your friends, family, favorite pets: they can all load it with the identifier `\"your-username/the-name-you-picked\"` so for instance:\n\n```python\nfrom transformers import AutoModelForSeq2SeqLM\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"sgugger/my-awesome-model\")\n```","metadata":{"id":"D1O_EqMbf3NC"}},{"cell_type":"code","source":"","metadata":{"id":"g1eG5u1-f3ND"},"execution_count":null,"outputs":[]}]}