birdclef-2024-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T18:29:06.421649Z",
     "iopub.status.busy": "2024-06-13T18:29:06.421122Z",
     "iopub.status.idle": "2024-06-13T18:29:06.469841Z",
     "shell.execute_reply": "2024-06-13T18:29:06.468269Z",
     "shell.execute_reply.started": "2024-06-13T18:29:06.421606Z"
    }
   },
   "outputs": [],
   "source": [
    "KAGGLE = True\n",
    "\n",
    "isTrain = False\n",
    "isInference = True\n",
    "\n",
    "if isTrain == False and isInference == True:\n",
    "    newDir = False\n",
    "else:\n",
    "    newDir = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T18:29:13.169179Z",
     "iopub.status.busy": "2024-06-13T18:29:13.168724Z",
     "iopub.status.idle": "2024-06-13T18:29:13.178084Z",
     "shell.execute_reply": "2024-06-13T18:29:13.176267Z",
     "shell.execute_reply.started": "2024-06-13T18:29:13.169145Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "print(newDir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T18:30:52.834568Z",
     "iopub.status.busy": "2024-06-13T18:30:52.834044Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in links: /kaggle/input/onnxruntime\n",
      "Processing /kaggle/input/onnxruntime/humanfriendly-10.0-py2.py3-none-any.whl\n",
      "humanfriendly is already installed with the same version as the provided wheel. Use --force-reinstall to force an installation of the wheel.\n"
     ]
    }
   ],
   "source": [
    "if KAGGLE == True:\n",
    "    !pip install /kaggle/input/onnxruntime/humanfriendly-10.0-py2.py3-none-any.whl --no-index --find-links /kaggle/input/onnxruntime\n",
    "    !pip install /kaggle/input/onnxruntime/coloredlogs-15.0.1-py2.py3-none-any.whl --no-index --find-links /kaggle/input/onnxruntime\n",
    "    !pip install /kaggle/input/onnxruntime/onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl --no-index --find-links /kaggle/input/onnxruntime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:16:11.058613Z",
     "iopub.status.busy": "2024-06-07T14:16:11.058006Z",
     "iopub.status.idle": "2024-06-07T14:16:11.072492Z",
     "shell.execute_reply": "2024-06-07T14:16:11.070949Z",
     "shell.execute_reply.started": "2024-06-07T14:16:11.058572Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pytorch version is 2.1.2+cpu\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import gc\n",
    "import sys\n",
    "import glob\n",
    "import time\n",
    "import shutil\n",
    "import random\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "\n",
    "import wandb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import KFold, GroupKFold, StratifiedGroupKFold\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from torch.cuda import amp\n",
    "import torch\n",
    "print(f\"pytorch version is {torch.__version__}\")\n",
    "import torch.nn as nn\n",
    "from torch.cuda import amp\n",
    "\n",
    "import torchvision\n",
    "from torchvision.transforms import v2 as transforms\n",
    "\n",
    "import librosa\n",
    "import torchaudio\n",
    "import torchaudio.transforms as audioT\n",
    "\n",
    "if KAGGLE == False:\n",
    "    import nnAudio\n",
    "    from nnAudio import features\n",
    "    import albumentations\n",
    "    from audiomentations import Compose, SpecCompose, OneOf, AddGaussianNoise, AddColorNoise\n",
    "    from audiomentations import TimeStretch, PitchShift, Shift, SpecFrequencyMask, TimeMask\n",
    "    from audiomentations import Gain, GainTransition\n",
    "    from torcheval.metrics.functional import multiclass_auroc, multiclass_f1_score, multiclass_precision, multiclass_recall, multilabel_accuracy\n",
    "\n",
    "    if KAGGLE == False:\n",
    "    from adan_pytorch import Adan\n",
    "    \n",
    "import timm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:16:11.075366Z",
     "iopub.status.busy": "2024-06-07T14:16:11.074943Z",
     "iopub.status.idle": "2024-06-07T14:16:11.103422Z",
     "shell.execute_reply": "2024-06-07T14:16:11.102148Z",
     "shell.execute_reply.started": "2024-06-07T14:16:11.075329Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['eca_botnext26ts_256',\n",
       " 'eca_halonext26ts',\n",
       " 'eca_nfnet_l0',\n",
       " 'eca_nfnet_l1',\n",
       " 'eca_nfnet_l2',\n",
       " 'eca_nfnet_l3',\n",
       " 'eca_resnet33ts',\n",
       " 'eca_resnext26ts',\n",
       " 'eca_vovnet39b',\n",
       " 'ecaresnet26t',\n",
       " 'ecaresnet50d',\n",
       " 'ecaresnet50d_pruned',\n",
       " 'ecaresnet50t',\n",
       " 'ecaresnet101d',\n",
       " 'ecaresnet101d_pruned',\n",
       " 'ecaresnet200d',\n",
       " 'ecaresnet269d',\n",
       " 'ecaresnetlight',\n",
       " 'ecaresnext26t_32x4d',\n",
       " 'ecaresnext50t_32x4d',\n",
       " 'nf_ecaresnet26',\n",
       " 'nf_ecaresnet50',\n",
       " 'nf_ecaresnet101',\n",
       " 'selecsls42',\n",
       " 'selecsls42b',\n",
       " 'selecsls60',\n",
       " 'selecsls60b',\n",
       " 'selecsls84']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "timm.list_models(\"*ec*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.096941Z",
     "iopub.status.busy": "2024-05-05T06:38:55.096609Z",
     "iopub.status.idle": "2024-05-05T06:38:55.180746Z",
     "shell.execute_reply": "2024-05-05T06:38:55.179728Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.096914Z"
    }
   },
   "outputs": [],
   "source": [
    "## decide name\n",
    "if KAGGLE == False:\n",
    "    if isTrain == True:\n",
    "        name = sorted(glob.glob(\"exp1*.ipynb\"))[-1][:-6]\n",
    "        print(f\"filename is {name}\")\n",
    "    else:\n",
    "        name = \"exp1057\"\n",
    "        print(f\"filename is {name}\")\n",
    "else:\n",
    "    name = \"exp1057\"\n",
    "    name = f'bird2024{name}'\n",
    "    print(f\"filename is {name}\")\n",
    "trial = \"trial1\"\n",
    "p_name = f\"BirdCLEF_cv_ver2\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:22:51.986886Z",
     "iopub.status.busy": "2024-06-07T14:22:51.986465Z",
     "iopub.status.idle": "2024-06-07T14:22:52.003226Z",
     "shell.execute_reply": "2024-06-07T14:22:52.001103Z",
     "shell.execute_reply.started": "2024-06-07T14:22:51.986853Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "class config:\n",
    "    if KAGGLE:\n",
    "        dir = \"/kaggle/input/birdclef-2024/\"\n",
    "    else:\n",
    "        dir = \"/mnt/d/kaggle/birdclef-2024/\"\n",
    "\n",
    "    wave_path = \"original_waves/second_30/\"\n",
    "\n",
    "    model_name = 'tf_efficientnet_b0'\n",
    "\n",
    "    pool_type = 'avg'\n",
    "\n",
    "    \n",
    "    train_duration = 30 \n",
    "    slice_duration = 5 \n",
    "\n",
    "    test_duration = 5\n",
    "\n",
    "    train_drop_duration = 1\n",
    "    \n",
    "    #spectrogram parameters\n",
    "    sr = 32000\n",
    "    fmin = 20\n",
    "    fmax = 15000\n",
    "\n",
    "    n_mels = 128\n",
    "    n_fft = n_mels*8\n",
    "    size_x = 512\n",
    "    \n",
    "    hop_length = int(sr*slice_duration / size_x)\n",
    "    test_hop_length = int(sr*test_duration / size_x)\n",
    "    \n",
    "    bins_per_octave = 12\n",
    "\n",
    "    nfolds = 5\n",
    "    inference_folds = [4]\n",
    "    \n",
    "    enable_amp = True\n",
    "    train_batchsize = 32\n",
    "    valid_batchsize = 1\n",
    "\n",
    "    # loss_type = \"BCEWithLogitsLoss\"\n",
    "    loss_type = \"BCEFocalLoss\"\n",
    "\n",
    "    lr = 1.0e-03 \n",
    "\n",
    "    optimizer='adan' # optimizer='adamW'\n",
    "    \n",
    "    weight_decay = 1.0e-02\n",
    "    es_patience =  5\n",
    "    deterministic = True\n",
    "    enable_amp = True\n",
    "\n",
    "    max_epoch = 9\n",
    "    aug_epoch = 6\n",
    "    \n",
    "\n",
    "    useSecondary =True\n",
    "    secondary_label_value = 0.5\n",
    "    oversample =False\n",
    "    oversample_threthold = 60\n",
    "    \n",
    "    seed = 42\n",
    "\n",
    "    wandb = True\n",
    "\n",
    "    #augmentation flags\n",
    "    aug_noise            = 0.\n",
    "    aug_gain             = 0.0\n",
    "    aug_wave_pitchshift  = 0.0\n",
    "    aug_wave_shift       = 0.\n",
    "\n",
    "    aug_spec_xymasking   = 0.\n",
    "    aug_spec_coarsedrop  = 0.\n",
    "    aug_spec_hflip       = 0.\n",
    "\n",
    "    ##mixup param\n",
    "    aug_wave_mixup       = 1.0\n",
    "    aug_spec_mixup       = 0.0\n",
    "    aug_spec_mixup_prob  = 0.5 #specmixup\n",
    "    alpha=0.95\n",
    "\n",
    "    smoothing_value      = 0.0\n",
    "    # spec_mix_mask_percent = 20\n",
    "    \n",
    "cfg = config()\n",
    "\n",
    "device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "\n",
    "# device = torch.device('cpu')\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## augmentation tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:36:38.696512Z",
     "iopub.status.busy": "2024-06-07T14:36:38.695996Z",
     "iopub.status.idle": "2024-06-07T14:36:38.709361Z",
     "shell.execute_reply": "2024-06-07T14:36:38.707243Z",
     "shell.execute_reply.started": "2024-06-07T14:36:38.696468Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain== True:\n",
    "\n",
    "    normal_augment = Compose([\n",
    "        OneOf([\n",
    "            Gain(min_gain_in_db=-15, max_gain_in_db=15, p=1.0),\n",
    "            GainTransition(min_gain_in_db=-24.0, max_gain_in_db=6.0,\n",
    "                           min_duration=0.2, max_duration=6.0,  p=1.0)\n",
    "        ], p=cfg.aug_gain),\n",
    "        \n",
    "        OneOf([\n",
    "            AddGaussianNoise(p=1),\n",
    "            AddColorNoise(p=1, min_snr_db=5, max_snr_db=20, min_f_decay=-3.01, max_f_decay=-3.01)\n",
    "        ],p=cfg.aug_noise),\n",
    "    \n",
    "        PitchShift(min_semitones=-1, max_semitones=1, p=cfg.aug_wave_pitchshift),\n",
    "        Shift(p=cfg.aug_wave_shift)\n",
    "    ])\n",
    "    alb_transform = [\n",
    "        # num_masks_x=1, num_masks_y=1 will change\n",
    "        albumentations.XYMasking(num_masks_x=2, num_masks_y=1, \n",
    "                                 mask_x_length=cfg.size_x//30, mask_y_length=cfg.n_mels//30,\n",
    "                                 fill_value=0, mask_fill_value=0, p=cfg.aug_spec_xymasking),\n",
    "        albumentations.CoarseDropout(fill_value=0, min_holes=20, max_holes=50, p=cfg.aug_spec_coarsedrop),\n",
    "        albumentations.HorizontalFlip(p=cfg.aug_spec_hflip)    \n",
    "    ]\n",
    "    albumentations_augment = albumentations.Compose(alb_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:36:45.507118Z",
     "iopub.status.busy": "2024-06-07T14:36:45.506725Z",
     "iopub.status.idle": "2024-06-07T14:36:45.517092Z",
     "shell.execute_reply": "2024-06-07T14:36:45.515515Z",
     "shell.execute_reply.started": "2024-06-07T14:36:45.507086Z"
    }
   },
   "outputs": [],
   "source": [
    "def mixup(data, targets, alpha, mode=\"same_wave\"):\n",
    "    \n",
    "    if mode == \"same_wave\":\n",
    "        data = torch.tensor(data)\n",
    "        indices = torch.randperm(data.size(0))\n",
    "        shuffled_data = data[indices]\n",
    "        lam = np.random.beta(alpha, alpha)\n",
    "       \n",
    "        new_data = data * lam + shuffled_data * (1 - lam)\n",
    "        return new_data.numpy()\n",
    "        \n",
    "    elif mode == \"other_wave\":\n",
    "        indices = torch.randperm(data.size(0))\n",
    "        shuffled_data = data[indices]\n",
    "        shuffled_targets = targets[indices]\n",
    "    \n",
    "        lam = np.random.beta(alpha, alpha)\n",
    "        new_data = data * lam + shuffled_data * (1 - lam)\n",
    "        new_targets = targets * lam + shuffled_targets * (1 - lam)\n",
    "    \n",
    "        return new_data, new_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T14:41:18.327596Z",
     "iopub.status.busy": "2024-06-07T14:41:18.327165Z",
     "iopub.status.idle": "2024-06-07T14:41:18.342040Z",
     "shell.execute_reply": "2024-06-07T14:41:18.340671Z",
     "shell.execute_reply.started": "2024-06-07T14:41:18.327562Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain== True:\n",
    "    spec_xymasking = albumentations.XYMasking(num_masks_x=2, num_masks_y=1, \n",
    "                                              mask_x_length=cfg.size_x // 10, mask_y_length=cfg.n_mels // 10,\n",
    "                                              fill_value=0, mask_fill_value=0, p=1)\n",
    "\n",
    "def spec_mixup(data, targets):\n",
    "    type = data.dtype\n",
    "\n",
    "    indices = torch.randperm(data.size(0))\n",
    "    # print(indices)\n",
    "    shuffled_data = data[indices]\n",
    "    shuffled_targets = targets[indices]\n",
    "\n",
    "    ##masking\n",
    "    data = np.array(data)\n",
    "    data_transposed = np.transpose(data, (2, 3, 1, 0))\n",
    "    data_transposed = spec_xymasking(image=data_transposed)[\"image\"]\n",
    "    data_transposed = np.transpose(data_transposed, (3, 2, 0, 1))  \n",
    "\n",
    "    ##masking \n",
    "    diff = data - data_transposed\n",
    "    mask = (diff != 0).astype(int)\n",
    "\n",
    "    ##mask\n",
    "    shuffled_data_masked = (shuffled_data * mask)\n",
    "\n",
    "    ##mixup\n",
    "    new_data = torch.tensor(data_transposed, dtype=type) + torch.tensor(shuffled_data_masked, dtype=type)\n",
    "\n",
    "    lam = mask.sum() / len(data) / (cfg.n_mels*cfg.size_x)\n",
    "\n",
    "    new_targets = targets * (1-lam) + shuffled_targets *lam\n",
    "\n",
    "    return new_data, new_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.264548Z",
     "iopub.status.busy": "2024-05-05T06:38:55.264109Z",
     "iopub.status.idle": "2024-05-05T06:38:55.388973Z",
     "shell.execute_reply": "2024-05-05T06:38:55.388014Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.264511Z"
    }
   },
   "outputs": [],
   "source": [
    "spec_layer = torchaudio.transforms.MelSpectrogram(\n",
    "    sample_rate=cfg.sr, hop_length=cfg.hop_length, n_fft=cfg.n_fft,\n",
    "    n_mels=cfg.n_mels,f_min=cfg.fmin,f_max=cfg.fmax,mel_scale='slaney',center=True, pad_mode='reflect'\n",
    "    # normalized=True\n",
    ").to(device)\n",
    "\n",
    "valid_spec_layer = torchaudio.transforms.MelSpectrogram(\n",
    "    sample_rate=cfg.sr, hop_length=cfg.test_hop_length, n_fft=cfg.n_fft,\n",
    "    n_mels=cfg.n_mels,f_min=cfg.fmin,f_max=cfg.fmax,mel_scale='slaney',center=True, pad_mode='reflect'\n",
    "    # normalized=True\n",
    ").to(device)\n",
    "\n",
    "test_spec_layer = torchaudio.transforms.MelSpectrogram(\n",
    "    sample_rate=cfg.sr, hop_length=cfg.test_hop_length, n_fft=cfg.n_fft,\n",
    "    n_mels=cfg.n_mels,f_min=cfg.fmin,f_max=cfg.fmax,mel_scale='slaney',center=True, pad_mode='reflect'\n",
    "    # normalized=True\n",
    ").cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.390705Z",
     "iopub.status.busy": "2024-05-05T06:38:55.390271Z",
     "iopub.status.idle": "2024-05-05T06:38:55.396781Z",
     "shell.execute_reply": "2024-05-05T06:38:55.395091Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.390676Z"
    }
   },
   "outputs": [],
   "source": [
    "print(newDir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.398846Z",
     "iopub.status.busy": "2024-05-05T06:38:55.398417Z",
     "iopub.status.idle": "2024-05-05T06:38:55.411649Z",
     "shell.execute_reply": "2024-05-05T06:38:55.410384Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.398814Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    if cfg.wandb == True:\n",
    "        wandb.login(key=\"1def2aa62f171b4a95415cc2ad76b5eb1bb37b41\")\n",
    "\n",
    "    if newDir == True:\n",
    "        new_dir_path_recursive = f\"{name}/checkpoint\"\n",
    "    \n",
    "        os.makedirs(new_dir_path_recursive, exist_ok=True)\n",
    "        shutil.rmtree(new_dir_path_recursive)\n",
    "        os.makedirs(new_dir_path_recursive, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# csv preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.41413Z",
     "iopub.status.busy": "2024-05-05T06:38:55.413539Z",
     "iopub.status.idle": "2024-05-05T06:38:55.455826Z",
     "shell.execute_reply": "2024-05-05T06:38:55.454735Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.414099Z"
    }
   },
   "outputs": [],
   "source": [
    "sample_submission = pd.read_csv(cfg.dir+\"sample_submission.csv\")\n",
    "LABELS = list(sample_submission.set_index(\"row_id\").columns)\n",
    "LABELS[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.457904Z",
     "iopub.status.busy": "2024-05-05T06:38:55.45756Z",
     "iopub.status.idle": "2024-05-05T06:38:55.492377Z",
     "shell.execute_reply": "2024-05-05T06:38:55.491103Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.457877Z"
    }
   },
   "outputs": [],
   "source": [
    "sample_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.494077Z",
     "iopub.status.busy": "2024-05-05T06:38:55.493657Z",
     "iopub.status.idle": "2024-05-05T06:38:55.689115Z",
     "shell.execute_reply": "2024-05-05T06:38:55.687792Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.49404Z"
    }
   },
   "outputs": [],
   "source": [
    "# if isTrain:\n",
    "if KAGGLE == False:\n",
    "    train_csv = pd.read_csv(cfg.dir+\"train_eda.csv\")\n",
    "    train_csv[\"fileID\"] = train_csv[\"filename\"].map(lambda x:x.split(\"/\")[1][:-4])\n",
    "    train_csv.head(2)\n",
    "else:\n",
    "    train_csv = pd.read_csv(cfg.dir+\"train_metadata.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:55.691014Z",
     "iopub.status.busy": "2024-05-05T06:38:55.690672Z",
     "iopub.status.idle": "2024-05-05T06:38:56.182433Z",
     "shell.execute_reply": "2024-05-05T06:38:56.181539Z",
     "shell.execute_reply.started": "2024-05-05T06:38:55.690987Z"
    }
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "train_csv['new_target'] = train_csv['primary_label'] + ' ' + train_csv['secondary_labels'].map(lambda x: ' '.join(ast.literal_eval(x)))\n",
    "train_csv['len_new_target'] =train_csv['new_target'].map(lambda x: len(x.split()))\n",
    "train_csv[\"len_new_target\"].value_counts().plot(kind=\"bar\", figsize=(4,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.184702Z",
     "iopub.status.busy": "2024-05-05T06:38:56.184035Z",
     "iopub.status.idle": "2024-05-05T06:38:56.204935Z",
     "shell.execute_reply": "2024-05-05T06:38:56.203643Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.184659Z"
    }
   },
   "outputs": [],
   "source": [
    "train_csv[\"filename_tmp\"] = train_csv[\"filename\"].map(lambda x:x.split(\"/\")[1][:-4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.207617Z",
     "iopub.status.busy": "2024-05-05T06:38:56.206611Z",
     "iopub.status.idle": "2024-05-05T06:38:56.240644Z",
     "shell.execute_reply": "2024-05-05T06:38:56.239416Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.207576Z"
    }
   },
   "outputs": [],
   "source": [
    "duplicated_filenames = train_csv[\"filename_tmp\"].value_counts()[train_csv[\"filename_tmp\"].value_counts() > 1].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.244681Z",
     "iopub.status.busy": "2024-05-05T06:38:56.243522Z",
     "iopub.status.idle": "2024-05-05T06:38:56.253082Z",
     "shell.execute_reply": "2024-05-05T06:38:56.251654Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.244643Z"
    }
   },
   "outputs": [],
   "source": [
    "duplicated_filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.25527Z",
     "iopub.status.busy": "2024-05-05T06:38:56.254554Z",
     "iopub.status.idle": "2024-05-05T06:38:56.290396Z",
     "shell.execute_reply": "2024-05-05T06:38:56.289103Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.255236Z"
    }
   },
   "outputs": [],
   "source": [
    "train_csv = train_csv[~train_csv[\"filename_tmp\"].isin(duplicated_filenames)]\n",
    "train_csv = train_csv.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.294402Z",
     "iopub.status.busy": "2024-05-05T06:38:56.293593Z",
     "iopub.status.idle": "2024-05-05T06:38:56.329775Z",
     "shell.execute_reply": "2024-05-05T06:38:56.328425Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.294366Z"
    }
   },
   "outputs": [],
   "source": [
    "class BirdCLEF_Dataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, df, augmentation=False, mode='train'):\n",
    "        if mode == 'train':\n",
    "            self.df = df.reset_index(drop=True)\n",
    "        elif mode == 'valid':\n",
    "            self.df = df.reset_index(drop=True)\n",
    "        else:\n",
    "            self.df = df\n",
    "        self.mode = mode\n",
    "        self.augmentation = augmentation\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def normalize(self, x):\n",
    "        valid_values = x[x != float('-inf')]\n",
    "        mean_value = np.mean(valid_values)\n",
    "        x[x == float('-inf')] = mean_value\n",
    "\n",
    "        x = x - x.min()\n",
    "        x = x / x.max()\n",
    "        return x\n",
    "\n",
    "    def wave_tile_and_cutoff(self, data):\n",
    "        ### ---PREPROCESS wave length to train duration & slice duration-----\n",
    "        drop_duration = cfg.sr*cfg.train_drop_duration\n",
    "        use_duration  = cfg.sr*cfg.train_duration\n",
    "        \n",
    "        if len(data[0]) > drop_duration: \n",
    "            data = data[:,drop_duration:]\n",
    "\n",
    "        if len(data[0]) < use_duration: \n",
    "            iter = 1 + (use_duration) // len(data[0])\n",
    "            data = np.tile(data, (1, iter))\n",
    "\n",
    "        data = data[:,:use_duration]\n",
    "        return data\n",
    "\n",
    "    def label_smoothing(self, idx, target):\n",
    "    \n",
    "        secondary_target = target * cfg.secondary_label_value\n",
    "    \n",
    "        out_of_target_noise_intensity = cfg.smoothing_value/(len(LABELS)-1) #smoothing_value\n",
    "        out_of_target_noise_array = torch.ones(target.shape) * out_of_target_noise_intensity\n",
    "        \n",
    "        secondary_target_with_noise = secondary_target + out_of_target_noise_array\n",
    "        secondary_target_with_noise = torch.clip(secondary_target_with_noise, min=0, max=cfg.secondary_label_value)\n",
    "    \n",
    "        primary_target = np.isin(LABELS, self.df.loc[idx, \"primary_label\"]).astype(int)\n",
    "        primary_target = torch.tensor(primary_target, dtype=torch.float32)\n",
    "\n",
    "        primary_and_secondary_target_with_noise = primary_target + secondary_target_with_noise\n",
    "        new_target = torch.clip(primary_and_secondary_target_with_noise, min=0, max=1)\n",
    "    \n",
    "        new_target = new_target - primary_target * cfg.smoothing_value\n",
    "    \n",
    "        return new_target\n",
    "\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        if self.mode == 'train':\n",
    "\n",
    "            ### ------------------READ DATA  ------------------------------------\n",
    "            if cfg.useSecondary == True:\n",
    "                target = np.isin(LABELS, self.df.loc[idx, \"new_target\"].split()).astype(int)\n",
    "            else:\n",
    "                target = np.isin(LABELS, self.df.loc[idx, \"primary_label\"].split()).astype(int)\n",
    "            target = torch.tensor(target, dtype=torch.float32)\n",
    "            ### ------------------label smoothing  --------------------------\n",
    "            target = self.label_smoothing(idx, target)\n",
    "            \n",
    "            fileID = self.df.loc[idx, 'fileID'] #filename : ****/****.ogg\n",
    "            \n",
    "            path = f\"{cfg.wave_path}{fileID}.npy\"\n",
    "            wave = np.load(path)\n",
    "            ### -----------------------------------------------------------------\n",
    "\n",
    "            # ---PREPROCESS wave length to train duration & slice duration-------\n",
    "            wave = self.wave_tile_and_cutoff(data=wave)\n",
    "\n",
    "            \n",
    "            input_duration = cfg.sr * cfg.slice_duration\n",
    "            # middle_shift = cfg.sr * cfg.train_duration // 2 \n",
    "            \n",
    "            if self.augmentation == True:\n",
    "                ### ------------------wave time mixup  --------------------------\n",
    "                if cfg.aug_wave_mixup > np.random.random(): \n",
    "                    #train_duration -> slice_duration\n",
    "                    wave_reshape = wave.reshape(-1, input_duration)\n",
    "                    wave = mixup(data=wave_reshape, targets=target, alpha=cfg.alpha, mode=\"same_wave\")\n",
    "                    wave = wave[:1,:]\n",
    "                else:\n",
    "                    wave = wave[:, :input_duration]\n",
    "                \n",
    "                ### ------------------wave augmentation  ------------------------\n",
    "                wave = normal_augment(samples=wave, sample_rate=cfg.sr)\n",
    "\n",
    "                ### ------------------MAKE SPECTROGRAM  -------------------------\n",
    "                wave = torch.tensor(wave).to(device)\n",
    "                mel_spec = spec_layer(wave)\n",
    "                mel_spec = np.array(mel_spec.cpu())\n",
    "\n",
    "                mel_spec = np.log(mel_spec)\n",
    "                for i in range(len(mel_spec)):\n",
    "                    mel_spec[i] = self.normalize(mel_spec[i])\n",
    "                mel_spec = torch.tensor(mel_spec)\n",
    "                mel_spec = mel_spec[:,:,:cfg.size_x]\n",
    "\n",
    "                ### ------------------spec augmentation  ------------------------\n",
    "                mel_spec = np.array(mel_spec.cpu())\n",
    "                mel_spec = np.transpose(mel_spec, (1, 2, 0))                \n",
    "                mel_spec = albumentations_augment(image=mel_spec)[\"image\"]\n",
    "                mel_spec = np.transpose(mel_spec, (2, 0, 1))\n",
    "\n",
    "                # ### ------------------label smoothing  --------------------------\n",
    "                # target = self.label_smoothing(idx, target)\n",
    "                \n",
    "            else:\n",
    "                wave = wave[:, :input_duration]\n",
    "                \n",
    "                ### ------------------MAKE SPECTROGRAM  -------------------------\n",
    "                wave = torch.tensor(wave).to(device)\n",
    "                mel_spec = spec_layer(wave)\n",
    "                mel_spec = np.array(mel_spec.cpu())\n",
    "\n",
    "                mel_spec = np.log(mel_spec)\n",
    "\n",
    "                for i in range(len(mel_spec)):\n",
    "                    mel_spec[i] = self.normalize(mel_spec[i])\n",
    "                    \n",
    "\n",
    "                mel_spec = torch.tensor(mel_spec)\n",
    "                mel_spec = mel_spec[:,:,:cfg.size_x]\n",
    "\n",
    "            \n",
    "            mel_spec = torch.tensor(mel_spec)\n",
    "\n",
    "            \n",
    "            return mel_spec, target\n",
    "\n",
    "        elif self.mode == 'valid':\n",
    "            \n",
    "            ### ------------------READ DATA  ------------------------------------\n",
    "            if cfg.useSecondary == True:\n",
    "                target = np.isin(LABELS, self.df.loc[idx, \"new_target\"].split()).astype(int)\n",
    "            else:\n",
    "                target = np.isin(LABELS, self.df.loc[idx, \"primary_target\"].split()).astype(int)\n",
    "            target = torch.tensor(target, dtype=torch.float32)\n",
    "            \n",
    "            fileID = self.df.loc[idx, 'fileID'] #filename : ****/****.ogg\n",
    "            \n",
    "            path = f\"{cfg.wave_path}{fileID}.npy\"\n",
    "            wave = np.load(path)\n",
    "            ### -----------------------------------------------------------------\n",
    "\n",
    "            # ---PREPROCESS wave length to train duration & slice duration-------\n",
    "            wave = self.wave_tile_and_cutoff(data=wave)\n",
    "\n",
    "            input_duration = cfg.sr*cfg.test_duration\n",
    "            wave_reshape = wave.reshape(-1, input_duration)\n",
    "\n",
    "            wave_reshape = torch.tensor(wave_reshape).to(device)\n",
    "            mel_specs = valid_spec_layer(wave_reshape)\n",
    "            mel_specs = mel_specs.cpu().numpy()\n",
    "\n",
    "            mel_specs = np.log(mel_specs)\n",
    "            for i in range(len(mel_specs)):\n",
    "                mel_specs[i] = self.normalize(mel_specs[i])\n",
    "            mel_specs = torch.tensor(mel_specs)\n",
    "            \n",
    "            mel_specs = mel_specs[:,:,:cfg.size_x]\n",
    "\n",
    "            targets = torch.tile(target, dims=(mel_specs.shape[0],1))\n",
    "            return mel_specs, targets\n",
    "\n",
    "        ###---------------------------------------------------------------------------------------------------------------------\n",
    "        elif self.mode == 'test':\n",
    "\n",
    "            filepath = self.df[idx]\n",
    "            wave, _  = torchaudio.load(filepath)\n",
    "            wave = wave[:,:60*4*32000]\n",
    "\n",
    "            wave_reshaped = wave.reshape(-1, 1, cfg.test_duration*cfg.sr)\n",
    "            \n",
    "            mel_spec = test_spec_layer(wave_reshaped)\n",
    "            mel_spec = np.log(mel_spec)\n",
    "\n",
    "            mel_spec = np.array(mel_spec)\n",
    "            for i in range(len(mel_spec)):\n",
    "                mel_spec[i] = self.normalize(mel_spec[i])\n",
    "            mel_spec = torch.tensor(mel_spec)\n",
    "\n",
    "            mel_spec = mel_spec[:,:,:cfg.size_x]\n",
    "            return mel_spec\n",
    "\n",
    "        elif self.mode == 'clean':\n",
    "\n",
    "            filepath = self.df[idx]\n",
    "            wave, _  = torchaudio.load(filepath)\n",
    "\n",
    "            wave = wave[:, :6*cfg.test_duration*cfg.sr]\n",
    "\n",
    "            chunk_length = len(wave[0]) // (cfg.test_duration*cfg.sr)\n",
    "            \n",
    "            wave = wave[:,:chunk_length*cfg.test_duration*cfg.sr]\n",
    "\n",
    "            wave_reshaped = wave.reshape(-1, 1, cfg.test_duration*cfg.sr)\n",
    "            \n",
    "            mel_spec = test_spec_layer(wave_reshaped)\n",
    "            mel_spec = np.log(mel_spec)\n",
    "\n",
    "            mel_spec = np.array(mel_spec)\n",
    "            for i in range(len(mel_spec)):\n",
    "                mel_spec[i] = self.normalize(mel_spec[i])\n",
    "            mel_spec = torch.tensor(mel_spec)\n",
    "\n",
    "            return mel_spec, filepath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.331869Z",
     "iopub.status.busy": "2024-05-05T06:38:56.33144Z",
     "iopub.status.idle": "2024-05-05T06:38:56.344653Z",
     "shell.execute_reply": "2024-05-05T06:38:56.343408Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.331839Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain:\n",
    "    print(\"train data\")\n",
    "    dataset = BirdCLEF_Dataset(df=train_csv, augmentation=True,  mode=\"train\")\n",
    "    data, target = dataset[270]\n",
    "    fig, ax = plt.subplots(figsize=(6,4))\n",
    "    plt.imshow(data[0], cmap=\"jet\", origin=\"lower\")\n",
    "    plt.show()\n",
    "    \n",
    "    print(\"validation data\")\n",
    "    dataset = BirdCLEF_Dataset(df=train_csv, augmentation=True,  mode=\"valid\")\n",
    "    data, target = dataset[270]\n",
    "    fig, axes = plt.subplots(figsize=(12,8), nrows=len(data), tight_layout=True)\n",
    "    for idx, ax in enumerate(axes.ravel()):\n",
    "        ax.imshow(data[idx], cmap=\"jet\", origin=\"lower\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## poolings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### simple efficientnet SEDmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.371917Z",
     "iopub.status.busy": "2024-05-05T06:38:56.37129Z",
     "iopub.status.idle": "2024-05-05T06:38:56.384714Z",
     "shell.execute_reply": "2024-05-05T06:38:56.38308Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.371886Z"
    }
   },
   "outputs": [],
   "source": [
    "class BirdModel(torch.nn.Module):\n",
    "    def __init__(self, model_name, pretrained, in_channels, num_classes, pool=\"default\"):\n",
    "        super().__init__()\n",
    "\n",
    "        self.pool = pool\n",
    "        self.normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "        \n",
    "        if pool == \"default\":\n",
    "            self.backbone = timm.create_model(\n",
    "                model_name=model_name, pretrained=pretrained,\n",
    "                num_classes=0, in_chans=3)\n",
    "        else:\n",
    "            self.backbone = timm.create_model(\n",
    "                model_name=model_name, pretrained=pretrained,\n",
    "                num_classes=0, in_chans=3, global_pool=\"\")\n",
    "\n",
    "        in_features = self.backbone.num_features\n",
    "\n",
    "        self.max_pooling = torch.nn.Sequential(torch.nn.AdaptiveMaxPool2d(1),\n",
    "                                               torch.nn.Flatten(start_dim=1, end_dim=-1))\n",
    "        self.avg_pooling = torch.nn.Sequential(torch.nn.AdaptiveAvgPool2d(1),\n",
    "                                               torch.nn.Flatten(start_dim=1, end_dim=-1))\n",
    "        self.both_pooling_neck = torch.nn.Sequential(torch.nn.BatchNorm1d(2*in_features),\n",
    "                                                     torch.nn.Linear(in_features=2*in_features, out_features=in_features))\n",
    "        \n",
    "        self.head = torch.nn.Sequential(\n",
    "            torch.nn.BatchNorm1d(in_features),\n",
    "            torch.nn.Linear(in_features=in_features, out_features=256),\n",
    "            torch.nn.Hardswish(inplace=True),torch.nn.Dropout(0.1),\n",
    "            torch.nn.Linear(in_features=256, out_features=len(LABELS))  \n",
    "        )\n",
    "\n",
    "\n",
    "\n",
    "        self.active = torch.nn.Sigmoid()\n",
    "    def forward(self, x):\n",
    "        x = x.expand(-1, 3, -1, -1)\n",
    "        x = self.normalize(x)\n",
    "        x = self.backbone(x)\n",
    "\n",
    "        if self.pool == \"max\":\n",
    "            x = self.max_pooling(x)\n",
    "        elif self.pool == \"avg\":\n",
    "            x = self.avg_pooling(x)\n",
    "        elif self.pool == \"both\":\n",
    "            x_max = self.max_pooling(x)\n",
    "            x_avg = self.avg_pooling(x)\n",
    "            x = x_max + x_avg\n",
    "            \n",
    "        x = self.head(x)\n",
    "      \n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PANNs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.419905Z",
     "iopub.status.busy": "2024-05-05T06:38:56.419382Z",
     "iopub.status.idle": "2024-05-05T06:38:56.464117Z",
     "shell.execute_reply": "2024-05-05T06:38:56.462792Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.419875Z"
    }
   },
   "outputs": [],
   "source": [
    "# if isTrain:\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\n",
    "skf = StratifiedKFold(n_splits=cfg.nfolds, shuffle=True, random_state=cfg.seed)\n",
    "for fold, (train_index, valid_index) in enumerate(skf.split(train_csv, train_csv['primary_label'])):\n",
    "    train_csv.loc[valid_index, 'fold'] = int(fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.46686Z",
     "iopub.status.busy": "2024-05-05T06:38:56.465708Z",
     "iopub.status.idle": "2024-05-05T06:38:56.471827Z",
     "shell.execute_reply": "2024-05-05T06:38:56.470714Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.466823Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain:\n",
    "    train_csv.groupby(\"fold\", as_index=False)[\"primary_label\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trainer Utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## set seed function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.474984Z",
     "iopub.status.busy": "2024-05-05T06:38:56.473749Z",
     "iopub.status.idle": "2024-05-05T06:38:56.486828Z",
     "shell.execute_reply": "2024-05-05T06:38:56.485693Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.474857Z"
    }
   },
   "outputs": [],
   "source": [
    "def set_random_seed(seed: int = 42, deterministic: bool = False):\n",
    "    \"\"\"Set seeds\"\"\"\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)  # type: ignore\n",
    "    torch.backends.cudnn.deterministic = deterministic  # type: ignore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## INIT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.489138Z",
     "iopub.status.busy": "2024-05-05T06:38:56.488266Z",
     "iopub.status.idle": "2024-05-05T06:38:56.499022Z",
     "shell.execute_reply": "2024-05-05T06:38:56.497979Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.489088Z"
    }
   },
   "outputs": [],
   "source": [
    "class BCEFocalLoss(nn.Module):\n",
    "    def __init__(self, alpha=0.25, gamma=2.0):\n",
    "        super().__init__()\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "\n",
    "    def forward(self, preds, targets):\n",
    "        bce_loss = nn.BCEWithLogitsLoss(reduction='none')(preds, targets)\n",
    "        probas = torch.sigmoid(preds)\n",
    "\n",
    "        \n",
    "\n",
    "        tmp = targets * self.alpha * (1. - probas)**self.gamma * bce_loss\n",
    "        smp = (1. - targets) * probas**self.gamma * bce_loss\n",
    "        \n",
    "        loss = tmp + smp\n",
    "        loss = loss.mean()\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.500486Z",
     "iopub.status.busy": "2024-05-05T06:38:56.500114Z",
     "iopub.status.idle": "2024-05-05T06:38:56.518149Z",
     "shell.execute_reply": "2024-05-05T06:38:56.516853Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.500459Z"
    }
   },
   "outputs": [],
   "source": [
    "def initialization():\n",
    "    model = BirdModel(model_name=cfg.model_name, pretrained=True, in_channels=3, num_classes=len(LABELS), pool=cfg.pool_type)\n",
    "    \n",
    "    if cfg.optimizer=='adan':\n",
    "        optimizer = Adan(model.parameters(), lr=cfg.lr, betas=(0.02, 0.08, 0.01), weight_decay=cfg.weight_decay)\n",
    "    else:\n",
    "        optimizer = torch.optim.AdamW(params=model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n",
    "    \n",
    "    scheduler = torch.optim.lr_scheduler.OneCycleLR(\n",
    "        optimizer=optimizer, epochs=cfg.max_epoch,\n",
    "        pct_start=0.0, steps_per_epoch=len(train_dataloader),\n",
    "        max_lr=cfg.lr, div_factor=25, final_div_factor=4.0e-01\n",
    "    )\n",
    "    \n",
    "    scaler = amp.GradScaler(enabled=cfg.enable_amp)\n",
    "    if cfg.loss_type == \"BCEWithLogitsLoss\":\n",
    "        loss_func = torch.nn.BCEWithLogitsLoss()\n",
    "    elif cfg.loss_type == \"BCEFocalLoss\":\n",
    "        loss_func = BCEFocalLoss(alpha=1)\n",
    "\n",
    "    return model.to(device), optimizer, scheduler, scaler, loss_func.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.5198Z",
     "iopub.status.busy": "2024-05-05T06:38:56.519462Z",
     "iopub.status.idle": "2024-05-05T06:38:56.531397Z",
     "shell.execute_reply": "2024-05-05T06:38:56.530054Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.519775Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "from sklearn.metrics import mean_squared_error, roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## trainer function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.534382Z",
     "iopub.status.busy": "2024-05-05T06:38:56.534008Z",
     "iopub.status.idle": "2024-05-05T06:38:56.561489Z",
     "shell.execute_reply": "2024-05-05T06:38:56.560237Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.534347Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_one_loop(model, optimizer, scaler, scheduler, dataloader, loss_fn):\n",
    "    trainloss = 0; model.train()\n",
    "\n",
    "    count = 0\n",
    "    for idx, (data, label) in enumerate(tqdm(dataloader,leave=False ,desc=\"[train]\")):\n",
    "        \n",
    "        data, label = data.to(device), label.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        with amp.autocast(cfg.enable_amp, dtype=torch.bfloat16):\n",
    "            pred = model.forward(data)\n",
    "            loss = loss_fn(pred, label)\n",
    "\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        scheduler.step()\n",
    "        \n",
    "        trainloss += loss.item()\n",
    "        del data, label, loss\n",
    "        count += 1\n",
    "    trainloss /= len(dataloader)\n",
    "    if cfg.wandb == True:\n",
    "        wandb.log({f\"train_loss\": trainloss, f\"lr\":scheduler.get_lr()[0]})\n",
    "    return model, optimizer, scaler, scheduler, trainloss\n",
    "\n",
    "\n",
    "def mixup_one_loop(model, optimizer, scaler, scheduler, dataloader, loss_fn):\n",
    "    trainloss = 0; model.train()\n",
    "\n",
    "    count = 0\n",
    "    for idx, (data, label) in enumerate(tqdm(dataloader,leave=False ,desc=\"[train]\")):\n",
    "        if np.random.random()>cfg.aug_spec_mixup_prob:\n",
    "            data, label = mixup(data=data, targets=label, alpha=cfg.alpha, mode=\"other_wave\")\n",
    "        else:\n",
    "            data, label = spec_mixup(data=data, targets=label)\n",
    "        data, label = data.to(device), label.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        with amp.autocast(cfg.enable_amp, dtype=torch.bfloat16):\n",
    "        # with amp.autocast(cfg.enable_amp):\n",
    "            pred = model.forward(data)\n",
    "            loss = loss_fn(pred, label)\n",
    "\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        scheduler.step()\n",
    "        \n",
    "        trainloss += loss.item()\n",
    "        \n",
    "        del data, label, loss\n",
    "        count += 1\n",
    "       \n",
    "    trainloss /= len(dataloader)\n",
    "    if cfg.wandb == True:\n",
    "        wandb.log({f\"train_loss\": trainloss, f\"lr\":scheduler.get_lr()[0]})\n",
    "    return model, optimizer, scaler, scheduler, trainloss\n",
    "\n",
    "\n",
    "def evaluate_validation(model, dataloader, loss_fn):\n",
    "    validloss=0\n",
    "    model.eval()\n",
    "\n",
    "    preds, trues, targets = [], [], []\n",
    "    \n",
    "    for idx, (data, label) in enumerate(tqdm(dataloader,leave=False ,desc=\"[valid]\")):\n",
    "        # label = label.reshape(-1, len(LABELS))\n",
    "\n",
    "        d = data[0].unsqueeze(1)\n",
    "        label = label[0]\n",
    "        \n",
    "        d = d.to(device)\n",
    "        # with amp.autocast(cfg.enable_amp):\n",
    "        pred = model.forward(d)\n",
    "\n",
    "        preds.extend(pred.detach().cpu())\n",
    "        trues.extend(label)\n",
    "        targets.extend(label.argmax(axis=1))\n",
    "        \n",
    "    #======================== metrics ========================#\n",
    "    # y_preds = torch.stack(preds)\n",
    "    t = torch.stack(preds)\n",
    "    t = torch.sigmoid(t)\n",
    "    targets = torch.tensor(targets)\n",
    "    y_trues = torch.stack(trues)\n",
    "\n",
    "\n",
    "    validloss = loss_fn(torch.stack(preds), torch.stack(trues))\n",
    " \n",
    "    sk_f1_30 = metrics.f1_score(np.array(y_trues), np.array(t) > 0.30, average=\"micro\")\n",
    "    sk_f1_50 = metrics.f1_score(np.array(y_trues), np.array(t) > 0.50, average=\"micro\")\n",
    "    \n",
    "    auc = multiclass_auroc(input=t, target=targets, num_classes=len(LABELS),\n",
    "                           average=\"macro\").item()\n",
    "\n",
    "\n",
    "    prec = multiclass_precision(input=t, target=targets, num_classes=len(LABELS),\n",
    "                           average=\"macro\").item()\n",
    "   \n",
    "    f1 = multiclass_f1_score(input=t, target=torch.tensor(targets), num_classes=len(LABELS),\n",
    "                             average=\"micro\").item()\n",
    "\n",
    "    f1_macro = multiclass_f1_score(input=t, target=torch.tensor(targets), num_classes=len(LABELS),\n",
    "                             average=\"macro\").item()\n",
    "\n",
    "    t_03 = (t>0.3).int()\n",
    "    t_03 = torch.tensor(t_03, dtype=torch.int64)\n",
    "    f1_03 = multiclass_f1_score(input=t_03, target=torch.tensor(targets), num_classes=len(LABELS), \n",
    "                                average=\"micro\").item()\n",
    "\n",
    "    t_05 = (t>0.5).int()\n",
    "    t_05 = torch.tensor(t_05, dtype=torch.int64)\n",
    "    f1_05 = multiclass_f1_score(input=t_05, target=torch.tensor(targets), num_classes=len(LABELS), \n",
    "                                average=\"micro\").item()\n",
    "\n",
    "    if cfg.wandb == True:\n",
    "        wandb.log({f\"valid_loss\": validloss,\n",
    "                   f\"AUC\":auc,\n",
    "                   # \"auc_micro\":auc_micro,\n",
    "                   \"precision\":prec, \n",
    "                   # \"recall\":rec, \n",
    "                   # \"accuracy\":acc,\n",
    "                   f\"F1\":f1,\n",
    "                   \"F1_macro\":f1_macro,\n",
    "                   f\"F1 30%\":f1_03,\n",
    "                   f\"F1 50%\":f1_05})\n",
    "    return validloss, auc, f1, f1_03, f1_05, sk_f1_30, sk_f1_50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.563763Z",
     "iopub.status.busy": "2024-05-05T06:38:56.562683Z",
     "iopub.status.idle": "2024-05-05T06:38:56.574859Z",
     "shell.execute_reply": "2024-05-05T06:38:56.573712Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.56372Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain == True:\n",
    "    tmp_params = dict(vars(config))\n",
    "    del tmp_params['__module__'],tmp_params['__dict__'],tmp_params['__weakref__'],tmp_params['__doc__']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.591313Z",
     "iopub.status.busy": "2024-05-05T06:38:56.590494Z",
     "iopub.status.idle": "2024-05-05T06:38:56.603353Z",
     "shell.execute_reply": "2024-05-05T06:38:56.602306Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.591272Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_oversampled_df(df):\n",
    "    \n",
    "    new_df = [df]\n",
    "\n",
    "    low_sample_birds = df[\"primary_label\"].value_counts()[df[\"primary_label\"].value_counts() < cfg.oversample_threthold].index\n",
    "    for bird in low_sample_birds:\n",
    "        tmp = df[df[\"primary_label\"] == bird]\n",
    "        data_num = len(tmp)\n",
    "    \n",
    "        tiles = 1 + cfg.oversample_threthold // data_num\n",
    "    \n",
    "        tile_df = []\n",
    "        for i in range(tiles):\n",
    "            tile_df.append(tmp)\n",
    "    \n",
    "        tiled_df = pd.concat(tile_df)\n",
    "        piece = tiled_df[data_num:cfg.oversample_threthold]\n",
    "        new_df.append(piece)\n",
    "    \n",
    "    return pd.concat(new_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### in train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.605577Z",
     "iopub.status.busy": "2024-05-05T06:38:56.604863Z",
     "iopub.status.idle": "2024-05-05T06:38:56.628505Z",
     "shell.execute_reply": "2024-05-05T06:38:56.626595Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.605538Z"
    }
   },
   "outputs": [],
   "source": [
    "if isTrain == True:\n",
    "    set_random_seed(seed=42)\n",
    "    \n",
    "    \n",
    "    if cfg.wandb == True:\n",
    "        wandb.init(project=p_name, name=f\"{name}\",\n",
    "                   config=tmp_params)\n",
    "        \n",
    "    for fold in cfg.inference_folds:\n",
    "        train_ = train_csv.loc[train_csv[\"fold\"]!=fold]\n",
    "\n",
    "        if cfg.oversample == True:\n",
    "            train = get_oversampled_df(df=train_)\n",
    "        else:\n",
    "            train = train_\n",
    "        \n",
    "        augme_dataset = BirdCLEF_Dataset(df=train, augmentation=True, mode='train')\n",
    "        augme_dataloader = torch.utils.data.DataLoader(dataset=augme_dataset, batch_size=cfg.train_batchsize, shuffle=True)\n",
    "\n",
    "        train_dataset = BirdCLEF_Dataset(df=train, augmentation=False, mode='train')\n",
    "        train_dataloader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=cfg.train_batchsize, shuffle=True)\n",
    "        \n",
    "        valid = train_csv.loc[train_csv[\"fold\"]==fold]\n",
    "        valid_dataset = BirdCLEF_Dataset(df=valid, augmentation=False, mode='valid')\n",
    "        valid_dataloader = torch.utils.data.DataLoader(dataset=valid_dataset, batch_size=cfg.valid_batchsize, shuffle=False)\n",
    "    \n",
    "        model, optimizer, scheduler, scaler, loss_func =  initialization()\n",
    "    \n",
    "    \n",
    "        best_f1 = 0\n",
    "        best_auc = 0\n",
    "        best_loss = 1.00000\n",
    "        for e in range(cfg.max_epoch):\n",
    "            start_time = time.time()\n",
    "            if e < cfg.aug_epoch:\n",
    "                if cfg.aug_spec_mixup > np.random.random():\n",
    "                    model, optimizer, scaler, shcheduler, train_loss = mixup_one_loop(model=model,optimizer=optimizer,scaler=scaler, \n",
    "                                                                                          scheduler=scheduler,dataloader=augme_dataloader, loss_fn=loss_func)\n",
    "                else:\n",
    "                    model, optimizer, scaler, shcheduler, train_loss = train_one_loop(model=model,optimizer=optimizer,scaler=scaler, \n",
    "                                                                                          scheduler=scheduler,dataloader=augme_dataloader, loss_fn=loss_func)\n",
    "\n",
    "            else:\n",
    "                model, optimizer, scaler, shcheduler, train_loss = train_one_loop(model=model,optimizer=optimizer,scaler=scaler, \n",
    "                                                                                          scheduler=scheduler,dataloader=train_dataloader, loss_fn=loss_func)\n",
    "            \n",
    "            valid_loss, auc, f1, f1_03, f1_05, sk_f1_30, sk_f1_50 = evaluate_validation(model=model, dataloader=valid_dataloader, loss_fn=loss_func)\n",
    "            \n",
    "            if best_loss > valid_loss:\n",
    "                end_time = time.time()\n",
    "                print(f\"[epoch {str(e).zfill(2)}] AUC{auc: .4f}, F1{f1: .4f}, F1_03{f1_03: .4f}, F1_05{f1_05: .4f}\")\n",
    "                print(f\"[epoch {str(e).zfill(2)}] SKF1_03{sk_f1_30: .4f}, SKF1_05{sk_f1_50: .4f}\")\n",
    "                print(f\"[epoch {str(e).zfill(2)}] valid_loss {valid_loss: .6f}\")\n",
    "                print(f\"[epoch {str(e).zfill(2)}] update loss {best_loss: .6f} --> {valid_loss: .6f} {(end_time - start_time): .1f}[s]\")\n",
    "                print(f\"[epoch {str(e).zfill(2)}] update auc score {best_auc: .6f} --> {auc: .6f} {(end_time - start_time): .1f}[s]\")\n",
    "                model_name = f'{name}/checkpoint/fold_{fold}_snapshot_epoch_{str(e).zfill(2)}.pth'\n",
    "                best_model = model\n",
    "                best_loss = valid_loss\n",
    "                best_auc = auc\n",
    "                best_f1 = f1\n",
    "            else:\n",
    "                end_time = time.time()\n",
    "                print(f\"[epoch {str(e).zfill(2)}] NOT update loss {best_loss: .6f} <-- {valid_loss: .6f} {(end_time - start_time): .1f}[s]\")\n",
    "                print(f\"[epoch {str(e).zfill(2)}] NOT update score {best_auc: .6f} <-- {auc: .6f} {(end_time - start_time): .1f}[s]\")\n",
    "\n",
    "        if cfg.wandb == True:\n",
    "            wandb.log({f\"best_loss\": best_loss,\n",
    "                       f\"best_f1\": best_f1,\n",
    "                       f\"best_auc\":best_auc})\n",
    "\n",
    "        torch.save(best_model.state_dict(), model_name)\n",
    "        \n",
    "        del model, best_model\n",
    "        gc.collect()\n",
    "        torch.cuda.empty_cache()\n",
    "        print(\"--\")\n",
    "        # break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:56.655415Z",
     "iopub.status.busy": "2024-05-05T06:38:56.653987Z",
     "iopub.status.idle": "2024-05-05T06:38:57.234755Z",
     "shell.execute_reply": "2024-05-05T06:38:57.233273Z",
     "shell.execute_reply.started": "2024-05-05T06:38:56.655373Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "models = dict()\n",
    "models_names = dict()\n",
    "# for fold in range(cfg.nfolds):\n",
    "for fold in cfg.inference_folds:\n",
    "    if KAGGLE == True:\n",
    "        bestmodel_path = sorted(glob.glob(f\"/kaggle/input/{name}/checkpoint/fold_{fold}*.pth\"))[-1]\n",
    "    else:\n",
    "        bestmodel_path = sorted(glob.glob(f\"{name}/checkpoint/fold_{fold}*.pth\"))[-1]\n",
    "    print(bestmodel_path)\n",
    "    model = BirdModel(model_name=cfg.model_name, pretrained=False, in_channels=1, num_classes=len(LABELS))\n",
    "    model.load_state_dict(torch.load(bestmodel_path, map_location=torch.device('cpu')))\n",
    "    model = model.eval()\n",
    "    models[fold] = model\n",
    "\n",
    "    models_names[fold] = bestmodel_path.split(\".\")[0]+\".onnx\"\n",
    "    print(models_names[fold])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.237302Z",
     "iopub.status.busy": "2024-05-05T06:38:57.236204Z",
     "iopub.status.idle": "2024-05-05T06:38:57.245735Z",
     "shell.execute_reply": "2024-05-05T06:38:57.244715Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.237263Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == True:\n",
    "    test_audio_dir = f\"{cfg.dir}test_soundscapes/\"\n",
    "    file_list = glob.glob(test_audio_dir+\"*.ogg\")\n",
    "    file_list = sorted(file_list)\n",
    "\n",
    "if KAGGLE == False:\n",
    "    test_audio_dir = f\"{cfg.dir}unlabeled_soundscapes/\"\n",
    "    file_list = glob.glob(test_audio_dir+\"*.ogg\")\n",
    "    file_list = sorted(file_list)[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.249104Z",
     "iopub.status.busy": "2024-05-05T06:38:57.247309Z",
     "iopub.status.idle": "2024-05-05T06:38:57.259066Z",
     "shell.execute_reply": "2024-05-05T06:38:57.257897Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.248844Z"
    }
   },
   "outputs": [],
   "source": [
    "test_dataset = BirdCLEF_Dataset(df=file_list, mode=\"test\")\n",
    "test_dataloader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                              batch_size=1, \n",
    "                                              shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.261187Z",
     "iopub.status.busy": "2024-05-05T06:38:57.260749Z",
     "iopub.status.idle": "2024-05-05T06:38:57.26954Z",
     "shell.execute_reply": "2024-05-05T06:38:57.26817Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.261147Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    print(test_dataset[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ONNX"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## set onnx config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.271929Z",
     "iopub.status.busy": "2024-05-05T06:38:57.271561Z",
     "iopub.status.idle": "2024-05-05T06:38:57.309339Z",
     "shell.execute_reply": "2024-05-05T06:38:57.308086Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.271897Z"
    }
   },
   "outputs": [],
   "source": [
    "input_tensor = torch.randn((48, 1, cfg.n_mels, cfg.size_x+1))  # input shape\n",
    "output_names=['output']\n",
    "input_names=[\"x\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## export onnx model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.311594Z",
     "iopub.status.busy": "2024-05-05T06:38:57.310734Z",
     "iopub.status.idle": "2024-05-05T06:38:57.317514Z",
     "shell.execute_reply": "2024-05-05T06:38:57.316264Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.311558Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    # for fold in range(cfg.nfolds):\n",
    "    for fold in cfg.inference_folds:\n",
    "        torch.onnx.export(model=models[fold].eval(),args=(input_tensor), \n",
    "                          input_names=input_names, output_names=output_names,f=models_names[fold])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model convert from pytorch to onnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.319175Z",
     "iopub.status.busy": "2024-05-05T06:38:57.318826Z",
     "iopub.status.idle": "2024-05-05T06:38:57.328797Z",
     "shell.execute_reply": "2024-05-05T06:38:57.327743Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.319147Z"
    }
   },
   "outputs": [],
   "source": [
    "import onnx\n",
    "import onnxruntime as ort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.330838Z",
     "iopub.status.busy": "2024-05-05T06:38:57.330168Z",
     "iopub.status.idle": "2024-05-05T06:38:57.342914Z",
     "shell.execute_reply": "2024-05-05T06:38:57.342021Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.330807Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "models_names = dict()\n",
    "\n",
    "for fold in cfg.inference_folds:\n",
    "    if KAGGLE == True:\n",
    "        onnxmodel_path = sorted(glob.glob(f\"/kaggle/input/{name}/checkpoint/fold_{fold}*.onnx\"))[-1]\n",
    "    else:\n",
    "        onnxmodel_path = sorted(glob.glob(f\"{name}/checkpoint/fold_{fold}*.onnx\"))[-1]\n",
    "    print(onnxmodel_path)\n",
    "\n",
    "    models_names[fold] = onnxmodel_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.344443Z",
     "iopub.status.busy": "2024-05-05T06:38:57.344114Z",
     "iopub.status.idle": "2024-05-05T06:38:57.714749Z",
     "shell.execute_reply": "2024-05-05T06:38:57.713522Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.344418Z"
    }
   },
   "outputs": [],
   "source": [
    "onnx_sessions = dict()\n",
    "\n",
    "for fold in cfg.inference_folds:\n",
    "\n",
    "    onnx_model = onnx.load(models_names[fold])\n",
    "    onnx_model_graph = onnx_model.graph\n",
    "    onnx_session = ort.InferenceSession(onnx_model.SerializeToString())\n",
    "\n",
    "    onnx_sessions[fold] = onnx_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.724634Z",
     "iopub.status.busy": "2024-05-05T06:38:57.723754Z",
     "iopub.status.idle": "2024-05-05T06:38:57.755734Z",
     "shell.execute_reply": "2024-05-05T06:38:57.754955Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.724594Z"
    }
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "predictions = []\n",
    "for data in tqdm(test_dataloader):\n",
    "    \n",
    "    preds = []\n",
    "    \n",
    "\n",
    "    for fold in cfg.inference_folds:\n",
    "        session = onnx_sessions[fold]\n",
    "        pred = session.run(output_names, {input_names[0]: data[0].numpy()})[0]\n",
    "        \n",
    "        pred = torch.sigmoid(torch.tensor(pred))\n",
    "        preds.append(pred)\n",
    "    preds_per_batch = torch.stack(preds, axis=0).mean(axis=0)\n",
    "    \n",
    "    predictions.extend(preds_per_batch)\n",
    "    \n",
    "if len(predictions)>0:\n",
    "    predictions = torch.stack(predictions)\n",
    "else:\n",
    "    predictions = predictions\n",
    "end_time = time.time()\n",
    "use_time = end_time - start_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.758072Z",
     "iopub.status.busy": "2024-05-05T06:38:57.756947Z",
     "iopub.status.idle": "2024-05-05T06:38:57.763236Z",
     "shell.execute_reply": "2024-05-05T06:38:57.761842Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.758042Z"
    }
   },
   "outputs": [],
   "source": [
    "print(f\"{cfg.inference_folds}fold +     3ogg is {round(use_time,1)}[s]\")\n",
    "print(f\"{cfg.inference_folds}fold + 1,100ogg is {round(1100*use_time/3,1)}[s], {round(1100*use_time/3/60,1)}[m]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.778066Z",
     "iopub.status.busy": "2024-05-05T06:38:57.777455Z",
     "iopub.status.idle": "2024-05-05T06:38:57.798608Z",
     "shell.execute_reply": "2024-05-05T06:38:57.797433Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.778036Z"
    }
   },
   "outputs": [],
   "source": [
    "bird_cols = sample_submission.columns[1:]\n",
    "df = pd.DataFrame(columns=['row_id']+list(bird_cols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.801415Z",
     "iopub.status.busy": "2024-05-05T06:38:57.800272Z",
     "iopub.status.idle": "2024-05-05T06:38:57.816349Z",
     "shell.execute_reply": "2024-05-05T06:38:57.814589Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.801374Z"
    }
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.819029Z",
     "iopub.status.busy": "2024-05-05T06:38:57.818231Z",
     "iopub.status.idle": "2024-05-05T06:38:57.829644Z",
     "shell.execute_reply": "2024-05-05T06:38:57.828414Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.818985Z"
    }
   },
   "outputs": [],
   "source": [
    "row_list = []\n",
    "for file in file_list:\n",
    "    dataname = file.split(\"/\")[-1][:-4]\n",
    "    for i in range(int(4*60/5)):\n",
    "        row = f\"{dataname}_{(i+1)*5}\"\n",
    "        row_list.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.83134Z",
     "iopub.status.busy": "2024-05-05T06:38:57.830989Z",
     "iopub.status.idle": "2024-05-05T06:38:57.841771Z",
     "shell.execute_reply": "2024-05-05T06:38:57.840906Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.831295Z"
    }
   },
   "outputs": [],
   "source": [
    "df['row_id'] = row_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.843969Z",
     "iopub.status.busy": "2024-05-05T06:38:57.843011Z",
     "iopub.status.idle": "2024-05-05T06:38:57.868284Z",
     "shell.execute_reply": "2024-05-05T06:38:57.866873Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.843936Z"
    }
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.870807Z",
     "iopub.status.busy": "2024-05-05T06:38:57.870034Z",
     "iopub.status.idle": "2024-05-05T06:38:57.881092Z",
     "shell.execute_reply": "2024-05-05T06:38:57.879952Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.870774Z"
    }
   },
   "outputs": [],
   "source": [
    "if len(predictions) < 1:\n",
    "    pass\n",
    "else:\n",
    "    df[bird_cols] = predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.883094Z",
     "iopub.status.busy": "2024-05-05T06:38:57.882524Z",
     "iopub.status.idle": "2024-05-05T06:38:57.906716Z",
     "shell.execute_reply": "2024-05-05T06:38:57.905827Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.883064Z"
    }
   },
   "outputs": [],
   "source": [
    "df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.908634Z",
     "iopub.status.busy": "2024-05-05T06:38:57.908055Z",
     "iopub.status.idle": "2024-05-05T06:38:57.922419Z",
     "shell.execute_reply": "2024-05-05T06:38:57.921429Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.908605Z"
    }
   },
   "outputs": [],
   "source": [
    "df.to_csv(\"submission.csv\", index=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:57.924758Z",
     "iopub.status.busy": "2024-05-05T06:38:57.923835Z",
     "iopub.status.idle": "2024-05-05T06:38:59.683444Z",
     "shell.execute_reply": "2024-05-05T06:38:59.682316Z",
     "shell.execute_reply.started": "2024-05-05T06:38:57.924728Z"
    }
   },
   "outputs": [],
   "source": [
    "df[:48].set_index(\"row_id\").max().T.plot(kind=\"bar\", figsize=(24,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.685539Z",
     "iopub.status.busy": "2024-05-05T06:38:59.684697Z",
     "iopub.status.idle": "2024-05-05T06:38:59.692786Z",
     "shell.execute_reply": "2024-05-05T06:38:59.69179Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.685507Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    def get_fold_data(fold):\n",
    "        \n",
    "        valid = train_csv.loc[train_csv[\"fold\"]==fold]\n",
    "        filelist = [f\"{cfg.dir}train_audio/{name}\" for name in valid[\"filename\"]]\n",
    "    \n",
    "        clearn_dataset = BirdCLEF_Dataset(df=filelist, augmentation=False, mode='clean')\n",
    "        return clearn_dataset, valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.695102Z",
     "iopub.status.busy": "2024-05-05T06:38:59.694091Z",
     "iopub.status.idle": "2024-05-05T06:38:59.706604Z",
     "shell.execute_reply": "2024-05-05T06:38:59.705395Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.69506Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    def predict_and_visualize(data_index):\n",
    "        use_fold = 4\n",
    "        c_dataset, df = get_fold_data(fold=use_fold)\n",
    "        model = models[use_fold].to(device)\n",
    "        \n",
    "        prediction = model.forward(c_dataset[data_index][0].to(device)).cpu().detach()\n",
    "        prediction = torch.sigmoid(prediction)\n",
    "        \n",
    "        df_index = df.index[data_index]\n",
    "        true_labels = df.loc[df_index, \"new_target\"].split()\n",
    "        true_guide_pos = [LABELS.index(l)+0.5 for l in true_labels]\n",
    "    \n",
    "        fig, ax = plt.subplots(figsize=(24, 1.5))\n",
    "        sns.heatmap(prediction, cmap=\"jet\", vmin=0, vmax=1)\n",
    "        ax.set_xticks(np.arange(0,182))\n",
    "        ax.set_xticklabels(LABELS, fontsize=8)\n",
    "        \n",
    "        ax.set_yticks(np.arange(0,prediction.shape[0]))\n",
    "        ax.set_yticklabels(np.arange(1,1+prediction.shape[0])*5)\n",
    "        \n",
    "        for pos in true_guide_pos:\n",
    "            ax.axvline(x=pos, color=\"red\", ls=\"--\", lw=0.9)\n",
    "        \n",
    "        plt.xticks(ticks=np.arange(0,182), labels=LABELS, color='black') \n",
    "        plt.title(c_dataset[data_index][1])\n",
    "        plt.show()\n",
    "\n",
    "        return prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.708953Z",
     "iopub.status.busy": "2024-05-05T06:38:59.707998Z",
     "iopub.status.idle": "2024-05-05T06:38:59.723903Z",
     "shell.execute_reply": "2024-05-05T06:38:59.723018Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.708915Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "\n",
    "    N = len(train_csv.loc[train_csv[\"fold\"]==fold])//30\n",
    "    print(N)\n",
    "    \n",
    "    predict_and_visualize(data_index=0)\n",
    "    predict_and_visualize(data_index=1)\n",
    "    predict_and_visualize(data_index=2)\n",
    "    predict_and_visualize(data_index=3)\n",
    "    predict_and_visualize(data_index=4)\n",
    "\n",
    "\n",
    "    for i in range(N):\n",
    "        try:\n",
    "            predict_and_visualize(data_index=i*30)\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.725871Z",
     "iopub.status.busy": "2024-05-05T06:38:59.725221Z",
     "iopub.status.idle": "2024-05-05T06:38:59.738221Z",
     "shell.execute_reply": "2024-05-05T06:38:59.737173Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.725833Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    model.eval().cuda();\n",
    "    \n",
    "    preds, trues, targets = [], [], []\n",
    "    for idx, (data, label) in enumerate(tqdm(valid_dataloader, leave=False ,desc=\"[valid]\")):\n",
    "        data = data.to(device)\n",
    "        label = label[0]\n",
    "        pred = model.forward(data[0].unsqueeze(1))\n",
    "    \n",
    "        preds.extend(pred.detach().cpu())\n",
    "        trues.extend(label)\n",
    "        targets.extend(label.argmax(axis=1))\n",
    "\n",
    "    t = torch.stack(preds)\n",
    "    targets = torch.tensor(targets)\n",
    "\n",
    "    auc = multiclass_auroc(input=torch.sigmoid(t), target=targets, num_classes=len(LABELS),\n",
    "                               average=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.739996Z",
     "iopub.status.busy": "2024-05-05T06:38:59.739647Z",
     "iopub.status.idle": "2024-05-05T06:38:59.754113Z",
     "shell.execute_reply": "2024-05-05T06:38:59.753175Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.739969Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    train_[\"primary_label\"].value_counts().sort_index().plot(kind=\"bar\", figsize=(20,4))\n",
    "    plt.scatter(x=LABELS, y=auc*100, color=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.755918Z",
     "iopub.status.busy": "2024-05-05T06:38:59.755186Z",
     "iopub.status.idle": "2024-05-05T06:38:59.767039Z",
     "shell.execute_reply": "2024-05-05T06:38:59.766033Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.755889Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    train[\"primary_label\"].value_counts().sort_index().plot(kind=\"bar\", figsize=(20,4))\n",
    "    plt.scatter(x=LABELS, y=auc*100, color=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.768669Z",
     "iopub.status.busy": "2024-05-05T06:38:59.768303Z",
     "iopub.status.idle": "2024-05-05T06:38:59.781664Z",
     "shell.execute_reply": "2024-05-05T06:38:59.780561Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.768642Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    valid[\"primary_label\"].value_counts().sort_index().plot(kind=\"bar\", figsize=(20,4))\n",
    "    plt.scatter(x=LABELS, y=auc*100, color=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-05T06:38:59.783435Z",
     "iopub.status.busy": "2024-05-05T06:38:59.782903Z",
     "iopub.status.idle": "2024-05-05T06:38:59.795186Z",
     "shell.execute_reply": "2024-05-05T06:38:59.794407Z",
     "shell.execute_reply.started": "2024-05-05T06:38:59.783405Z"
    }
   },
   "outputs": [],
   "source": [
    "if KAGGLE == False:\n",
    "    fig, axes = plt.subplots(figsize=(24,62), ncols=6, nrows=31, tight_layout=True)\n",
    "    \n",
    "    axs = axes.ravel()\n",
    "\n",
    "    LABELS_COUNT = train_csv[\"primary_label\"].value_counts().sort_index().to_numpy()\n",
    "    LABELS_OCCUPY = train_csv[\"primary_label\"].value_counts(normalize=True).sort_index().to_numpy()\n",
    "    \n",
    "    oof_predictions = torch.sigmoid(torch.stack(preds))\n",
    "    oof_predictions = pd.DataFrame(oof_predictions, columns=LABELS)\n",
    "    \n",
    "    oof_predictions.head()\n",
    "    \n",
    "    oof_trues = torch.stack(trues)\n",
    "    oof_trues = pd.DataFrame(oof_trues, columns=LABELS)\n",
    "    \n",
    "    oof_trues.head()\n",
    "    LABEL_df = pd.DataFrame([LABELS,LABELS_COUNT]).T\n",
    "    LABEL_df.columns = [\"name\", \"count\"]\n",
    "    LABEL_df = LABEL_df.sort_values(\"count\").reset_index(drop=True)\n",
    "    iter = len(LABEL_df)\n",
    "    \n",
    "    for idx in range(iter):\n",
    "        ax = axs[idx]\n",
    "        \n",
    "        l = LABEL_df.loc[idx, \"name\"]\n",
    "        count = LABEL_df.loc[idx, \"count\"]\n",
    "        \n",
    "        tmp = pd.concat([oof_trues[f\"{l}\"], oof_predictions[f\"{l}\"]], axis=1)\n",
    "        tmp.columns = [\"label\", f\"{l}_prob\"]\n",
    "        \n",
    "        tmp[\"10_label\"] = (10*tmp[\"label\"]).astype(int)\n",
    "        \n",
    "        sns.boxplot(tmp, x=f\"{l}_prob\", y=\"10_label\",orient=\"h\", width=0.5, ax=ax)\n",
    "        ax.set(title=f\"{l}_{count}\", xlim=(0,1))\n",
    "    \n",
    "\n",
    "    \n"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 8068726,
     "sourceId": 70203,
     "sourceType": "competition"
    },
    {
     "datasetId": 4789213,
     "sourceId": 8108072,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4941521,
     "sourceId": 8319412,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30698,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02fe8717a4174559b29d090c46116c2d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "03f0f01f7c7f43f1b85d0e2bdff06d9a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "061b756217a249749200246514290335": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "06d33df4d6744e26ae0037159f71c384": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "LabelModel",
      "state": {
       "layout": "IPY_MODEL_270b73c4145d4f6f8ee6da27320f4f9e",
       "style": "IPY_MODEL_eeffc7c179324bc1b1e3d5906d436929"
      }
     },
     "06f30db6bb1b47f8ad238b88cd9d4c1c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "07405e55b9a643879d1d398434f27cfe": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "091e4ffed6354178b87ac9aa2a4209e2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d5126feb65a84bf783aa3284e376584e",
       "style": "IPY_MODEL_bbc9a503bcde4d1ebeff8398d56103d2",
       "value": " 4882/4884 [01:02&lt;00:00, 79.03it/s]"
      }
     },
     "0933dd87781e4431a8f04ab5b65e3a22": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "0b2fc8732aed4a599824830267af2f73": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "0c9e0276e3514a0597f1f31b7872d9ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "0cf766e3112c4ee0985496009562e4aa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_54deeee9b5b44412b904c2f47b533a77",
       "max": 1,
       "style": "IPY_MODEL_b5a921221b104d6ca418d56c5276a121"
      }
     },
     "112b6346380e42e1801d61eee0de9026": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "140ad88ae7994469a3b1c7f01870f9d1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "14fc82d4c6b8459b81b4d7d09e98e903": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "1544aa3d209d42d8a12ab9f1fd0ad099": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "15bdfecbe1f6473abc87d1e80f4ca6f1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "17a54d28655e4691b9c52b0ed404d3d7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "17b144777f214073b2828a7ea0b50ef1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_dfe735036baf4b9c846db77d23020db5",
       "max": 4884,
       "style": "IPY_MODEL_50998d280c2f4c4183beb25609e59c82",
       "value": 4884
      }
     },
     "186e913ca7684366a17d4557ca0ec308": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "195763d98e914c908cf7cd344a634e5a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "1976e03c78e2441fab5e2c7921ddd9e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4c0ac39781ec4c4f86b841d399eb4a45",
       "style": "IPY_MODEL_46ce40b8284d49c0a4286157844fa134",
       "value": " 611/611 [02:16&lt;00:00,  4.72it/s]"
      }
     },
     "1a06a20e4f0b48a0804383ab0a9a1bfe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "1ab2af883837482b954e1bbe20c26966": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_9032768dcf2449bf8952b32889de4215",
       "style": "IPY_MODEL_c0f322f5999c4440b1a691347113772b",
       "value": "[valid]: 100%"
      }
     },
     "1b07e1114fc343569a52f66c9d826fb2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "1da955485f914bad8183d3dbd92ed407": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_f32e62905f264f4faef84608247bf38f",
        "IPY_MODEL_ab32ce4c072747a7be743029028cd59f",
        "IPY_MODEL_4c85565caefd4a8293c0382a873d4844"
       ],
       "layout": "IPY_MODEL_8639778685754dd48bf26533df357753"
      }
     },
     "1dee12ab1c3747b7ac4f0c3b180e6211": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_cba7cc09798a4184b9d038f1adddc995",
       "style": "IPY_MODEL_0c9e0276e3514a0597f1f31b7872d9ee",
       "value": " 4882/4884 [01:12&lt;00:00, 71.93it/s]"
      }
     },
     "1e3b6bac7ecd41848626b564fef4623e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_195763d98e914c908cf7cd344a634e5a",
       "style": "IPY_MODEL_d3318e85bcd948adbf9d570b8dfb561c",
       "value": " 611/611 [01:41&lt;00:00,  6.19it/s]"
      }
     },
     "1fd9c512d9824aa6b9e2301a8a526263": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "20b4961ee51342e98dc0e7dc920f980d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "20c00ba41907414f8472daa151d580e2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "22f299428e2b482e9f7fe90fbb4527a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_8e98a27dbc974de49bdac906f104ede7",
       "max": 4884,
       "style": "IPY_MODEL_2cd29b22c2474ba4a2e892a463b9b8e9",
       "value": 4884
      }
     },
     "2317dc59e31847d1b67a44a6d7dde5a1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "23220efd32d14e129a9e4c9e9d723fb0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2355887aaba540e19dcccfc3d778e89c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "25e7c11eb60a424699ccef04bc3d7b66": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "270b73c4145d4f6f8ee6da27320f4f9e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "27fdc5c0761b4459a0cfe9c6c8ade0bb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_20b4961ee51342e98dc0e7dc920f980d",
       "style": "IPY_MODEL_2921a4e2d33f4335b9f4cb46d289f52b",
       "value": "[valid]: 100%"
      }
     },
     "28984a5809054bcf83950dd5a03082f8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2921a4e2d33f4335b9f4cb46d289f52b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "293f145d85de4c53a4ffe148a04c65f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "295ee76c7cc744f7860b27ea58c96e29": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2c18e73d89fa46d19633c059430ea609": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2c90df9a4e1b4f3480ed6348a80879d0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_2c18e73d89fa46d19633c059430ea609",
       "style": "IPY_MODEL_9d895b5f9b3345bb8ee48906fb57a60f",
       "value": "[train]: 100%"
      }
     },
     "2cd29b22c2474ba4a2e892a463b9b8e9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "2d3869eba94a4d18bddba3dfce1cecbf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "2decc01d2a2446bd88b0dfecc61cf9d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "2e925960ed12472db35195134647c046": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "30eaa95ca7a24837b81d4890d597b449": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "30fa31ec8f6b4e83a30d3b3103db9e36": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4fc482d2ad1e469286db6875e7df2720",
       "style": "IPY_MODEL_2d3869eba94a4d18bddba3dfce1cecbf",
       "value": "[train]: 100%"
      }
     },
     "35495808e600467bbe02a9f2dc767da7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_5ed5653ab8b34fe291247ca1552b8c23",
       "max": 611,
       "style": "IPY_MODEL_e0f6759dddc44440b520e67fa7e38f82",
       "value": 611
      }
     },
     "361a6767bb0d449799781aa85ff1fd89": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "LabelModel",
      "state": {
       "layout": "IPY_MODEL_57bbec94c7134ca58678038923f51b93",
       "style": "IPY_MODEL_e9644fd5104649f5ae01a6683c2c0a70"
      }
     },
     "36a905e637684084be11e9b842532c75": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "37b322e318c14ea88b6d70de68ce650b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "381431430f30483b8d779e5bac901eb6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "387eab9a2013484185e404497b2e7cda": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_423b6c7ab21e47128ea8badc0a50d385",
       "max": 1,
       "style": "IPY_MODEL_186e913ca7684366a17d4557ca0ec308"
      }
     },
     "38c38e2d21b9490e8a5d05d78b495259": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "39cc77d0541e44ad9a7690de505eafae": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "3bcdfe3e1c4d47d58f7a600903e248a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_5f278ac36e0247ffa67f1d7423e5752f",
       "max": 611,
       "style": "IPY_MODEL_b8f7b440e8be467db07ee8649834972e",
       "value": 611
      }
     },
     "423b6c7ab21e47128ea8badc0a50d385": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "42c7926a934842fda6dd3a9ecf01aa13": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_28984a5809054bcf83950dd5a03082f8",
       "max": 4884,
       "style": "IPY_MODEL_80d0f5a240d145d5ac49a50ff0bacbaf",
       "value": 4884
      }
     },
     "438846f3b01740e2a8977b7ca346ef5a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_b6848109aa194c878ad2dc0b873ba70b",
       "style": "IPY_MODEL_b1f73d56731745c383dec77892f3e84a",
       "value": " 4882/4884 [01:09&lt;00:00, 63.73it/s]"
      }
     },
     "43c030e0da7f45759fd2cf2c02fc203b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "43d9ba117c7c4320af778d545fee5a96": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4bf473847b8c4cd384b1c20da59c70a7",
       "style": "IPY_MODEL_fbb6c4535a954480b189532142f2e0b6",
       "value": "[valid]: 100%"
      }
     },
     "44bb2ee826b042a99f92fd00626ab63b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "44c1825ca9ab4d19b5d3d062a96b9a59": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "44e12afca69545afb862d70aff9ef183": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "45ab25fe097a4833b30514cfbe7cdb87": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_39cc77d0541e44ad9a7690de505eafae",
       "max": 4884,
       "style": "IPY_MODEL_4c7660b2d5274ea9aeb007ec38b08541",
       "value": 4884
      }
     },
     "46ba6f3704dd4bd3a289f95c93fe25de": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "46ce40b8284d49c0a4286157844fa134": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "4868eac352e54bb0af4b2b3d13d79822": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_5b1c2abcc8ac4aafba1d5b58a8b0ccef",
       "style": "IPY_MODEL_ab7268b858214d9ba296ac5e3ac14228",
       "value": " 4882/4884 [01:03&lt;00:00, 70.05it/s]"
      }
     },
     "48ab02a7d319440785559f990ae5b2b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_f786420c6b5a4649aa257f6bad91b251",
       "max": 4884,
       "style": "IPY_MODEL_dbadece046f44a37984bdce97aae92fa",
       "value": 4884
      }
     },
     "4943b7235e014c5cb2b42448fe613f98": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_ea23b4838454451e94855bc2f9df9ae2",
       "style": "IPY_MODEL_2decc01d2a2446bd88b0dfecc61cf9d4",
       "value": "[train]: 100%"
      }
     },
     "4bbaa725987a4626895cf10f76579ddb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4bf473847b8c4cd384b1c20da59c70a7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4c0ac39781ec4c4f86b841d399eb4a45": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4c7660b2d5274ea9aeb007ec38b08541": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "4c85565caefd4a8293c0382a873d4844": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_30eaa95ca7a24837b81d4890d597b449",
       "style": "IPY_MODEL_a908addfc91247cc9c7a513306ab5ae2",
       "value": " 3/3 [00:02&lt;00:00,  1.33it/s]"
      }
     },
     "4d8f3078c4a64d22b9e4c4640d8ae220": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4dcbcfb7d85d4d88904f79c3736634d1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4e294638bebc477b8205aa112f00f373": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "4ee1217c36c4446bacd591970580e38e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "4fc482d2ad1e469286db6875e7df2720": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "503771bbda2440019a7b08a8edbf050e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "50998d280c2f4c4183beb25609e59c82": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "518084d9c98848e3afda881660683879": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f0e3f458d71a4c73bcb62cf074d8de3a",
       "style": "IPY_MODEL_a7068f3f3de240b7a9ef686e0deedc61",
       "value": "[train]: 100%"
      }
     },
     "5444055d667d4c8996e3c9fd9d7ec541": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_1544aa3d209d42d8a12ab9f1fd0ad099",
       "style": "IPY_MODEL_36a905e637684084be11e9b842532c75",
       "value": " 611/611 [01:42&lt;00:00,  6.31it/s]"
      }
     },
     "54deeee9b5b44412b904c2f47b533a77": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "56c1b090c5694b9e8d71201e9d19d3a0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "57bbec94c7134ca58678038923f51b93": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "57ef677b81f74d5385c2fea49bba0285": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "58fd4f425514402fbde832abd73f1869": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_cbca16677b8c49358921b3ff84da79a2",
       "style": "IPY_MODEL_b01e37f2b0e046c393ac80fcfd3aeb0d",
       "value": " 611/611 [02:18&lt;00:00,  4.52it/s]"
      }
     },
     "5b1c2abcc8ac4aafba1d5b58a8b0ccef": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "5c04fab07c3d4ae99a4420153d688a24": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "5d7a48a9beee46c6842c5753c6d81b57": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_25e7c11eb60a424699ccef04bc3d7b66",
       "style": "IPY_MODEL_061b756217a249749200246514290335",
       "value": " 611/611 [02:10&lt;00:00,  4.64it/s]"
      }
     },
     "5ed5653ab8b34fe291247ca1552b8c23": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "5f278ac36e0247ffa67f1d7423e5752f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "60a1b70fdb77407c845ba795f89917b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d1a822fe66064867b2adc415034c0b2e",
       "style": "IPY_MODEL_293f145d85de4c53a4ffe148a04c65f3",
       "value": " 611/611 [02:04&lt;00:00,  4.93it/s]"
      }
     },
     "60c90539ee3b4a288abff3ec430a2efd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "694a7df59f9f49f494fb1cccb7255d28": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "6c0a0ab62016423e87e3be098ff0fee2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_7d20bc72263b49889a8ed0960f2a72c2",
       "style": "IPY_MODEL_57ef677b81f74d5385c2fea49bba0285",
       "value": "[valid]: 100%"
      }
     },
     "6c0ce36a3d2942ae83e880a3dbbd5146": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6ed44b5ef88a43ee9960850b5b2f639d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4d8f3078c4a64d22b9e4c4640d8ae220",
       "style": "IPY_MODEL_37b322e318c14ea88b6d70de68ce650b",
       "value": "[valid]: 100%"
      }
     },
     "720a038eb12f4d0f8772699490108086": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "74df4d778ade494e83a9d820bab4d21e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_a122834936c74f7d8808eae90b90e6b0",
       "style": "IPY_MODEL_17a54d28655e4691b9c52b0ed404d3d7",
       "value": " 611/611 [02:07&lt;00:00,  4.86it/s]"
      }
     },
     "77a51de5ed554142938a71537d953ddd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7a4fa6bb9e49409fb94a72c12e47ef90": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "7b22364d13a943f38ac4408c3b288cd0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7d20bc72263b49889a8ed0960f2a72c2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7d70b14ed6ce4aa3a121a068b8f1efe1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "7ec941a5f1bf4fc3a7a24e6140e1baba": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "7f98c349285a4565a69192e822fefb20": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "800857501aae421da256a358023132db": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f4501d9766c8492d85304660e21701af",
       "style": "IPY_MODEL_afa75fc3d8f64540a1ed05bf9f3a57ee",
       "value": " 611/611 [01:44&lt;00:00,  5.73it/s]"
      }
     },
     "80d0f5a240d145d5ac49a50ff0bacbaf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "81795ad89f3146529e854ec3482d54eb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_23220efd32d14e129a9e4c9e9d723fb0",
       "style": "IPY_MODEL_f54b89fa9d8842588061271b06db177a",
       "value": "[train]: 100%"
      }
     },
     "84f8fe6a67ae446d8547656cba64ad3c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8639778685754dd48bf26533df357753": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "89916767fc9d4e3da97dcb3cae03cfdb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_46ba6f3704dd4bd3a289f95c93fe25de",
       "style": "IPY_MODEL_44e12afca69545afb862d70aff9ef183",
       "value": "[valid]: 100%"
      }
     },
     "8d3a77d9e5884eba8999539de9a5855e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_112b6346380e42e1801d61eee0de9026",
       "style": "IPY_MODEL_ac5576f180334e8da6eca075e9aa6713",
       "value": " 4884/4884 [01:08&lt;00:00, 71.12it/s]"
      }
     },
     "8e403d0f73914f7ea15f30936a4866c9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_b23f085cae76428fab55f364a4093133",
       "style": "IPY_MODEL_c1635ad889c2486b839924e8bb5127f4",
       "value": " 4879/4884 [01:11&lt;00:00, 67.53it/s]"
      }
     },
     "8e98a27dbc974de49bdac906f104ede7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "9032768dcf2449bf8952b32889de4215": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "90352f76a5394be6842e3a035f140039": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_43c030e0da7f45759fd2cf2c02fc203b",
       "style": "IPY_MODEL_694a7df59f9f49f494fb1cccb7255d28",
       "value": "[valid]: 100%"
      }
     },
     "917dc73e097848ee89338fa8054ed91c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4ee1217c36c4446bacd591970580e38e",
       "style": "IPY_MODEL_d62d26349f7a49eda837a5de838ae11f",
       "value": "[train]: 100%"
      }
     },
     "92d9f6900c2a4a4d84be052ee834290f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_295ee76c7cc744f7860b27ea58c96e29",
       "style": "IPY_MODEL_9a466cb881e64fabb33c1468c4ce9e65",
       "value": " 4876/4884 [01:05&lt;00:00, 74.61it/s]"
      }
     },
     "96c372e7cad3495c807278ccfab0420b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_4dcbcfb7d85d4d88904f79c3736634d1",
       "style": "IPY_MODEL_7a4fa6bb9e49409fb94a72c12e47ef90",
       "value": " 4882/4884 [01:02&lt;00:00, 81.43it/s]"
      }
     },
     "9811e50e70e94ef89eb7868308b84597": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "9884e44b9bcf42c58cb9c15bacc0e24e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "989af5b761774d3f8af0f49dd721c775": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_07405e55b9a643879d1d398434f27cfe",
       "style": "IPY_MODEL_15bdfecbe1f6473abc87d1e80f4ca6f1",
       "value": " 611/611 [02:18&lt;00:00,  4.69it/s]"
      }
     },
     "9a466cb881e64fabb33c1468c4ce9e65": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "9a9c04550e264b049c571cc4c039794f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_a9fbb173f3614b5fa4bd47a3a7251166",
       "style": "IPY_MODEL_56c1b090c5694b9e8d71201e9d19d3a0",
       "value": "[valid]: 100%"
      }
     },
     "9d895b5f9b3345bb8ee48906fb57a60f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "9f150db6f159483cb1ead4246f44fec7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "a122834936c74f7d8808eae90b90e6b0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a441d284a49b44afa153c5295c2fa29b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "a6ce0fe7f84a426c848889b12709fdb7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_af6c8bec8dd5408cb3ba7ad9124b0af5",
       "max": 4884,
       "style": "IPY_MODEL_c0f6dcd6daeb47f8ba04625fbee2f985",
       "value": 4884
      }
     },
     "a7068f3f3de240b7a9ef686e0deedc61": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a8d2030528da49d08d5d388a9c3e8457": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_a9540922c2254105bc151affce8ee221",
       "style": "IPY_MODEL_1b07e1114fc343569a52f66c9d826fb2",
       "value": "[valid]: 100%"
      }
     },
     "a908addfc91247cc9c7a513306ab5ae2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a9540922c2254105bc151affce8ee221": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a9fbb173f3614b5fa4bd47a3a7251166": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ab32ce4c072747a7be743029028cd59f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_9811e50e70e94ef89eb7868308b84597",
       "max": 3,
       "style": "IPY_MODEL_ea9aeb3a4e1142eebb5411c393f964cc",
       "value": 3
      }
     },
     "ab7268b858214d9ba296ac5e3ac14228": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "ac5576f180334e8da6eca075e9aa6713": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "acfcba84b6134c029bebeb4ec0ac1da9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_03f0f01f7c7f43f1b85d0e2bdff06d9a",
       "style": "IPY_MODEL_b84da86552d04eb3bb1d5f12ecc00b64",
       "value": "[train]: 100%"
      }
     },
     "adf0fd9a368640c58cc1b387b9734b07": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ae88f84d0d8741f9951fbbc5087fb264": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "aefed2992a594d398031fd9237a6bfae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "af6c8bec8dd5408cb3ba7ad9124b0af5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "afa75fc3d8f64540a1ed05bf9f3a57ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "b01e37f2b0e046c393ac80fcfd3aeb0d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "b1f73d56731745c383dec77892f3e84a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "b23f085cae76428fab55f364a4093133": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b2cc8b8759614bc28f89582a5778a495": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b5a921221b104d6ca418d56c5276a121": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b6848109aa194c878ad2dc0b873ba70b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b786738430b840d2b11005b2fc97c09c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "b84da86552d04eb3bb1d5f12ecc00b64": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "b8f7b440e8be467db07ee8649834972e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "baf56bc36f2441b8b9a86c6da3b4324e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_02fe8717a4174559b29d090c46116c2d",
       "max": 611,
       "style": "IPY_MODEL_bbd67d5315a047048fee7dfdba370f92",
       "value": 611
      }
     },
     "bafdf2ed70b44a4682e45a86cd0543f0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_6c0ce36a3d2942ae83e880a3dbbd5146",
       "style": "IPY_MODEL_720a038eb12f4d0f8772699490108086",
       "value": " 4883/4884 [01:07&lt;00:00, 71.73it/s]"
      }
     },
     "bbc9a503bcde4d1ebeff8398d56103d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "bbd67d5315a047048fee7dfdba370f92": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "bc9e24e42c6c4f45be3b1840edf16302": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_20c00ba41907414f8472daa151d580e2",
       "style": "IPY_MODEL_d1c53938c299455d83c5ec6b9c0d4b23",
       "value": " 4883/4884 [01:09&lt;00:00, 57.23it/s]"
      }
     },
     "be452a144d5942418154485e84d6a4d7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "beead1b8336c4bc590a6eb2be3aa66eb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "bf3e6d4ef4164401897268c25432c216": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_14fc82d4c6b8459b81b4d7d09e98e903",
       "style": "IPY_MODEL_38c38e2d21b9490e8a5d05d78b495259",
       "value": "[train]: 100%"
      }
     },
     "c0f322f5999c4440b1a691347113772b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "c0f6dcd6daeb47f8ba04625fbee2f985": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "c1635ad889c2486b839924e8bb5127f4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "c53857add44a43279ffc790f9f274ae2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "c8e8b3c9405c4420a2110aa1359c9659": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "cba7cc09798a4184b9d038f1adddc995": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "cbca16677b8c49358921b3ff84da79a2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ccbd13fce37a4274bb023c093fcab63b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "d1a5559fe40e49139ad784eb33d90951": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "d1a822fe66064867b2adc415034c0b2e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d1c53938c299455d83c5ec6b9c0d4b23": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d3318e85bcd948adbf9d570b8dfb561c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d5126feb65a84bf783aa3284e376584e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d62d26349f7a49eda837a5de838ae11f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d789837d080a4fa0a96ef26f4cce271a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_7d70b14ed6ce4aa3a121a068b8f1efe1",
       "max": 611,
       "style": "IPY_MODEL_2e925960ed12472db35195134647c046",
       "value": 611
      }
     },
     "d85f714070f9404396e18a116c73bd28": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_c53857add44a43279ffc790f9f274ae2",
       "max": 611,
       "style": "IPY_MODEL_1a06a20e4f0b48a0804383ab0a9a1bfe",
       "value": 611
      }
     },
     "db4fea4177a849669ce9c996948a0fcf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "dbadece046f44a37984bdce97aae92fa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "dc5cdf2bb09a46c4be8a1541421b7bb3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "dd67c9fc37b7436f8ebf2066bdfdba8d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "df964b59d36c4639ba229411334bd664": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_77a51de5ed554142938a71537d953ddd",
       "max": 611,
       "style": "IPY_MODEL_503771bbda2440019a7b08a8edbf050e",
       "value": 611
      }
     },
     "dfe735036baf4b9c846db77d23020db5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e04b35fe1a8a4ebe90626a3082eb03d0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_2355887aaba540e19dcccfc3d778e89c",
       "style": "IPY_MODEL_140ad88ae7994469a3b1c7f01870f9d1",
       "value": "[train]: 100%"
      }
     },
     "e0f6759dddc44440b520e67fa7e38f82": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "e1362fd7d8724156802ebf87ac83a5af": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "e4b0e19e9414447e999eb9a960884749": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_0b2fc8732aed4a599824830267af2f73",
       "max": 4884,
       "style": "IPY_MODEL_db4fea4177a849669ce9c996948a0fcf",
       "value": 4884
      }
     },
     "e9099c2b53d54f1486487fb6e1a00c63": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_beead1b8336c4bc590a6eb2be3aa66eb",
       "max": 611,
       "style": "IPY_MODEL_ae88f84d0d8741f9951fbbc5087fb264",
       "value": 611
      }
     },
     "e9644fd5104649f5ae01a6683c2c0a70": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "LabelStyleModel",
      "state": {
       "description_width": "",
       "font_family": null,
       "font_size": null,
       "font_style": null,
       "font_variant": null,
       "font_weight": null,
       "text_color": null,
       "text_decoration": null
      }
     },
     "ea23b4838454451e94855bc2f9df9ae2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ea9aeb3a4e1142eebb5411c393f964cc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "eb12e34ce6d64b26931615dce154a83b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "VBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_361a6767bb0d449799781aa85ff1fd89",
        "IPY_MODEL_387eab9a2013484185e404497b2e7cda"
       ],
       "layout": "IPY_MODEL_4bbaa725987a4626895cf10f76579ddb"
      }
     },
     "eef8a7cbffa94d1380cda46aa7b14492": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "eeffc7c179324bc1b1e3d5906d436929": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "LabelStyleModel",
      "state": {
       "description_width": "",
       "font_family": null,
       "font_size": null,
       "font_style": null,
       "font_variant": null,
       "font_weight": null,
       "text_color": null,
       "text_decoration": null
      }
     },
     "f01eefc17443457cbe3b3521e047932f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_adf0fd9a368640c58cc1b387b9734b07",
       "max": 611,
       "style": "IPY_MODEL_5c04fab07c3d4ae99a4420153d688a24",
       "value": 611
      }
     },
     "f0e3f458d71a4c73bcb62cf074d8de3a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f123cab305424f96a57b4b0006f516d2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "f32e62905f264f4faef84608247bf38f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_f7b6764a6cdd49d798c443a96fd821fd",
       "style": "IPY_MODEL_9884e44b9bcf42c58cb9c15bacc0e24e",
       "value": "100%"
      }
     },
     "f447f181f9994ce0bc9d536c53fb40a9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_be452a144d5942418154485e84d6a4d7",
       "max": 4884,
       "style": "IPY_MODEL_eef8a7cbffa94d1380cda46aa7b14492",
       "value": 4884
      }
     },
     "f4501d9766c8492d85304660e21701af": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f54b89fa9d8842588061271b06db177a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "f6bfedd312e247baaa84dc076c6cf884": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_0933dd87781e4431a8f04ab5b65e3a22",
       "max": 4884,
       "style": "IPY_MODEL_f70a45910a8342ff84e28c0553566e91",
       "value": 4884
      }
     },
     "f70a45910a8342ff84e28c0553566e91": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "f786420c6b5a4649aa257f6bad91b251": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f7b6764a6cdd49d798c443a96fd821fd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "f9601e5efd654022aa04ec39769782aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "fbb6c4535a954480b189532142f2e0b6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "fc26e7a1d1e542caadf42eb3b3db292d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "VBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_06d33df4d6744e26ae0037159f71c384",
        "IPY_MODEL_0cf766e3112c4ee0985496009562e4aa"
       ],
       "layout": "IPY_MODEL_84f8fe6a67ae446d8547656cba64ad3c"
      }
     },
     "fca15a52cee2466ca29a1cbd55528a9d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_7b22364d13a943f38ac4408c3b288cd0",
       "max": 4884,
       "style": "IPY_MODEL_aefed2992a594d398031fd9237a6bfae",
       "value": 4884
      }
     },
     "fd379b98f80d45c9a509cd67371b8da1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "layout": "IPY_MODEL_1fd9c512d9824aa6b9e2301a8a526263",
       "max": 611,
       "style": "IPY_MODEL_b2cc8b8759614bc28f89582a5778a495",
       "value": 611
      }
     },
     "fdb29577e8b045419e8906cb2fcd3755": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "visibility": "hidden"
      }
     },
     "ffdb4e5e92ff4b9a87e5f63a16179773": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_06f30db6bb1b47f8ad238b88cd9d4c1c",
       "style": "IPY_MODEL_2317dc59e31847d1b67a44a6d7dde5a1",
       "value": "[valid]: 100%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}