From 61dfd16f9a6fd93a1f3ff6a60805929e8d582365 Mon Sep 17 00:00:00 2001
From: Fafa-DL <516451964@qq.com>
Date: Fri, 25 Feb 2022 21:39:59 +0800
Subject: [PATCH] Update L2
---
.../ML2022Spring_HW2.ipynb | 922 ++++++++++++++++++
.../hw2_slides 2022.pdf | Bin 0 -> 828027 bytes
.../theory (v7).pdf | Bin 0 -> 1118252 bytes
README.md | 2 +
4 files changed, 924 insertions(+)
create mode 100644 2022 ML/02 What to do if my network fails to train/ML2022Spring_HW2.ipynb
create mode 100644 2022 ML/02 What to do if my network fails to train/hw2_slides 2022.pdf
create mode 100644 2022 ML/02 What to do if my network fails to train/theory (v7).pdf
diff --git a/2022 ML/02 What to do if my network fails to train/ML2022Spring_HW2.ipynb b/2022 ML/02 What to do if my network fails to train/ML2022Spring_HW2.ipynb
new file mode 100644
index 0000000..33c23f8
--- /dev/null
+++ b/2022 ML/02 What to do if my network fails to train/ML2022Spring_HW2.ipynb
@@ -0,0 +1,922 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "OYlaRwNu7ojq"
+ },
+ "source": [
+ "# **Homework 2 Phoneme Classification**\n",
+ "\n",
+ "* Slides: https://docs.google.com/presentation/d/1v6HkBWiJb8WNDcJ9_-2kwVstxUWml87b9CnA16Gdoio/edit?usp=sharing\n",
+ "* Kaggle: https://www.kaggle.com/c/ml2022spring-hw2\n",
+ "* Video: TBA\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "mLQI0mNcmM-O",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "7d5b4d81-9438-4d50-8153-cd235c47ee21"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Wed Feb 23 14:42:18 2022 \n",
+ "+-----------------------------------------------------------------------------+\n",
+ "| NVIDIA-SMI 460.32.03 Driver Version: 460.32.03 CUDA Version: 11.2 |\n",
+ "|-------------------------------+----------------------+----------------------+\n",
+ "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
+ "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n",
+ "| | | MIG M. |\n",
+ "|===============================+======================+======================|\n",
+ "| 0 Tesla K80 Off | 00000000:00:04.0 Off | 0 |\n",
+ "| N/A 30C P8 29W / 149W | 0MiB / 11441MiB | 0% Default |\n",
+ "| | | N/A |\n",
+ "+-------------------------------+----------------------+----------------------+\n",
+ " \n",
+ "+-----------------------------------------------------------------------------+\n",
+ "| Processes: |\n",
+ "| GPU GI CI PID Type Process name GPU Memory |\n",
+ "| ID ID Usage |\n",
+ "|=============================================================================|\n",
+ "| No running processes found |\n",
+ "+-----------------------------------------------------------------------------+\n"
+ ]
+ }
+ ],
+ "source": [
+ "!nvidia-smi"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "KVUGfWTo7_Oj"
+ },
+ "source": [
+ "## Download Data\n",
+ "Download data from google drive, then unzip it.\n",
+ "\n",
+ "You should have\n",
+ "- `libriphone/train_split.txt`\n",
+ "- `libriphone/train_labels`\n",
+ "- `libriphone/test_split.txt`\n",
+ "- `libriphone/feat/train/*.pt`: training feature
\n",
+ "- `libriphone/feat/test/*.pt`: testing feature
\n",
+ "\n",
+ "after running the following block.\n",
+ "\n",
+ "> **Notes: if the google drive link is dead, you can download the data directly from [Kaggle](https://www.kaggle.com/c/ml2022spring-hw2/data) and upload it to the workspace**\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Bj5jYXsD9Ef3"
+ },
+ "source": [
+ "### Download train/test metadata"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "OzkiMEcC3Foq",
+ "outputId": "0c8644b9-8a1e-4d23-de78-bef46c22bb1f"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Requirement already satisfied: gdown in /usr/local/lib/python3.7/dist-packages (4.2.1)\n",
+ "Collecting gdown\n",
+ " Downloading gdown-4.3.1.tar.gz (13 kB)\n",
+ " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+ " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+ " Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from gdown) (4.62.3)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from gdown) (3.6.0)\n",
+ "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from gdown) (4.6.3)\n",
+ "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gdown) (1.15.0)\n",
+ "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.7/dist-packages (from gdown) (2.23.0)\n",
+ "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.24.3)\n",
+ "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2.10)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (2021.10.8)\n",
+ "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (3.0.4)\n",
+ "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown) (1.7.1)\n",
+ "Building wheels for collected packages: gdown\n",
+ " Building wheel for gdown (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
+ " Created wheel for gdown: filename=gdown-4.3.1-py3-none-any.whl size=14493 sha256=3e61ade9bf1f058ddfc642d7504028cd8dfce8c1ae9220b30d487f42c64b58e8\n",
+ " Stored in directory: /root/.cache/pip/wheels/39/13/56/88209f07bace2c1af0614ee3326de4a00aad74afb0f4be921d\n",
+ "Successfully built gdown\n",
+ "Installing collected packages: gdown\n",
+ " Attempting uninstall: gdown\n",
+ " Found existing installation: gdown 4.2.1\n",
+ " Uninstalling gdown-4.2.1:\n",
+ " Successfully uninstalled gdown-4.2.1\n",
+ "Successfully installed gdown-4.3.1\n",
+ "Downloading...\n",
+ "From: https://drive.google.com/uc?id=1o6Ag-G3qItSmYhTheX6DYiuyNzWyHyTc\n",
+ "To: /content/libriphone.zip\n",
+ "100% 479M/479M [00:04<00:00, 101MB/s]\n",
+ "feat test_split.txt train_labels.txt\ttrain_split.txt\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install --upgrade gdown\n",
+ "\n",
+ "# Main link\n",
+ "!gdown --id '1o6Ag-G3qItSmYhTheX6DYiuyNzWyHyTc' --output libriphone.zip\n",
+ "\n",
+ "# Backup link 1\n",
+ "# !gdown --id '1R1uQYi4QpX0tBfUWt2mbZcncdBsJkxeW' --output libriphone.zip\n",
+ "\n",
+ "# Bqckup link 2\n",
+ "# !wget -O libriphone.zip \"https://www.dropbox.com/s/wqww8c5dbrl2ka9/libriphone.zip?dl=1\"\n",
+ "\n",
+ "!unzip -q libriphone.zip\n",
+ "!ls libriphone"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "_L_4anls8Drv"
+ },
+ "source": [
+ "### Preparing Data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "po4N3C-AWuWl"
+ },
+ "source": [
+ "**Helper functions to pre-process the training data from raw MFCC features of each utterance.**\n",
+ "\n",
+ "A phoneme may span several frames and is dependent to past and future frames. \\\n",
+ "Hence we concatenate neighboring phonemes for training to achieve higher accuracy. The **concat_feat** function concatenates past and future k frames (total 2k+1 = n frames), and we predict the center frame.\n",
+ "\n",
+ "Feel free to modify the data preprocess functions, but **do not drop any frame** (if you modify the functions, remember to check that the number of frames are the same as mentioned in the slides)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "IJjLT8em-y9G"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import random\n",
+ "import pandas as pd\n",
+ "import torch\n",
+ "from tqdm import tqdm\n",
+ "\n",
+ "def load_feat(path):\n",
+ " feat = torch.load(path)\n",
+ " return feat\n",
+ "\n",
+ "def shift(x, n):\n",
+ " if n < 0:\n",
+ " left = x[0].repeat(-n, 1)\n",
+ " right = x[:n]\n",
+ "\n",
+ " elif n > 0:\n",
+ " right = x[-1].repeat(n, 1)\n",
+ " left = x[n:]\n",
+ " else:\n",
+ " return x\n",
+ "\n",
+ " return torch.cat((left, right), dim=0)\n",
+ "\n",
+ "def concat_feat(x, concat_n):\n",
+ " assert concat_n % 2 == 1 # n must be odd\n",
+ " if concat_n < 2:\n",
+ " return x\n",
+ " seq_len, feature_dim = x.size(0), x.size(1)\n",
+ " x = x.repeat(1, concat_n) \n",
+ " x = x.view(seq_len, concat_n, feature_dim).permute(1, 0, 2) # concat_n, seq_len, feature_dim\n",
+ " mid = (concat_n // 2)\n",
+ " for r_idx in range(1, mid+1):\n",
+ " x[mid + r_idx, :] = shift(x[mid + r_idx], r_idx)\n",
+ " x[mid - r_idx, :] = shift(x[mid - r_idx], -r_idx)\n",
+ "\n",
+ " return x.permute(1, 0, 2).view(seq_len, concat_n * feature_dim)\n",
+ "\n",
+ "def preprocess_data(split, feat_dir, phone_path, concat_nframes, train_ratio=0.8, train_val_seed=1337):\n",
+ " class_num = 41 # NOTE: pre-computed, should not need change\n",
+ " mode = 'train' if (split == 'train' or split == 'val') else 'test'\n",
+ "\n",
+ " label_dict = {}\n",
+ " if mode != 'test':\n",
+ " phone_file = open(os.path.join(phone_path, f'{mode}_labels.txt')).readlines()\n",
+ "\n",
+ " for line in phone_file:\n",
+ " line = line.strip('\\n').split(' ')\n",
+ " label_dict[line[0]] = [int(p) for p in line[1:]]\n",
+ "\n",
+ " if split == 'train' or split == 'val':\n",
+ " # split training and validation data\n",
+ " usage_list = open(os.path.join(phone_path, 'train_split.txt')).readlines()\n",
+ " random.seed(train_val_seed)\n",
+ " random.shuffle(usage_list)\n",
+ " percent = int(len(usage_list) * train_ratio)\n",
+ " usage_list = usage_list[:percent] if split == 'train' else usage_list[percent:]\n",
+ " elif split == 'test':\n",
+ " usage_list = open(os.path.join(phone_path, 'test_split.txt')).readlines()\n",
+ " else:\n",
+ " raise ValueError('Invalid \\'split\\' argument for dataset: PhoneDataset!')\n",
+ "\n",
+ " usage_list = [line.strip('\\n') for line in usage_list]\n",
+ " print('[Dataset] - # phone classes: ' + str(class_num) + ', number of utterances for ' + split + ': ' + str(len(usage_list)))\n",
+ "\n",
+ " max_len = 3000000\n",
+ " X = torch.empty(max_len, 39 * concat_nframes)\n",
+ " if mode != 'test':\n",
+ " y = torch.empty(max_len, dtype=torch.long)\n",
+ "\n",
+ " idx = 0\n",
+ " for i, fname in tqdm(enumerate(usage_list)):\n",
+ " feat = load_feat(os.path.join(feat_dir, mode, f'{fname}.pt'))\n",
+ " cur_len = len(feat)\n",
+ " feat = concat_feat(feat, concat_nframes)\n",
+ " if mode != 'test':\n",
+ " label = torch.LongTensor(label_dict[fname])\n",
+ "\n",
+ " X[idx: idx + cur_len, :] = feat\n",
+ " if mode != 'test':\n",
+ " y[idx: idx + cur_len] = label\n",
+ "\n",
+ " idx += cur_len\n",
+ "\n",
+ " X = X[:idx, :]\n",
+ " if mode != 'test':\n",
+ " y = y[:idx]\n",
+ "\n",
+ " print(f'[INFO] {split} set')\n",
+ " print(X.shape)\n",
+ " if mode != 'test':\n",
+ " print(y.shape)\n",
+ " return X, y\n",
+ " else:\n",
+ " return X\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "us5XW_x6udZQ"
+ },
+ "source": [
+ "## Define Dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Fjf5EcmJtf4e"
+ },
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "from torch.utils.data import Dataset\n",
+ "from torch.utils.data import DataLoader\n",
+ "\n",
+ "class LibriDataset(Dataset):\n",
+ " def __init__(self, X, y=None):\n",
+ " self.data = X\n",
+ " if y is not None:\n",
+ " self.label = torch.LongTensor(y)\n",
+ " else:\n",
+ " self.label = None\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " if self.label is not None:\n",
+ " return self.data[idx], self.label[idx]\n",
+ " else:\n",
+ " return self.data[idx]\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.data)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IRqKNvNZwe3V"
+ },
+ "source": [
+ "## Define Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Bg-GRd7ywdrL"
+ },
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "import torch.nn.functional as F\n",
+ "\n",
+ "class BasicBlock(nn.Module):\n",
+ " def __init__(self, input_dim, output_dim):\n",
+ " super(BasicBlock, self).__init__()\n",
+ "\n",
+ " self.block = nn.Sequential(\n",
+ " nn.Linear(input_dim, output_dim),\n",
+ " nn.ReLU(),\n",
+ " )\n",
+ "\n",
+ " def forward(self, x):\n",
+ " x = self.block(x)\n",
+ " return x\n",
+ "\n",
+ "\n",
+ "class Classifier(nn.Module):\n",
+ " def __init__(self, input_dim, output_dim=41, hidden_layers=1, hidden_dim=256):\n",
+ " super(Classifier, self).__init__()\n",
+ "\n",
+ " self.fc = nn.Sequential(\n",
+ " BasicBlock(input_dim, hidden_dim),\n",
+ " *[BasicBlock(hidden_dim, hidden_dim) for _ in range(hidden_layers)],\n",
+ " nn.Linear(hidden_dim, output_dim)\n",
+ " )\n",
+ "\n",
+ " def forward(self, x):\n",
+ " x = self.fc(x)\n",
+ " return x"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Hyper-parameters"
+ ],
+ "metadata": {
+ "id": "TlIq8JeqvvHC"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# data prarameters\n",
+ "concat_nframes = 1 # the number of frames to concat with, n must be odd (total 2k+1 = n frames)\n",
+ "train_ratio = 0.8 # the ratio of data used for training, the rest will be used for validation\n",
+ "\n",
+ "# training parameters\n",
+ "seed = 0 # random seed\n",
+ "batch_size = 512 # batch size\n",
+ "num_epoch = 5 # the number of training epoch\n",
+ "learning_rate = 0.0001 # learning rate\n",
+ "model_path = './model.ckpt' # the path where the checkpoint will be saved\n",
+ "\n",
+ "# model parameters\n",
+ "input_dim = 39 * concat_nframes # the input dim of the model, you should not change the value\n",
+ "hidden_layers = 1 # the number of hidden layers\n",
+ "hidden_dim = 256 # the hidden dim"
+ ],
+ "metadata": {
+ "id": "iIHn79Iav1ri"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Prepare dataset and model"
+ ],
+ "metadata": {
+ "id": "IIUFRgG5yoDn"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import gc\n",
+ "\n",
+ "# preprocess data\n",
+ "train_X, train_y = preprocess_data(split='train', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes, train_ratio=train_ratio)\n",
+ "val_X, val_y = preprocess_data(split='val', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes, train_ratio=train_ratio)\n",
+ "\n",
+ "# get dataset\n",
+ "train_set = LibriDataset(train_X, train_y)\n",
+ "val_set = LibriDataset(val_X, val_y)\n",
+ "\n",
+ "# remove raw feature to save memory\n",
+ "del train_X, train_y, val_X, val_y\n",
+ "gc.collect()\n",
+ "\n",
+ "# get dataloader\n",
+ "train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)\n",
+ "val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False)"
+ ],
+ "metadata": {
+ "id": "c1zI3v5jyrDn",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "7fd90470-ef44-404a-a043-cb8e83fd1801"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[Dataset] - # phone classes: 41, number of utterances for train: 3428\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "3428it [00:01, 2436.46it/s]\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[INFO] train set\n",
+ "torch.Size([2116368, 39])\n",
+ "torch.Size([2116368])\n",
+ "[Dataset] - # phone classes: 41, number of utterances for val: 858\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "858it [00:00, 2322.86it/s]"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[INFO] val set\n",
+ "torch.Size([527790, 39])\n",
+ "torch.Size([527790])\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "CfRUEgC0GxUV",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "75dcb672-a97d-43ff-b0f1-cb23d27fe65a"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "DEVICE: cuda:0\n"
+ ]
+ }
+ ],
+ "source": [
+ "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
+ "print(f'DEVICE: {device}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "88xPiUnm0tAd"
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "#fix seed\n",
+ "def same_seeds(seed):\n",
+ " torch.manual_seed(seed)\n",
+ " if torch.cuda.is_available():\n",
+ " torch.cuda.manual_seed(seed)\n",
+ " torch.cuda.manual_seed_all(seed) \n",
+ " np.random.seed(seed) \n",
+ " torch.backends.cudnn.benchmark = False\n",
+ " torch.backends.cudnn.deterministic = True"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "QTp3ZXg1yO9Y"
+ },
+ "outputs": [],
+ "source": [
+ "# fix random seed\n",
+ "same_seeds(seed)\n",
+ "\n",
+ "# create model, define a loss function, and optimizer\n",
+ "model = Classifier(input_dim=input_dim, hidden_layers=hidden_layers, hidden_dim=hidden_dim).to(device)\n",
+ "criterion = nn.CrossEntropyLoss() \n",
+ "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Training"
+ ],
+ "metadata": {
+ "id": "pwWH1KIqzxEr"
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "CdMWsBs7zzNs",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "cfb0046d-52a5-4a90-c073-f4091e8b230e"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 4134/4134 [00:23<00:00, 178.15it/s]\n",
+ "100%|██████████| 1031/1031 [00:03<00:00, 286.39it/s]\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[001/005] Train Acc: 0.335711 Loss: 2.508261 | Val Acc: 0.401501 loss: 2.131011\n",
+ "saving model with acc 0.402\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 4134/4134 [00:23<00:00, 175.35it/s]\n",
+ "100%|██████████| 1031/1031 [00:03<00:00, 281.23it/s]\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[002/005] Train Acc: 0.380122 Loss: 2.270708 | Val Acc: 0.409386 loss: 2.085472\n",
+ "saving model with acc 0.409\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 4134/4134 [00:23<00:00, 174.85it/s]\n",
+ "100%|██████████| 1031/1031 [00:03<00:00, 271.75it/s]\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[003/005] Train Acc: 0.388530 Loss: 2.227409 | Val Acc: 0.414295 loss: 2.063263\n",
+ "saving model with acc 0.414\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 4134/4134 [00:23<00:00, 177.08it/s]\n",
+ "100%|██████████| 1031/1031 [00:03<00:00, 276.20it/s]\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[004/005] Train Acc: 0.393600 Loss: 2.201657 | Val Acc: 0.417833 loss: 2.046759\n",
+ "saving model with acc 0.418\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 4134/4134 [00:23<00:00, 177.41it/s]\n",
+ "100%|██████████| 1031/1031 [00:03<00:00, 280.50it/s]"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[005/005] Train Acc: 0.397076 Loss: 2.184807 | Val Acc: 0.420696 loss: 2.035235\n",
+ "saving model with acc 0.421\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "best_acc = 0.0\n",
+ "for epoch in range(num_epoch):\n",
+ " train_acc = 0.0\n",
+ " train_loss = 0.0\n",
+ " val_acc = 0.0\n",
+ " val_loss = 0.0\n",
+ " \n",
+ " # training\n",
+ " model.train() # set the model to training mode\n",
+ " for i, batch in enumerate(tqdm(train_loader)):\n",
+ " features, labels = batch\n",
+ " features = features.to(device)\n",
+ " labels = labels.to(device)\n",
+ " \n",
+ " optimizer.zero_grad() \n",
+ " outputs = model(features) \n",
+ " \n",
+ " loss = criterion(outputs, labels)\n",
+ " loss.backward() \n",
+ " optimizer.step() \n",
+ " \n",
+ " _, train_pred = torch.max(outputs, 1) # get the index of the class with the highest probability\n",
+ " train_acc += (train_pred.detach() == labels.detach()).sum().item()\n",
+ " train_loss += loss.item()\n",
+ " \n",
+ " # validation\n",
+ " if len(val_set) > 0:\n",
+ " model.eval() # set the model to evaluation mode\n",
+ " with torch.no_grad():\n",
+ " for i, batch in enumerate(tqdm(val_loader)):\n",
+ " features, labels = batch\n",
+ " features = features.to(device)\n",
+ " labels = labels.to(device)\n",
+ " outputs = model(features)\n",
+ " \n",
+ " loss = criterion(outputs, labels) \n",
+ " \n",
+ " _, val_pred = torch.max(outputs, 1) \n",
+ " val_acc += (val_pred.cpu() == labels.cpu()).sum().item() # get the index of the class with the highest probability\n",
+ " val_loss += loss.item()\n",
+ "\n",
+ " print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}'.format(\n",
+ " epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader), val_acc/len(val_set), val_loss/len(val_loader)\n",
+ " ))\n",
+ "\n",
+ " # if the model improves, save a checkpoint at this epoch\n",
+ " if val_acc > best_acc:\n",
+ " best_acc = val_acc\n",
+ " torch.save(model.state_dict(), model_path)\n",
+ " print('saving model with acc {:.3f}'.format(best_acc/len(val_set)))\n",
+ " else:\n",
+ " print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f}'.format(\n",
+ " epoch + 1, num_epoch, train_acc/len(train_set), train_loss/len(train_loader)\n",
+ " ))\n",
+ "\n",
+ "# if not validating, save the last epoch\n",
+ "if len(val_set) == 0:\n",
+ " torch.save(model.state_dict(), model_path)\n",
+ " print('saving model at last epoch')\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ab33MxosWLmG",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "7ea97815-15cc-4afa-fa7e-b65460b91640"
+ },
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ "50"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 12
+ }
+ ],
+ "source": [
+ "del train_loader, val_loader\n",
+ "gc.collect()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "1Hi7jTn3PX-m"
+ },
+ "source": [
+ "## Testing\n",
+ "Create a testing dataset, and load model from the saved checkpoint."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "VOG1Ou0PGrhc",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "81077fbc-a6ea-46b7-9a57-a690480fbb6b"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[Dataset] - # phone classes: 41, number of utterances for test: 1078\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "1078it [00:00, 2784.86it/s]"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[INFO] test set\n",
+ "torch.Size([646268, 39])\n"
+ ]
+ },
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# load data\n",
+ "test_X = preprocess_data(split='test', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes)\n",
+ "test_set = LibriDataset(test_X, None)\n",
+ "test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ay0Fu8Ovkdad",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "a29d1dbc-3222-4cec-8f84-04475b77cceb"
+ },
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "execution_count": 14
+ }
+ ],
+ "source": [
+ "# load model\n",
+ "model = Classifier(input_dim=input_dim, hidden_layers=hidden_layers, hidden_dim=hidden_dim).to(device)\n",
+ "model.load_state_dict(torch.load(model_path))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zp-DV1p4r7Nz"
+ },
+ "source": [
+ "Make prediction."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "84HU5GGjPqR0",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "bbaaa8c5-d88c-4ef3-f7be-d75b208cd5df"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stderr",
+ "text": [
+ "100%|██████████| 1263/1263 [00:02<00:00, 426.93it/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "test_acc = 0.0\n",
+ "test_lengths = 0\n",
+ "pred = np.array([], dtype=np.int32)\n",
+ "\n",
+ "model.eval()\n",
+ "with torch.no_grad():\n",
+ " for i, batch in enumerate(tqdm(test_loader)):\n",
+ " features = batch\n",
+ " features = features.to(device)\n",
+ "\n",
+ " outputs = model(features)\n",
+ "\n",
+ " _, test_pred = torch.max(outputs, 1) # get the index of the class with the highest probability\n",
+ " pred = np.concatenate((pred, test_pred.cpu().numpy()), axis=0)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "wyZqy40Prz0v"
+ },
+ "source": [
+ "Write prediction to a CSV file.\n",
+ "\n",
+ "After finish running this block, download the file `prediction.csv` from the files section on the left-hand side and submit it to Kaggle."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "GuljYSPHcZir"
+ },
+ "outputs": [],
+ "source": [
+ "with open('prediction.csv', 'w') as f:\n",
+ " f.write('Id,Class\\n')\n",
+ " for i, y in enumerate(pred):\n",
+ " f.write('{},{}\\n'.format(i, y))"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "collapsed_sections": [],
+ "name": "ML2022Spring - HW2.ipynb",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/2022 ML/02 What to do if my network fails to train/hw2_slides 2022.pdf b/2022 ML/02 What to do if my network fails to train/hw2_slides 2022.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..b562358c19e0618bc4dbdf47c69ff55e097d7e95
GIT binary patch
literal 828027
zcmdRVcRXBQyYGmWj39#OBuccz=$#3I=pjnS$jQet#4V+8f^FPD%|Gd5d^bcpPTO;01JY7
z!A_5D!IF|-h_<)0B^V-S=5A*1WDSO>n>kpzo!?M4dusN;+||a}9V`Hp*8)pRgCY0L
ztSwc|9$VV~Dx>b<;087X?w-~301UYc{B{TPofS1cFR$U@Zg1lV)X0Bc#rfymfC9hk
zvvjmLYljeU{zrS{TrJJqom{~j*i(=q$O(8^gX}?;oIp!$-0dyTuRj5Kg7`tEAUEKS
z4afpy3B0=wT;T-{e$KN8|E^g0Zx#Ov^elk0FkCGi&l-Bx4v3nig^ii4lh=79ykHT2
zUa%mq5D)=G&dJdoDCh9x-LKV-tGvsBrHZ73{p41s~o1vrz
z$}!8F#rBsI#aD2?*S_1cU@*IQmEic1>n8e6S&$$6-4pdr;=%Q2g;+!pr1bNs2idDJ
zFG69IHr0>vA=PwTyZzAK{(?v<~K9QO4u9O)}F8h_a+aY6aDx_J<1_LwHX
z$S&%udH9@sf1f1~Y$0=cR?#7s-L@QuBcWS-Tl%A)
z!eY`h{fdk`5gLFYb|lq
z#d)38^r^}w8Uzp~pD32yU9!W~gJV2(Q3_E%j%h9KKi23xc+stEj`3vQ`YyaJxu{x&
zoYI{hyphoTpdveUudFQ}x1ybGkm3iC|5NYqjr2f7ts}7o4rUP5(LMw0MHp
zi0a+(e1pgK3wL~(Z*gvX3H-{z7B+|A|3u-$S+|_-;&(7*Sq87$dd{P!mw-P=UFc@H
zBTyhIJ)!0o8IT@0jidE#ce6XzPEOYLmeyy4;>Kg{~qFse-uyv`0l*XXK&6z2JQg_d3GiM<~x&(KV-liU@6D5
zqQIjd4>ca!p7qx6Q+Y3U#Ru*Hf1Ot)WAFULj2F!Jn{3Zd`M~Gx`dyi;llcS7v%ZD^
zItB2&rI$MxqT~QDwe0V|a=-s70pb5Gw!iDfe~aGv&V&L8^e=wr`_1@&5d1Ihh5XkD
z{=E7>`}p55Jl`)_0Ydzf;wyACoyhqv3$Kx5Sw4kM(zC1cz~4Klex#+C)MfONd0!|y
z`93J^D`{Kj?b`K&tcX5g<;`j;Si`nkKV@PVZ$!|r3#)CT$vilhnZIvDmIumbv@X|a
zlzDJfa&fcEv5B>b%{nwl9%V8^??hqoGNd>GN@aSn*niRBTV1`6)*-&h*1T%_bs-&L
zN{tGirqM6VYsWJ$9;bXEvHxysTaGvmBJt)3Ym6$WP${)}h&aPA!
z9;#(l*0eUqGKnW(UU5DaGY|Bbg8cY``czQ
z-xpP5-0W)_Lun3G;P{rDglH;%<{v@bRt}zO8&~j5IEweWH*?1>Nsj10CP&ug45l`R
zP>ebX4SzDp^|>EHxfy2CToRVy+5BCMQT{@4k57TBdfeLUO3Uq)DZOBL`MZI*Bgd{=
zvHAN>q8(nE@P>naB$4H(fU$?m@eb^hN@^e?FM|7N-0Q0G5m6@Ux=YfwM0{{IB({C~3PpHTmx<)DJgcbNxy
zyU~KZNM#@!!bF*Jomn-HyoEHwK4c<02@A7-mMTIjfB&m-YL;4RNEY4wt2f50@d?C^
zLn!S*Y#HETfkanx>Kv7yg?9{}UQ_WNAbhc*NFfBd
zah}t``ruaS?igZ&%t3ug)QjZ)$hYLB-+2krI@{iaM%R^H
zUTVD>$87FKrAGge&3yQUL~gc*80$#(=g-o6O7W}Z+fRRHmUi&me|bloUm?J{Q(?Wl
z3vL-ANkM*)Bsz&cA{?6S2&4
z?RdXnak*90DvxiWT+U9?qT1aWS51*Kr=UtMt>LBH#Lyxj8hT36)};JYeSP~)1lKE#
zck>CCD%-co>Jo^XH{y==f`A@xcte}dbb
zHTe!sa{g+oz@P2kr9;P{V3^8c@^7WjWAwF3VlwF3W?T7mzT)Sg%WCt3d+S`qkHsjW9uahte%
zjTyQy2=Zb7d0%T3LZOgEzB!lJ4_3Ps(aguQ^Zre#={_m`KO7h3AB
z7aA+2JC=D7RAqTCPm
z%B$|R@MKzP3T2Oe*
z-%q)H;OO!8%+WMfHeH7}0Gll8H{h9i$4Rb`VX|tj91%G%GN)K#SX28|v3;!YFn@1t
zy!Jv;e#)A^hmoMdokB6cMR8$yVag1>pWoG}Y0d5EWtmLV&Y
z^)b2G!}nL@3g^;zd}3UM3J@7^v*wk^KHZ78U(b>BB3
zaPE0z-^SiY{6`tHdnC_LvyJ-^7-3&WwXV0>V9H5mb_2_$__`n0zq}>)DyYcd5UAHg
zuJSCt#;l#Jee{u8N*!>?@zIr1@}B3IMWDhXLqe;0aS=Y~
zwB6>L0T`P>CO}5;Mo&jol`g7+NA`wK5nV@K1FLi<-Bwog&-A_7QD1SfpBaS?C_}Z%
zE4J^Gm%GNgTlo7Oc;&jFEGlWSaT>7)>
z#rIr-nEZ&CJVhEgnzDiresT6yBd^ex)&jh}Hr1)e@D+?R@sVI~8@0rYw0dBFu=D{#
zhuHVcu9bprfm!)$OOK?>?qBsD6-A4Wkr{8i%)6&9!Ijt~HO6R1V}Vz8Xz8v#RQgrZk531}u$h71K_%)fge%)8o2jnb`yPA7;Mce2ZPA4-FU&YYIBV
zKPl$)uvHr33A{n-XbrNn3!3Ri3nN`>^}l8pnh$J%>jto9O4MJJbw%wOpf&(KL(6%Z%Sv3gntz
z6o2}pD2XmNd?yO>=E!?yH!O)ikV95Vk`j02rtiIuo|Gstua8abg8oBp)z>x&*;_*B
zo?c02i>he+DflHs%&x_uN6Hmd=DhVm-n~@4`9^v=50thtPoaJ_WUVP9v|eI8zLtNb
z$&2JRKi<_3woiRO$6q$Dy_2+soHH2{W%+tz%KmVUk%K+gSvJq^a*g_?4Cso2+!a+p
zzwN%E{HjLklGA63{T`|<#E@4>7kQq0M0{nyyPkx3`N7ZrbzwTnX>Fkdgvfs-K|KA6
z$y6EPGdYt+AE~0lpi;g=
znwp>f>W-fxhxf=k4@~5yL&TpbxGH8jCQRJf`QlcF2{+nbSR|LKQaOkZ?i#s?0m=Bd
zUpBDlSgnq>L()g5uNrL81I-r^~j1~a-;nX4EK&1x1Jpm8M9HA
zyXtM(yM8ic{=n75vyR-ZBlO>)su=3yZ$XB}SZIE+0=3ZqQ8<{2N`y6%H
z7~U%CuK>qr6S1fTs}q`E|-;TfJhW-py7ENXrrXgjY?zuKFIXYoYXR`G*ltN~j}hPqT{{JXSV
zw)?bM_^lMhaoYZOU+P4?Pk8Bek@IIcAY}?-PIMm~-liMgii+N)dN(uYldIb#LX?L7
zF#C8QEKA|o+V$@AfnoBdkK5y3?Q0x`$@-iA{TH6@^iogCsTB5;7DwPi=ysc0ajbdG-{F#om$75MwA?$4C;c{UVCf&$x7uFg(pdqZG%
zR}WxoNYBOs*clQ6GK`RWmNwQ;fK4q?!LyAkdneZi&SvI7k-v7GAhI^@Zuc!+<(wRx
zogB}0z(j%M?r-OGS5Ess>^A`x_iWYoS2pr=^mGPvMOi^f0fch_1i}Gcpwn@X9Ebo9
zp8y|^fB>I>kdWXa_zD<&=@OWNg!J+iDhg_9DvE2@Xc^d;Xz5t!uU)%->pII#c1|u%
z8m8O4w>fy(I5;`ZD#0NnBm`dslY_zJ9JJSHIsUJIPQQT2E`q8r;NjvhgD#Na;F95-
zwt$#`cH-ll|A2n~;9S53+Dmxx5|{`mP;mux0S6Zs_!vGu9v)EI54aD)Bg4OXgHMKl
zT+@t@*@c2XAUgdbi)={~rB>e#tH5K|z)Rq3RMa%IY&Y3CIJpFcghfQf#O3bFD=6Mm
zQhuPVqpSB&-@x3$(#raYjjfxz$5T%)Z=awSFM~r~g@(n%zKM%Zc>69fBQxtmcFxD#
zywbAripr|$n$KUGTUy)NJ37Dh4-5_skBt5to1L3qSX^3OSzSZy?(H8O9wCoWXK~?x
zaDRmb{Qec#e~60=i0c9#9xfi?SzI_5Jb?q33=jVXAHh`_O+qsla%TR3ixjfa=_O5<
zSOm0oC?C7_fv>R&&axrSLOYM_|17Y;|59Xs2KHB6W1!2pIKbfHl7XNgRCXB0`wRcW
zj>}Hu>zT5FV{-OKweqlhMdin
zrf5wto#STTi)FT(vEsc*^=a($fm6`*{>Lrp>vUeMiE50_(YaPFoj>#-i8+N#$k?e<
z&FFZ6!-F+`xk|@RVb#8X$1z3a7FqdA#Rmb&I$=Es5%lZC9E?xrDd>IT
zE|&;(W#pKRvW@hT2Aa+36m;h&YkgQKn&okakLW(g_HZ47$6e=`Z
zA<$`^K&K53OLLs4VJgZ-JE>AnaNkipch+$F;Ch_9Uv>&2pM+p*(=a%zpLxrEC}$N_
zNL?!vn$))8y|YB{?n7)BxPVYX#X9ZCPQ50L0oLA-bP6&xJyBZk&SwV(=$!<2dfHzt
zQvBpSQrHY$2>S|Mq4h9GAn>@S6KRz4E?ol#4Fp^c*Zete-N3-4;5NasJ6{Q2V1f3obleA{_Gb7o!Tp_K%_@QEUkMr<}r)aeH)o*m+C$#y%yc`GS2QBOb8`q1B#VX#mTPMYa
z8RR5xMSy*CEBZWn-QuQl^#5MY^De^kFtw2Q^=kikd0ux>KR4+cYkEE6#QvQ$bOg#1
z;|6rL3`LG?;%giCJ$x}^Tc4K}>th}LCO@UW@6|1Sq70mWLbHBQXuDA7atdPltbwV(
zs+cG+J5r2CqWrDXg9B6KX&j0dRQ`sVa@behQ_wzf0v22N$SQn<>NkKwegkMD6|x&u
zm*}2OsKxdmY%Ra*fCU=_`2iC@0IxJ_zZuu^96u0qaZgv1Z#6i=Ty$c;R#~d5{YfU7
zS`o0743t-^RGW7Ho(8)>1#g<{mr)P-}WjI|aQe5Z`?y
z{!i}eFPJ8Wb=2g+MhenUmx>9Bn;>$1ndp0{%#a?M?4|IL1h?@|8zaUujmIFM4~&7n
z=qo&H#*QiaH3FbZo5p=RU8K_-I&H~AXDIfC
z-L2lakluYU=X%AZ@PQ}A!pr)Z?_uqWy%;x*rJAD
z-$OuKV33X(v^H$+LlU+=1DLm|Wo`!2x6ata^f#My_4pA(UovVp+=L35D?<@d{imSy
znzJdMXH~FDh-Du$#dtzdxH>A{>ozh2fOeOK2G6{3-?E$>uoU1wV}l|f6e`3C!!sw
zmM!}3yG+6ElfE>c)$9ADAxyj6>=@q(*vfKIY7IR26tv3?bh;@O6H077ru`Aj1x>Wz6)t5yp9msr?wz6CI3{hAl`4S|!Q5Ost6b
zLQg^0-%11J(rJGH%t#7+;3jE4Y(=4*d%6jz2H;dM8qQw@^u*UNj-puJj4%$+zd3Nj
zkm&$k5*5!p0VSJ@e$l`j9PPczbby5IYsa-%)vH2xMh*s-(puV=@@$u}Y+IM-JLNK8fDo9Ma)IG4g{|DI12^K;
zlPbUwENhXZKuCb*Yi^VDJT(CW75YO^>mt|)+qpgCw7|v%lh@eKJ*S{dgQ~BmAgYB^
zkX$Ss8j^@n8(V-znxX)aMv)aYR0HFX@CSUne9uZ<8uXwT-Y4PpUsV625p$Px`uX!7|}DcuLC*QXzi_Pe+d*B{YCh6$^mJ
zS`mtX{s|snVy!gpGW~g;XG*khQP{WY(d~8g7ha5|T(Q!?!__3P4ULCMBcY}<*7jKP
z1b{vO%1(8Qwk5<|SpKD*>wtFFCEbUvP(crR8smJvVfyxqk^%Mf&n~aBQv3%>!p>0g
z0Fi(Wqg}>$^#U_}tnP!xH|e-7Giky#HPctI=KwXn%lLX{94r42fc)J2DM+E-gR%=P
z1VpF>K_P+u>nk>v_tpVM5~6#Kom5UyM0VK$z^C%J@AnuZ0GSIL_zk79J(FaAY80en
z`B}+vp(#N!vRUI4ROAL*VJ_J-z+MJw{o0JJ0mkipn2t2zN|Pg8w@9u%4RQPdHg395
zF*3q5KaD);hiw+`8tm}FzQQ0qj73DquH|C^!e0}@W#h^kA%^4}HbV=jfU_YzgH5Z)
zBkUP$f_RbUdl<0zGCI)lOq6;@WQt%NH_t({>AVCz2hGbf(1?dQP`0=;5CV}bkP{9y
z?z0$Brp%s#j`3s;oG=%MBm786F;i(sy#;TNMj(0$O?mfsFDW$d;QyP0Ua=K|-0O#~
z?*R0b%N^3=$wUdA?*gcx_>63_ipYi5W)rreyB^G$@97AcpMtK|Hlh}$zZF>J+2&x`
z5?}|NjdAYswy$D;9F6^3tx3(_QTeg|1gJs!;~t6=u%QbRM;qd`rg#A8>!~mvfTK{T
zaG@!hZi4Re08clz0!A~!nA-GAY+TJ7g8;FqHS&hm{YJ>ABGI%dz&{uRN+K&1Z&N1?
zOYaO?<9X5}B4|~YpAwuC0-N7j#*|4^bky4
zyOJku&|Usc0M&i46%BowLMOn&b+Kcf$pgG3Wxmu9lkf%33?pYqtjb_QeiUp(ZNT;k
z^7Qi)X@JQT0yLqh9q_ZZxxSFTP)E3WktuMBSk3|Das)fzv&J2d1je1w0EY2{ZHg3=
zH17Nye=uv!E`Qd}_2d>kp!7W8l8nZW^$Lq8nUr_A_pm{~FBF|!So^vSNmz4G$&H?e
z98(DOq7Z4oe6H{oPkZCXJ^UXG?w_^jD=6r=Th%CghG!fznVfVf8f07;o1+6XpRG3@^dm?Iww
zCPwn{w-o#Q6$N|%(7^GTkOlV8_2fp>#RApg-aryxbKuiPi+~)v4;&kfBS8{
z39WVPpOg01m-iU^lEDotBLdwz1sN>68D9g&_j*#=5v?eQ7aImvR>M!rQ48Q(tzwN#$3EOPgB
zDge)Lb-eftPM6r9Xgxc)7IlFktXNW~d~>iQC`Y}`zQdNTmO<~9RArUJUXs0_B!
z24Hs5iv=s+0b4JaM)Bq6Ttb`1Vn}YmcKq#jtqxoYv88_q&bR8eQ;;D9;|>D|M89(9
z7#qS!x=AE|6Ht5+2{ls^So=`#Nho@(Dj&BQhB$D;&cTEWQK-vhn8`E6apXQz97gYX
z(_|%9UPp3%BMc(E&5*vtZ^hT?SUnp+Imwl
zTG(?Cso!B0KH#Qc*HhCi?+g1;1YJ*paUC207R?0e2S84b=}dQ?T{zR7x|`>^wVMk(~jk2mlZRA^-r;_-_D}?irZyLlK5{Ob=oETEn__!`by6
zfaolqiB2Q#8Gc~D@Iybm96*ZJ)rB5>GsWO(<<)ok@J6m2jLrLBhqVv?hL5F@bqfTE
z`XxHYb*4LMU3-cp0a)>4tm?9Wf(<~dixY<%^8TyLHaly;*<2(|1TH|DiO+
zQawJMnOzCS{$oVz6P!CNG
zCgubiP<0AI0=8gQ@h7Blg${NKdftnroU`>~n4hq+tJ{I1Uq=B`TfcGNT#xurWgA5>
z4YU`4`Ag5#%fSyNp%*kLqLmtCwdA?J&`^g5Z5lt6I5AB)rq>TLX;&QCx0-Z`z*hc&
z&hOR7kE+R4u&EV5TjZ<8U>8qbapYF
zNxMg1&TJzR1#sb+)?v)NTsgI##mCT?<7*SI=t-s-OcrN$kw5f+L3>iIe&N?+9(GKV
z_8Y&S_KXQ53V9X~mtu${ax;ZjMC6G?J0Q_fQmtkfV4;8wNDRT^LXmm{ns(8)UPF-X
zjy*lJBy28Y0J>gsz6$gETQS{`5c_S#SAxqiUk^q5WxIFpe0y(i*2yxJ=iplJLr%BH
zGGG^n3<1DO`%52-@@n8rGLk62w4XDHt_gJS9?LEeM+yXN$$L?rf`WTxbH0AR83sEa
zfRC{A&SkP_0La8K{nv0T=_glO+D#+((y%pEyTy-9KTF3g=m{IVh~LdvO$=VrB18$G
z{J2N-fVG{+%(ge+!fd5f;r~e6v<1xQl_QHd*>@^SE??|dVN33K6~q=Q3&O1_rn^R)
zN{CiEFh%3WqPbU`Z@WUZ?0oNfX9}^83IQax)`%ozM{53kE{=OlwCryOLw7l30Kc+)
zwx~ANaf6cFUas8vQ0IAH1+T%rXYt!hYFrkwDDM@!?Bc792+Ota-(X-#>$VU^kzwgy
z;+s6Y9APg+fQZi(6t_h0bS_fF&{=!fE4-fxSO+ZeKQYoOT*Fc)*k!8CViNll2z`|nb}
zWcP8HENg)0nvm7wIja@V7%KR5Gm{sTy?AG6n+LinTjpt0e?$W%cpl6z=
zp#HdYU)=-3jGBYV|7>+YIoC+uz-KE>iY!U@QXf|-uc;WsFY#V@O449o5JEj4ZdIT6
zGOyaB-KeVBqlMlsC7m*T>wK}}pg6g4R@v7U
zScwFtA;s$f*XUmS5yhm}XNv196*Q+2iPn2Ii(zT|{6?fQTIm+tBX6R>f+dfha>6a1
zA%C=vVdwSSRMK2MMyFcx}X$hD8@Y!R|WYb+SO09&Me?pP{R6t3DhV
zQDw(s(Eh+deCiaWS`)VRZPv3@
zP_4$;Po|gzQx08-8-uL)JZPp*eeY46@p0vC>gNA`>85l0>9E}<`Tg?p
zgwE>Jse2g-toBw~xE8Lw>RwrsacyxwpN8!~f7Z@ILK_(De1?`^@oYrYPY_*|a}ba|
zB2HQ6;tPm^P6=(tsa*x8^o@Tk6wSOcTc?j~96o6+dE*n6t<6J
z5a+7vKt|>M7C;iqKRop@`3xaCg)|&GH3-}PV7LDHpBHQ(@P$@L$P(NOIgMau3U0U}
z9{4puJf-XSn(mf!0kYGoT|(Q{Wh5Xv8BV;@7>pG}YYfD;=rKa7e{4A7N)_0-u~ogz
zNhfO|>|E*Tq~9e`GaY|Ar+_dGb_ng({MpG|e4>hkqQE1D<_NjD#lpDyIw+BHD}@~`
zUf`qmH~|5v_+(}3s<}`4#m=%(>WC2_h4bnu_8Rhot5f;UZ`W($n&0+?5{auk*B2NR
zVIfS%W6EgYn4YwO5u-I=IqE-53~dIwF6Rz0+>cV6H)Z!VsY0e9WRFymGt=zpd@o=9`L5kncZgo41h~JkS!92-f7?7dLdfye-o1Fz}zy`A#pHQAyy$ydS
zNs6$WgN5x^``(xhN!&9^v`#eE(kenU?}l3xjSjlSi?KW764E5JX`-!>3WuMka_8)l
zPQ*T3{0?E<0!=;@zeh0f+#7dArqzdOL&*t@WH+rQQ{Z9JBr8u>n9C5lp|t*7KT@i<
z&=XdUVnLH47!Qcc?=-Wd@69hIPxIt=T(3!q%$bF~&P4PExEEhhyz+7@llfslN2ENP
z5vhF;++Huh>txO}QqmKly!>VQ@)opgrXqSKe$ij?amdwsTQVGBUUc5l#~{z?Qg(E7
zk|_!e5cH4yW;DxOE%ZfWxoPOa2n=Q>cyz75$0_u(3hQg;#s*$ooQ~x{1!5(3P4$h0dDlOGVkDd8TgI=ijtEjy}moVV}4Vox>H*d^}@~|j9ikF~>5xM_u&C%$J%-Y3GQ`6^XTi!D`%ABs~
z(ea~(`NiI!+jEU!itwx*2>e6h++yOrjF=fx@k^O_Z>*TSt`v55eiyqwGU&0(6iP;Z
zI8;8G?&j&(j!Sd%r+0BNSL{+?lu*~rz%Y)iFUaLX^;1yH;+zI*_cIXItMv`r|LmF_
z(s>{Aa1qfp2&jgv7$huJsB(ue(5YhmBl_qySrfsAcWKGjB0q_=M|MEgfrz~=Eu48-
zf1plLIw2=3Jo@uwrzMV5-?i&1<^i#3xnf1@hf~!aJ2}CHArmo&N~BW5k|fX>t9sZ>
z;hSqrmwefX3m%J(s|9O;*0$0|vA0^Vf*HjZGZk8ze
z3p&5s9|Y_Cfo#TX0z|B=TWdOQj#c{p`V?fQ1GrHBL1-Wp(hU&Fh}^5bI!%Pry!>ZF
zFZG2LAGL74W~uElT5`cA1r^o&>Jl-Bk04O^0p8+D8igcmC#c;a1AXUn{ZPRbt0&e8
zy`BmurOn)~-W%%TfjIPjd^^Gy9k5rBO1I}C*&M_s{)Sxnf
z0%ep_>|_#HL)HAk^cP5&|E&^a=qczT6MU(h}D$H%XU;`%c4F;fw3SL$Pc88EbB{y
zkgpMPiaWZYW|_XiC%P&4S$0m&QC*?2G^9;yKRcy5KSrdCuV|MmvGBua(Q3#Wb6s>$
zdQ+w1Cn7Y>k`128&m(&1!$m@j;BnQ<6SSx{i4~`EoyJe^EUQwddxN#bE*I3sh0Mql
zNBi?kqaPsOW&X4ogE!YEi56O)$adnFzw8{-qhSx|?ET@R*Pljw%c;Eg#b@HkLR`aX
zLz_^F624AZa%yR0+mkpCd`p7LuO7vZ0@6QJ?R
zn|!fNx7uxAvPRB3wIwpOlj7Lqv>Jqj6>XWx(t1=idJkWY;zx-744?6RAFy9YI_f_~
zHR7w|+@~)m{b)_+T^;CGerKNraK*_LAQ6!R&=3YlOUe&BwQUEd3jDybHMT%Qx^7Q|
zfAx$6F+O3vR8m57veNvib0OS3(XBMGq2(*OXgo%31{&FLW%q!XskL|R!Uxs(7!%2s
zy$&MLCR1mbeaiizuFs?2PC-v)Xe6JC7AO1SpJ*7W`6gP!_v8DEg1j}n9jV(QejGw166G4(uxNdnt}6+y7S_8=u!oKrJd)>QGD
z@p>^TK}7YPSL&rLwhu;ATu0~UGz-7T41L9OMY8P89&xoJnWng4Nb&XOd`-Rd&<()B
z+|ZzcjnFw^z{;IFD8*v`pPq!8cfK;H&&J)PW^E?Fh}h@I^5Yy&)Px_rk1c%@@ll%2
z*Jr3OR;L>AW>}A(BX+|nUyVF+h9biElV90EKGQ{i0c5@cVi(Ch=YM=N#>KX*JFIm4
zn@c&L2?HGqaqko%tKJYLqH*(`V?6X~IfLv7-(=V0g75X7tP$lHZwx+}KgyUfXtB<4
z4gOL3FW=NXd=x+CzT3>@p{UJDwlA~`55V5`)j*gekiL-Q-7~R^Rozp`Q`VUh_16rK
z8>fwQZD-vAJ(`DV`kFkRz9{*yXKFeX=Dpe^L5DJT?Y=$>W>lk
zw0vf+nZw=ImAR*E4#|CP!X*)JU^G~=`l6~luf!$ncNMT{e3aH_5|-a%b)
zF@1oBAl&`Mc6j3EF3w%4AGA=pB1YDcl^?Jf<+2&J*l4q_h-8|RTBCas5UeP-+}>k>
z?MT~IaDc7v&I9`tIaD63C=%sizh0YSVI%^<=1=D(X)-QPkE&Qml~304`!U-2ltKC9
zhIkAiO^=V7@5>WMBf(Uzoz~pm1cs~_l4oVly?{IiIN|A_o>u12A-9*_9ppTMRMnK+
z&?|eZZOSxt(yB3LP#tG2TA|h@Z5u>dT`Yxcotvh%UCl;%&Qa#V_4`arr^WHTb=BTZ
zRxR2)%q9eq`sZv*^YHE1
zolFBY%4T50&49kqe>K-KuX
ze`j8+?%6v@$xrvtrQ=2_SN$&*mivRWB8Eop`237sYh?1px6adFJ@k1qRyfqX3cALd
zpnc1ky4FEU40~0h!o86Owh-&pjBV0`qHgDfylf!xB5I~`nOyZ}s1$MegIvX9(
zVL5eAi&PAxXH&2mpEw-roo%b>JuDI}NZwvuv~L#3e~~3b=P6&0!YiS&m<*(L%_8d{3{v;s{oy{%t(K~D!n};7Xe|ULs`U@t!
zU&PU;w2;H+au9<98C1=rdKiDJ;$ZA;G|dmE`MOU?*fS>&I!C3uWU75>IuLfkiN_7`
zJ5-T^qztRKKYBUt8{Lg4efU#ZiQI=jRdFPJL(R-rb`VRi@8!B#r#p7J(t$==<>Hi*
zx+vMlQfN+Qo5zzkSmAM5kQ#+&QSSlDkQ>xw@-FA(=6-Vu
zvaMO@lNR%`Hmt6S-n>y+`A+#1)TO}%{ELd8H`CkVbTl8rsmY&XxWGMy=;4V;x+mGY
z8~0f0v^mYI2Hi==lo!T0Rc1}SNyeSU9VwAa+&exLqvq~RnZ;fXw~v)fAoGfs-(
z^g(Uakdpa|>=eI3Ynsp9wpKY&FScBK#eQ=4Bb7OoM%+Wr=BwUJ2J0&Z3RCZoYIY%j
zS6Vm#zG<<(Hhv0Hif{N_q_95Q{?f6|iRpDt|B+Xqu&!aTwC1;>A5RI!oJCiSJ;?gG
zSmH|?-)qWKT>MvmzDgR-C}|oZeL2=xnDD-fpff9@)0<0=y>ADFgPv(OTt(WBe+iRnBX2^z*wE$;inNtw0zMMCStS{(->qcTP5j*iE}%%
z5c?-kS=IZdRi)|pD~2nLSIb@Bq>iWd?YwTsons0tJU9h?f?1cxbdt*~dslOfe%~Nz
zRNh#Bhjf_bhT$T)Rdz6$#~H)WDV1>V7`|JZkzYzg8G`ZMhqNm8Z{^@4T_D>+&1P^woBQ#G(mT4M`-0z>?)m+r&6y!d-
zcb)zUZ?$Byg>_Z`<@7iCAnougKD4`&hBH*nDe7328bKVq7>4fnk*o4F(jA2r{?Eb6
zYwOihmAlfC5t9x!iw&0gfA^=Kh%gZvOtsaH4K3;m?iREM#)m
zmK^dVEJN85g2SQd5|!_E)4q@%i!)Kf>OY=>R+UHWPV8ZwVtud;u13^Ng98}p*(3_7
zd*C-G#iDW|UGi*+X8Gi9ac#Fl?`|0-4T~h{2V+EI;65q6P1Guuy+ow)(>L}AyngwL
zFWkwQxP5R2v5$6>DxZco-!4kp!8LLly
zcw()yZK95^2}j@JZLO|s(R|LY#2&dYA~g>Sk)}kn4#l;H2{u-#Sb@L#9-EI{!V6~s
z`H)n29u3y=Mn;zm$zU^}M~b_}(P!U`>YQDH9vqmV;~Q#093W@H3pETXz?AlY(u^aE
zPE__h)?FhQdeb;SIg=8&4x*_0DLFJxAD@s&^?kt(`;*loM4Bz@2Fz@>_Cv9Q=JH*F
z^2!a;?}+yw7%guAThP7DHZ%1i%~FoJ7xM+FYVXa92=!
zPGsmSNWVE^neE~_DGE<5hJK-f(mV9atZ@oNu@mz`Tx*!O!1!}|Wdbgsd6M92R{?{T
z{|}qKcS@(A0DAVNq1Bs95|iYc_qWJ2oR*4agMy91u^wNZz%F1Jo{U`Zk`%#j$xN?{
z-lD=HUG?^hm7fe{^2O0z98*;1h^aGJyR`1MFg@yql}+UV_JM_`Q513eo~@+EHyeNr
z=}$T%*f86mFO4L1w`Y_fOkc;WM@@D(-X?`wPEbiGes5>;+AoTIP*OR#zn)LD@?tna
zQ&2!biLNi;GQZdYd4DoywlH`-*wgN`Z>i`u_T5$qfV*l#)bLUJ%+MDOrQa}$+3d8r9`S|X*m
zsoIzb(Zq9`GTR3!;F}GuDieksI|6CCqj$`=WCRzswp4ZE;3=mdJ0@x`mR8OzNbrZM
zL7|R?n|9+CGjio~!mv4sSJ&OD2(!IW*0Y;0rxJGZ@01LhKiR&$|289DcrBb6lzYSg
z1%KN~Yl7n-^)%u4`^%{#HY})@YT}kT6lK#pK_>x;`|R@3H1ucLe_J&kMan_XqgX%E
zSg2X=G1#jXTRje2(b8pa>+;i>b=t&tn&&pOivPA|S^okv6cslvC|^sy=Z70)s_8{F
zJMyyb65Bk|Q1ygp3ZIlt13G|}@)fV`)6?|xyX}9c*FjR9T@GIR3Ag(2+rqPv@2HnUBbzvg
zsh;u^rI9=3e3>+asC@0m-Mym%F?See+`>m1L^jrU27BtB#J=pEpc()soLuj^N$9L8
z(Ml@$KtuCv5C6DECA-@#?!aa0F1|&04eNlemevOQvxjZS4iHjhZifKW?+63-#^3n2
zH11xvYdj=}qByyyJyQ4w4Emh}dz2yKZM!!Ey^biS!g&>B@SJ^u5hin!zr$yc;=0yPjV#M
zfnuN?TP8xJq2i8zSfWOVgLLPk$o`9U?XgSdF|I9X!>?9hd|lcJ=C%!%mAXyxWp`DoJLtmgdWXH3BMy{QEMhT=pc3;RS1HWgq6nTL_78eD4tB<#wN;A~WiQ=&9Jy<$ns$dS
zV1v*2X4Ot@Q(|vOfjjpk7$@uJJw(iO7nC}`|1b950<4Z@O&7)?I0X0L
zF2UVBcyNah+}$m>LvVN3;O-KFFC@6T>%#BKK6{_pcjnx=GxN-yGxPuFd02FHtzOmD
zUsqLE_4j`7>zGx)bH*Dka2mSL2kOA2gP^sko*l>o4$(l%t3lS#7`_Vq6H$sxeB3U_
z0Ofl@SwfqYlA1KgVW9l4VkE1{F0zmB*A+1I$Gqm6Jam`OI@M=(;0R+>Y6Gb!YA}>!
zdK?kto|B#sES>6>SB{cP1a~6m;Yxhk(6lc}JJS2okdrqmDG6DHYl8;|nc6U9{j;{S
z3bMAh)2NH>YKz4&6v6-e(NqFDipS_D=*UqHIHzJ;uEAlC8~Qb`MP7qB?}N}a>}LNL
zO!RNAqJPIRuRs4rRM?+s0{+dxUd$j8uit~c*ng$}9l>5Ia)yqUAa=vQ(0TlyL%-Pm
z!q)XaD)bBd8vObnpUpl%77lIQBxpKL(|6HGa~Ul1qggFc(mC{4SAvf
z)a^p)+BX1AO6g)=I3=GL*Fx5=nSHkthhD+(tHbwmP#Od`T%JGx>fFFrF!Cxv;0p11
zn-8(?WDm#}qE7r|(@B@-Cjf#v{i8h?4`r(CPhez0
z?*Ht0)4ye&^$#=r7QXl2%M8)GZr0X?&8G&a=U@c81)%r-k7bPs
z=lMErHpZZ^cd-V!B}bCD-d1XU>6jk}!(efdpZx@={?xYrwb-iv%zD32%{i*M`oWfNSz|rYq9iO3T&z5894df^&O(&gEH{pd<<~2t=
zjJZX+Y+<&ZwiNYIxZnY;iR9>T<0!c~C=;}Q8teZzu-yN{d_jqlKdrN$cK*J@0;mF05|M
zU0-^}`y7`o+l)k{DpH}m6A-{n&tKx1IZ#zxA(yLw}!V^)2yFTg{gVVDzF_jQ!4`
zOT1WZ?mc4IWA`}gPccurKy!~-xJ%^haY`kY5RPQ)Wn55XrBT9
zbX(o*1AC}q(FM=PZEsgwUf9wI4i0H0KSi^hC6hz7#?PY&>-hUX0K3QVM@Rjy+Si^YF^Cz)94O{Jr*Eepd&
zlQ#JeQ>(~h!kXvzdF;b`&jx(14;FvtG3U{Eozkz97^q*sxG*sA;A{3__(xA<4hG*f
zsUU+Cfr)}%@np$AMO6NM2FCwhvv<$~?jL6Ot=axJF~hYjP+1?9A?*c<3(NI=
z1*5VC1sLuSpBHq3=IE24^CdKbe(dmTR{oEQ{ljAaXk-8LdS%1E<;~hZj``P_vHmmc
z^EW-ie-O>HuyTMH*?-q~`OR_r2j%%c_ZR<81^R#Y4of;1f&}UQ78mI6LUSOstKZ}v
z{_qd~S&Z(VT*7V{Cz-Z<*!8IO
z;6lSf(NOqMDez+8!m;5=X-RU-P;H^M;Q~Ubj~y624*~P$GTf#d<8;2P68Gw9+lgwJsZx*dh9kyBqh?hvuw4W={LiqWWo9
z5pBgYwJ%rpcY2ErG&iOO?aIZn*^k8)=X+Me;Ui>>Jk7{AdxnY*$cJ|TK9sIaS`6>Y
zeR0_Hd;v0Zm&J2U?EvRHdH6Nlg$AMZvi-nWgn~PA-PS6Cp4fq=gg5W
z9TY(ZL_{7kyn!pshHZV8KeO$1_PB
zDhHlISxi+>o@GsSQR1nevu(KbtzD5^Fw|g
zHb8)7qTdBTYl?xb+0s%abHOZNQ<=N^7O)BzLGc8sFL$I+s*6UN?=IUIPt>LS`0gVX
zu^a;zzq$rCZm#st?v$=Z6G5>P2nfnS9=*B|a*JnajpaBkHy*9Ae6{RQq+&%`1@Sft
zz+!^ZEw;slITu@au~`-_!BAhLF{k3F{OB-{yTg;dQJW^rCi-wgsr!1Sv@^x_AuHl5%qL4Mz`J{9O
z4Mwg~UIHQG3}1z}pU+vaK#jZ4jp8!m?$h>r)%upKMXI8f1g(u9`P--##VI@WlQPHJ
z{Q*+yn9IKW4V}0n)D~Sh1tWKW!_U-H-m(gvB9+!fb!>HAr_ZH@V>+k#w^8p}&wIkj
z3NuSPjK9FNzXmgt#q+Y(O3PdgaRXjhy2Ai@pSec!=2r?w94wSrGx1AxF}U(eDaPD<
zEJaT8{SJgHOP~>pT$F3bO;r#qz=PZYHt=7w03KRBnZHuz00*s}Me37oBFV}k3LVjt
z5-|4|tq-yq=$VX5l1i55NTy0{w9J%>vgSxCrMj3H)}y6`^&!utnV6_#B|-I}u%HxT
z6nrTt$v(AzWXIkRt!59?&=x>q3EmK?cEQuo6_A775bmNx5Tqx!RiauSIa)$kG*N|i
zkc5SIrsVk{+p?6sK5AOGM7Kx|(ZdGn1F=Q&fdgDaz%7gaqG>CiNM(+*=@vpP3Cai<{$5U=3?dI3Fsr60OFIZ_|Fj?&|)%dO%GO|JWU+AG)ehp$xD>G(Z%t3+S~|Y>ssBkxs#5h+pSWU$w;m=
zot?Pe@GjWIP@xl+Hq-~Rf~L9Fp#-v^pkz_*rm_Xi$%dkZK6F13Kt!vb
z5HJoO>iElkn!i#ZzfyIfAK#=OyeOsOFKovIGVW9|D8(G6CEn&13`*6+gHk~MMSbweSS_)oh0I0c2Lh?eTz7lvpB^5!Oaq6=M8Ip|6#>XDC@%1siLWX%Yf
zR31MhCjsCWhm0(;4|*wj0k!-#nFuR{LD7fuT;aY6r%+=!#0gYvH2x+gCEu<
zOT!G?1_72cJiUUVSEiN_zBy>P;TOmO-MLoKor{?$x%n0+TiQ_sd#82T!w=A#F~)50
z;ke{$n7^j}bySfG=qM!6?Hv}es6MC-(-Opg)6i(UWGTtdlH(XlW|KQNA=6t9vvB?M
z@MSw;5sGtH&FJQsynft*p{u%Or#b|R!|Y9=(;T{5o{_`DxF;3?A)tf+ZzrTtc;pO1
zN?-v8(8A9teIx3JqW(2(y-M2yhLyG}`!QFXd9#fa!(?TdKUT?|cpSwUVrrGpzM+98
zb|!OxJWX>vjpXZH@5E*%CroYZW10x-(mgHM20x2%>rE@NjeHb2Hjv$r9Sfp`Mb}MA#FPxS>9a;>X1Q%)E=9@#i#lTK0U=9P~3z9`elSqGFsPUDjd&D#23$jvk
zbnLGBKz`V0u{~mgLL+o4r+?QLD8f;Wdmw8n8Sni$sYn?ChW*0ek(Y+JXWo3Y6r)aa
zEX3!NOM&JX_Z;=66Ng)*y~%cyIaj?9UDB)#|qJq%OUz9Ay3%VI_7
z2#DnJ@?)tJp~5_sl@oj)s_{XEf;nH2k92rmXgW&W%e{+`$!=zC%l6-x)1$0sZRz+)
zaX96b)qncGUGFY5ZPO=BG|4LSSsFK_!6aN&0p-8+4i^$_(hY^pC{?Q`H!zYHl7301
zXry`wBg4+#^pIP1*hpIgxPD$mlrDk0s3LwX%1>u=ihK02K(X(7(*J(f!9tKp8uZvS
zh3`BkC2GqBxG94ju>2~k=malbXja?Y$%H++OdskH7C-Z7l2eVzmsmnY)}$uT9u>IF*XD*|u)iG7
z459yI8VStabBV*Ld5YeYQ$QBaFRATu_C86px=-&MsGKuJxEQ*JYVe%9XCO!<9c=H8
z$8_8_eNb-7t5b)^_NsEbyb|Lg6_29w=G81X^_*T$(D6#S39{?qwrJwxAc!KBB!1Py
zG9y=ywwIMh&*adzHm0=Hq(&EyX6V73vF%hOXg}6SmhCTG8?bOCz>1erGFGy)SAKk<
zMegj~iCdMBGSw0G(t$AR?8!NQ%I#eH4=n%5#8;XOH~xlbZVUxg9{X)rs$?%xx^~yYlxWW6-{y3
zUWE>9q-iv=F20-d*aS-jaqAD9RHOb_fh@Oh^p=7Uk9Dq+b3b%_o(XI7a0vWp{^$f~
zuR`4}nCogvz4jP(+j-kADjPASO1FL4)WN2?U{c6@`<~BC0%%BWQ@VaBw)wLLBbi~2Lt>4WC+dxJv^rejp*2-C`XNHuLHpr@V)4ut9khqXm2)T{HgpHB@Dp
zqNBw3Ez(N93{w*LmK_fz3J|9AqGyMsv?MvQpb%w~*TMfgXKbk5&WGU-)Y3iUO;M7F?Lt9l~`^VdkY|0Dn(7)=vDzvfaUm1{oZRB%Xl=#N-dUwBGQ%
zH_#g0A7ITMiN&)ycedCWV0ib_yfo`|0z-z`QKvsL_C$6;6a~q|y$y3JhKPg^wKYNK5F^j1l-9$m}cNz4kxzvxx*n!Bsod=qEh
zA<|8wqlWPiF1HOUjSpPiOTXlSj|H?8Q8H~brjD$4>%zPVoV>83XgqkKX09|VJiJ0S
zkzfzw8#gz$MM%QGaC6$AiQtF-@>vEO^vjE8C27(L$>`o)gpGa1IOSd#dbN-m@DI-T
z`NvmD_+#Pd&d3fQZxMUytOS3cxHGqy5#GYX$SN?X-EucA&TN63nt)soTZ)2kXV)4l
zZn?-^vMm8@Y#5?UxR|@VbW2_gY?VTm3U4TQvPC`wH8mh4dXEoCTn}GW(Lab!_^0QU
z$0fE+YV6rNXYYC0Y9yKkezlZ?-+BGMWbA{gHnRXuPfzkt*g8=-0Fw`Kt47i8oG-Xw
zl$_;>s^UheQiS!UC5k*5Wo8{ErM_BdiL_!w1nbHEm4JmG0=Zmc-yL5EN-c1U*<=af
z9lWc-paSF!{3jGD*tr*ni43Ve?V_MCX5j52>!cfZQmq|c{lM+~XE50(ib?-*hkDAb
zs53j0JFL!qjWHZUU!Wq%6YWrou>{Zz4lC`INcIvyd
zCcf%Z@#WnQ8_;*k=%&4NpUH}(H5Q(g`maR|?o;Zzn&K#D!tIUALRR^H)+FfJf_&tZjl7FF-)NrZzulbRXJF|eN4#s^{9
z{Xvr6Fgr6p1Bn}LbQDdt89SWseq(o4R%_rAcP
zMh$A52U5Knmw}lK8}7LAC2kzd*9qxbo<2`v*ov0U30OGkcJbdf0=v%*O+__2eB*YX
z+zuGe8;f!he?UTJ9a(W<;E>+S+S%F`?mhttbqf-T^1}Kb4ch54kPJ;JK5-+*EY2Uu
z$8v5#0Ukz+Avu~QlRh)MAxql^N-NWH^duj#8RH(Lf5WtqtLEuA
z`XpvpYKbnnsj4V0Frb;6A#N=TIB{6PBGR$#sY`RrqrBbEq1rcN#;KcipkWq6&*XvV
zQ|>shhCDrHDOK|Dh=vY)M*~;m3{w}iii?AFAZLqjG*P{9RlW*y%*DI_2C~5#u6ctC
zGxKr*d)tYQxsLc~_vPS2L5m4rZBuG)H5xsFFFC#>3dvduK3g7@cvLiULVrOXsd#CLvrRLfHk+
za*HXJ2^84v8o#{zAypV=>Hn_Iuo&toHH2%z6{G0Ui-EqxqnUA7|}tKMphL+~S6~L0G+rHMD8zs_Yb+A=A4g
zk~3L7|F)9}(xe(P{As97DFY3(ZsJ{JE@nn{4ekDpbw6L{#t*&G1E$>CS&)e8<`cD8
zuJ~90i0h_i#z07(CT00!B!)Pca!&9*?nTFnf!tu|20w**@h@+Xpr!i9i2Y4)l_lLU
zYOZIkN5d*+2ieD*YBH4-0D-o98b^g~eiUlBShqq$-x13ovE%JOrB!_??z^aA^9I{u-h#XA-A2kCz?&m*r>GGNCj?yS;LO1?NAg7P+jMgHS=!U9c%itN
z7CbXXIt^;o-jg6m-1NtgcC1U5YTX8g9%SdzOo!ii88nMCg-aT>8p^Q}R?_OIcB`3{
zR)rprbvr;Cz0uD1F65Q;PjS_t?|vj|`z9F4ry1l;VHk^;Ltw4gb=>B|S%cI{3w?<5a!W-@Ey|NUvvn?Cioah3$hV4YrSPlg`(-}g_5)o_G
z9OGLTG6GcVcgdhqHl#I^C->FAp2Nv#!4
znf^xOOBrA50q`0aUkkc9(Jdjf2=Z!pUZ4iAyl;QQ>?-{$zr0HQQrG4{AYj)JVgnBjkrVA0H
zxrkS=LQZ02YOG&~{b-aUdztLV3|c>`H*Z@Riq&(YS3Cy=aZ}xXDdOCh|G3MZ|NS#a
zYaCOae&mY(p%Ddv$C$#jf5_dh_JqBEc28=IQsKBk7zb5XkLnfHYSdNKDjTxs=o>p#
zTJSuQWqPSu(vegrQ{)})1KSyRRIOjdHdFV(OAK!3vT3u
zrPuW&c&jm7ls`a!l2A+iIAVb#sJuhdhbpW#LP?huw@dPCWWAQ6Geju*
z!|c|dd*n)A_M>q8ZDM%8ozdFoC?_ZOpat+xBXP&MVK2f=cB@Ma*F>OU-%ppHycYN`
zi^Lh)-vQ9A^gFq|0C_Bloh^d}6}U*s)1#?ujM<0i(lkC}Q;zR7b}^5R^Lj*g#m4NO
z$f-`7MDTVco8HIw4nDOGrm*mt4ACl?k?;u`L1C3{b{kq2C^fcr>}HebFe9>JO@A4C
zs;AmsD{Ibqsd8y|^U@ljV4M#f;ApDvt}w(80g&v_c)Xj)C`yfXh^yf+*V1lQdq!|&x_6+gb-#lJ@=|78biSu?@UI-5k!H<+)Lu5IBmdW;OQdjB>+ssd0r7}v2
zX;fCn6~o7TiTC-nVvGDUf&{?*iV9tZ^!n*ZFt=Z8fTr|
zeCb(eEG6;@I9^QczlfjSz^}e#yoa@5*se2nE@U$^R0JUIk*QYtFd^0~blhwP^->&7
zZn7gG6lrk
z@zzpbV66}RoMIy>Vrn?NLVS{^#)klcrRvBlqfR5!!nq**=v~77hl?aLf%Hjk7S+7B
z)ghD1^);!onxf7v9%pXa>ESkM>IqD9)852Q*br~^m9w*{3P`en>*e?+^M~s_45D#b
z%)F>uP{V+fO-FFF=N;1Hl%Wn@Fk@M6a1@~R0Kv5
zAu$mLxjE9NW!h5wv1vO&J2yfA8PDtVM
z=lvNuo&ZrTdR6;8gXnsl*{!5lsBxhHrY_a=4qwc(J3b#_^jlq7>uKBB<8ITL$|D`G>^;?73N`bL{KsOEzKatqh31&t${OY^vSh2Rvbo-Ixz_Vyo6$8G<6NKd
zcxuZ&*E@B+V^oJ5$ekiR_vq(%(^4P7BKP({OGVyM+G7}de2g0lAJ44FDMm?g5oelq}};yBhJ
z8D9P6!eESL5K4}mks;x<=>oMWVjPvpzEq+>*g=>Y&lE?$o}JQcM^-DysjE(9h0%gm
zq7rK5eJ$2dNS=N0MM=x766utxhxFBSqgRF9fJ7h@CX1DG7x@C^CTaGtYrmza+B?jv
zmf2+}?cp;FG}Eh08w4}{#nGmI6~%HhMhQ`0OS`uHMn$J_g1P6F1*}__{+Y>-78Q>W
zUIBJt+T7BFb;%)Q&cV!&4#f8e;Zkv9Ol@~N?H}4DxP#=u51`+*e)VQEunm+s75Z>X
zaF%s5DSWK;;K&;IHd0!)f#I7(29a2k7*qY1(Odgd#)(8yMO~xa`hDtEdWWZnwO2%e
zJFvcY$EfF4K2v>_#Mo5UCH;PVE2KSh`dDM`duu4m4pU)-KLk5XZZ*3cY4iE77qCXr
z3Vc{wepq1zoz9z-cRm?BrNF|hOrEvRcoMA?T3>m3ZWw5Wkwe
z7jPB+OZ6gNBE^Sj43!xbV$V|T4dMf7BiL`fi#jX{_!pljR3``bkr-BDQjKXjIlV3m
ztQz6#g(C!vsrib=(2d>Ga#d{8t`AOGJ%`lPh%&H=KdB|6N=GLMtg!WY_8Xa-^ES^^
zP#a!`UT>&1tHaObP4Pg)n16epaAydJr2}H5{-`bK9fyCG@OTq%m+-XlqmcQd%FuUU
zibJ2E&&HJgi`^it3}BMVD+IdRYNqasxF9#g3(`#6i1fkOB>a==!4E#hWL<#?pmbMZ
zX-S9N5jyhoc0ovD)laL%4^(y^ZkACV6a`oxqxDc6H6^Ru5h>s>tT&YZh${^88ufDdh2upHz;7gN
z=!qntwFEDkr}2p~*)5}qt6sTB@Dc#hyfIcyXZQ5~D4ki`e<53v%8?
zFXyJ0bQJZ?Jvx+P*2lFA#3*2f?tprPIeZq
zzv5(m0a$)za&f(9{v(r>_4iCZK9K6IElBG=Mtgs_#JnISVg%U@U*{<6yg&+-=-&Tpr3ur*dOae~(domET%p6R2B
zo73;biu}G6{e3G9uluVye;|hbe9<4>?Ld0KiiW1Y{df2!0Q~zIw`semf#U})oLX9zl%F$i-c3d;E>Ibu*P|5VXm#s2|q`dh#wra!<*zXX#1tV-;b
zF{ZX+i@}N7^p#Vvp$w3G`N2_&YwKdcSg;ZPotRL%9jfYA*pD)#_243OH~Rb8E4XJT
z^R+x*M^!f~R&HKw_WH6?*MN&2uxdlkE+VKq_d|&tjq6}u(%cZUOmk~HWZpVIV@at>
zHpk8bP*a_z7aFiT^cNi;^&)ka0<+U%lB;|4o>aXLsuZxb&AaMt&wi@lT9)-dI}_FY
zSb!bGF-b}x{aP`28d5fE`lE^!-#wDuxPr4eKHW9Pu}2;I5LJbUrc`7C*UOR~hMp@R
z4LlDjhz>Af2NHaot$-4*uixX*%&(E_br!#FrrPl{)XMWrB25
z2SHXz1WoOM=rIJ`n6XlB%>{KAcOznY*9~njCy@5ICdwVQ%e_2%T47)Ls%qIFG
z;$@KNSM%fA&tKe0lYlap>>bBA_&Zals@ZGh1A8q(m$b^9;f0MOiSNnCBLv-?ZXPx#
ztWX56m##H0Ts)QfX?Eh2;Y4n5a>bCzN^pfsftcGg5}6(}t#!AlAz$xVBSnzQZ>l4s
zYi22<+pV_lV}R~P9p0wSXITr$)!m&3>UOv1mRddEqRd0|!i7!V7gJ+W*HbzLV1!Z{
zu=){^F;`9M`2YS{}yF?)lJ11z-RG)A1`oiDG3m?)8)jm{Fx9zalr=E=wUPZZ6
zf4wh{rAt}E-mPj`y7(>o>h)D(BMLqz#RU{c-1u<*Xn<1v>Ynpz6CV{Nw
z0xL>^c}80C68_B_T}GCf_JgI$;EowRWy*+qwV_*&Lf-cz##ysI`SR@CuLe_)vB{bM
zJ@^uDRb)YuFq#)}1zZdB8U==}fiGPCY2ed7?EySJ;H|hP;g$`>o;RU5M&r~}ACi~c
z6$Mr+UG;MSsJTQFc@rPCv@?^C@o3~TjAbc0*1=XXIKHss_^~c2)+Ol0tHah4#t)Lw
zREp9xY>jEk;D0@;?U6)G=ibEHqltveS^1&@Z;e5#(g$Y&Q<9_TjNlvqSuc+9$@@#j
z&Gp%*%=lQ|n1HNc-?wSJ=yklT6HTUw$#;Z5b33uYl29GzwDb5B?Y$4_
z6HZ%ITJqDJLs`!vIqB9VF0y?d9d62}!VLr5B7B1s*m3?M8$;oxT@W(FO^1V_f<7w)O#Yoi^Ydv@i4hV_O-XYnG`(*M
zKQemPxgXLtHf>ue>1D8bg1+Yo;(;XJd8~Ad-8Gt^5R?sybv4-
zssoD0{pQ52&biYIVUzxex0>u-l>W)&aF%v_hbl+hw`SalN7bV)1TQ+lfOdo;pwE#a
zPh^rXK*x%%MfMwY4;2m+qGr%kbr#@D
z#2yo>Cx}OQ8&|xj#fSc*8iCt+DH1}u=V+fcq|f$e6^;$QCNR$Zxx9oISMI9=ON={t
zQj$W)JANq~+;Q2@?R~e>irILx9}U%}*TL^q#XaY|gP4MkS_QWA!iy^o8}TSp1@g;R
zdQ6QWPUGD(XiE+NkL%#eY1%M5-|2cA1Q$cP)m%U0n>n`uhn||68pK7f(RrE<^!(4y
zHeG`{Ny3u*lE)|j@I>0?`7GbVQ{!Z$-Cw?2#
zhB>yDG|t901%@3^)E)z>tIi8o&mdgY_-{doxNNmO2&1e8_M}O!j($1L;W{r0Drt-lz)MmB%7;^C2dL
zCq2}~@5kf`zyTMpHevnN=biDjJ+(u!q$2I!euK9|Ax}HA9O(L_a8JgOeRFRQ3K~?e
zTmF!AdpSnaZ(r&n;D?wt?V>*)ht|f|wIw*{O}vpeV1DZYxEV?I&@4!YGJih6hc)9{
zH%N%?OFN%`E)*b>vu4b<+~}Y-4tP>It_oU!gn)hl651E=0X#pH_M?d*#bibyDgQzTsi%2tv8yxNets_
z#=twIRqsLBVALvKOa0tOpT5yd;*gVpY10hLJ*2a)-ZJ02RzLqG_Z)iw!CJI$DuPaX
zTYDc171JP98xa*b@SQeclYMt}71c$C^m%H#QAFa)$jju%AvKm$g!>Eec`8mr%`1mH
zE``W7Vy7;!L%Do&7i0CgSf6{Zl{oF~PcS6t+Qr{ldbXnzx1%0qyd;|%BNSWDcjH?v
zW5tTTZz%P=9Hp`VuQc`RCceD8nD3X9)ww{$o-*&&l}bSBwG6{;OrFq-UQ?tyu`Jj2
zY?HRO$qBbKR~#Z&Ci&z{1}(*h77cKyPw)(yaoKlQ@6nG+Q}0iunAvII(Q83>5B^!O
znm}?m)d*w1A~ihWBX!dX#2-qt1BzeVn@ryg7@N9c-qZyqYzrVz%|OrW)1A2$Xc|eA
zaZ7^x-@s%=(@JTTYDR;^P_x5_0=&R0jy9Bb8$#~jUcsCTjWEH7tbeecq#c&;>MvSu
zRtK}&!dP~q%~Vraf2QjX)`)7O-Q^axZ;rH6zZek9N)jLP%rab8K=zUaPB@oX)B=;-
zHkv?dRcBXn`!Kli;VE!PNCGf0D0P4!1Rv+-1UyjfO75aD_x7=&{c}_D34`FN3nkcR
zF#l#wn0DS<8(_=&D;R_u{Fdwm^w>bh)tWRn+7g@p*E}!>!4H_VdG`RCO7xBx(
z52zs%S89|DO)ksUM;RAdyyE7y@RuH^&KT+$Kc7s`RVM;Y`IMb$NTzd}IkG#0k|fj3
ztV1(3HvG35rMqG3I*Gl!&Xc40}AGpiDk2z
z#2tuufC(FdkXHKWmZ(cQ&C5x{h~Jp^V2uKV)-Ub&kAAG(7N5UgzBlUYMCJmnxZ^v^^?O}tU8@699a2_*=0_#@%VfF@}P`vWEJ}1hVefu+}(EZZL
zQG9`MmHm;9?%s*$I-R%B(g1RkGn|3IEM^(pL?>>a@e5=c&jR0sb6(3CyxRnbZ~FYv
z(u1fN>cPS0%EWl*o?4#rTdeUGco
zvk=L{