@@ -7,7 +7,7 @@ | |||||
"provenance": [], | "provenance": [], | ||||
"collapsed_sections": [], | "collapsed_sections": [], | ||||
"toc_visible": true, | "toc_visible": true, | ||||
"authorship_tag": "ABX9TyMsVwxzy9cP7RXmbb8AmT4Y", | |||||
"authorship_tag": "ABX9TyPuibV8rME8Y2Er3TVnCm93", | |||||
"include_colab_link": true | "include_colab_link": true | ||||
}, | }, | ||||
"kernelspec": { | "kernelspec": { | ||||
@@ -45,7 +45,10 @@ | |||||
"Author: Heng-Jui Chang\n", | "Author: Heng-Jui Chang\n", | ||||
"\n", | "\n", | ||||
"Slides: https://github.com/ga642381/ML2021-Spring/blob/main/HW01/HW01.pdf \n", | "Slides: https://github.com/ga642381/ML2021-Spring/blob/main/HW01/HW01.pdf \n", | ||||
"Video: TBA\n", | |||||
"Videos (Mandarin): https://cool.ntu.edu.tw/courses/4793/modules/items/172854 \n", | |||||
"https://cool.ntu.edu.tw/courses/4793/modules/items/172853 \n", | |||||
"Video (English): https://cool.ntu.edu.tw/courses/4793/modules/items/176529\n", | |||||
"\n", | |||||
"\n", | "\n", | ||||
"Objectives:\n", | "Objectives:\n", | ||||
"* Solve a regression problem with deep neural networks (DNN).\n", | "* Solve a regression problem with deep neural networks (DNN).\n", | ||||
@@ -378,7 +381,7 @@ | |||||
"\n", | "\n", | ||||
" def cal_loss(self, pred, target):\n", | " def cal_loss(self, pred, target):\n", | ||||
" ''' Calculate loss '''\n", | " ''' Calculate loss '''\n", | ||||
" # TODO: you may implement L2 regularization here\n", | |||||
" # TODO: you may implement L1/L2 regularization here\n", | |||||
" return self.criterion(pred, target)" | " return self.criterion(pred, target)" | ||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
@@ -31,7 +31,11 @@ | |||||
"id": "OYlaRwNu7ojq" | "id": "OYlaRwNu7ojq" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"# **Homework 2-1 Phoneme Classification**" | |||||
"# **Homework 2-1 Phoneme Classification**\n", | |||||
"\n", | |||||
"* Slides: https://speech.ee.ntu.edu.tw/~hylee/ml/ml2021-course-data/hw/HW02/HW02.pdf\n", | |||||
"* Video (Chinese): https://youtu.be/PdjXnQbu2zo\n", | |||||
"* Video (English): https://youtu.be/ESRr-VCykBs\n" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -30,8 +30,11 @@ | |||||
"id": "eNSV4QGHS1I1" | "id": "eNSV4QGHS1I1" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"# **Homework 2-2 Hessian Matrix**\r\n", | |||||
"\r\n" | |||||
"# **Homework 2-2 Hessian Matrix**\n", | |||||
"\n", | |||||
"* Slides: https://speech.ee.ntu.edu.tw/~hylee/ml/ml2021-course-data/hw/HW02/HW02.pdf\n", | |||||
"* Video (Chinese): https://youtu.be/PdjXnQbu2zo\n", | |||||
"* Video (English): https://youtu.be/ESRr-VCykBs\n" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -171,7 +174,7 @@ | |||||
"id": "ZFGBCIFmVLS_" | "id": "ZFGBCIFmVLS_" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"### Import Libraries\r\n" | |||||
"### Import Libraries\n" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -180,16 +183,16 @@ | |||||
"id": "_-vjBvH0uqA-" | "id": "_-vjBvH0uqA-" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"import numpy as np\r\n", | |||||
"from math import pi\r\n", | |||||
"from collections import defaultdict\r\n", | |||||
"from autograd_lib import autograd_lib\r\n", | |||||
"\r\n", | |||||
"import torch\r\n", | |||||
"import torch.nn as nn\r\n", | |||||
"from torch.utils.data import DataLoader, Dataset\r\n", | |||||
"\r\n", | |||||
"import warnings\r\n", | |||||
"import numpy as np\n", | |||||
"from math import pi\n", | |||||
"from collections import defaultdict\n", | |||||
"from autograd_lib import autograd_lib\n", | |||||
"\n", | |||||
"import torch\n", | |||||
"import torch.nn as nn\n", | |||||
"from torch.utils.data import DataLoader, Dataset\n", | |||||
"\n", | |||||
"import warnings\n", | |||||
"warnings.filterwarnings(\"ignore\")" | "warnings.filterwarnings(\"ignore\")" | ||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
@@ -212,17 +215,17 @@ | |||||
"id": "uvdOpR9lVaJQ" | "id": "uvdOpR9lVaJQ" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"class MathRegressor(nn.Module):\r\n", | |||||
" def __init__(self, num_hidden=128):\r\n", | |||||
" super().__init__()\r\n", | |||||
" self.regressor = nn.Sequential(\r\n", | |||||
" nn.Linear(1, num_hidden),\r\n", | |||||
" nn.ReLU(),\r\n", | |||||
" nn.Linear(num_hidden, 1)\r\n", | |||||
" )\r\n", | |||||
"\r\n", | |||||
" def forward(self, x):\r\n", | |||||
" x = self.regressor(x)\r\n", | |||||
"class MathRegressor(nn.Module):\n", | |||||
" def __init__(self, num_hidden=128):\n", | |||||
" super().__init__()\n", | |||||
" self.regressor = nn.Sequential(\n", | |||||
" nn.Linear(1, num_hidden),\n", | |||||
" nn.ReLU(),\n", | |||||
" nn.Linear(num_hidden, 1)\n", | |||||
" )\n", | |||||
"\n", | |||||
" def forward(self, x):\n", | |||||
" x = self.regressor(x)\n", | |||||
" return x" | " return x" | ||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
@@ -297,12 +300,12 @@ | |||||
"id": "OSU8vnXEbY6q" | "id": "OSU8vnXEbY6q" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"# load checkpoint and data corresponding to the key\r\n", | |||||
"model = MathRegressor()\r\n", | |||||
"autograd_lib.register(model)\r\n", | |||||
"\r\n", | |||||
"data = torch.load('data.pth')[key]\r\n", | |||||
"model.load_state_dict(data['model'])\r\n", | |||||
"# load checkpoint and data corresponding to the key\n", | |||||
"model = MathRegressor()\n", | |||||
"autograd_lib.register(model)\n", | |||||
"\n", | |||||
"data = torch.load('data.pth')[key]\n", | |||||
"model.load_state_dict(data['model'])\n", | |||||
"train, target = data['data']" | "train, target = data['data']" | ||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
@@ -490,13 +493,13 @@ | |||||
"id": "1X-2uxwTcB9u" | "id": "1X-2uxwTcB9u" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"# the main function to compute gradient norm and minimum ratio\r\n", | |||||
"def main(model, train, target):\r\n", | |||||
" criterion = nn.MSELoss()\r\n", | |||||
"\r\n", | |||||
" gradient_norm = compute_gradient_norm(model, criterion, train, target)\r\n", | |||||
" minimum_ratio = compute_minimum_ratio(model, criterion, train, target)\r\n", | |||||
"\r\n", | |||||
"# the main function to compute gradient norm and minimum ratio\n", | |||||
"def main(model, train, target):\n", | |||||
" criterion = nn.MSELoss()\n", | |||||
"\n", | |||||
" gradient_norm = compute_gradient_norm(model, criterion, train, target)\n", | |||||
" minimum_ratio = compute_minimum_ratio(model, criterion, train, target)\n", | |||||
"\n", | |||||
" print('gradient norm: {}, minimum ratio: {}'.format(gradient_norm, minimum_ratio))" | " print('gradient norm: {}, minimum ratio: {}'.format(gradient_norm, minimum_ratio))" | ||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
@@ -27,7 +27,15 @@ | |||||
"- Baselines:\n", | "- Baselines:\n", | ||||
" - Easy: Run sample code and know how to use transformer.\n", | " - Easy: Run sample code and know how to use transformer.\n", | ||||
" - Medium: Know how to adjust parameters of transformer.\n", | " - Medium: Know how to adjust parameters of transformer.\n", | ||||
" - Hard: Construct [conformer](https://arxiv.org/abs/2005.08100) which is a variety of transformer. " | |||||
" - Hard: Construct [conformer](https://arxiv.org/abs/2005.08100) which is a variety of transformer. \n", | |||||
"\n", | |||||
"- Other links\n", | |||||
" - Kaggle: [link](https://www.kaggle.com/t/859c9ca9ede14fdea841be627c412322)\n", | |||||
" - Slide: [link](https://speech.ee.ntu.edu.tw/~hylee/ml/ml2021-course-data/hw/HW04/HW04.pdf)\n", | |||||
" - Data: [link](https://drive.google.com/file/d/1T0RPnu-Sg5eIPwQPfYysipfcz81MnsYe/view?usp=sharing)\n", | |||||
" - Video (Chinese): [link](https://www.youtube.com/watch?v=EPerg2UnGaI)\n", | |||||
" - Video (English): [link](https://www.youtube.com/watch?v=Gpz6AUvCak0)\n", | |||||
" - Solution for downloading dataset fail.: [link](https://drive.google.com/drive/folders/13T0Pa_WGgQxNkqZk781qhc5T9-zfh19e?usp=sharing)" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -36,7 +44,10 @@ | |||||
"id": "TPDoreyypeJE" | "id": "TPDoreyypeJE" | ||||
}, | }, | ||||
"source": [ | "source": [ | ||||
"# Download dataset" | |||||
"# Download dataset\n", | |||||
"- **If all download links fail**\n", | |||||
"- **Please follow [here](https://drive.google.com/drive/folders/13T0Pa_WGgQxNkqZk781qhc5T9-zfh19e?usp=sharing)**\n", | |||||
"- **Data is [here](https://drive.google.com/file/d/1T0RPnu-Sg5eIPwQPfYysipfcz81MnsYe/view?usp=sharing)**" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -63,8 +74,10 @@ | |||||
"# !gdown --id '1MUGBvG_JjqO0C2JYHuyV3B0lvaf1kWIm' --output Dataset.zip\n", | "# !gdown --id '1MUGBvG_JjqO0C2JYHuyV3B0lvaf1kWIm' --output Dataset.zip\n", | ||||
"\"\"\" Download link 7 of Google drive \"\"\"\n", | "\"\"\" Download link 7 of Google drive \"\"\"\n", | ||||
"# !gdown --id '18M91P5DHwILNyOlssZ57AiPOR0OwutOM' --output Dataset.zip\n", | "# !gdown --id '18M91P5DHwILNyOlssZ57AiPOR0OwutOM' --output Dataset.zip\n", | ||||
"\"\"\" For all download links fail, Please paste link into 'Paste link here' \"\"\"\n", | |||||
"# !gdown --id 'Paste link here' --output Dataset.zip\n", | |||||
"\"\"\" For Google drive, you can unzip the data by the command below. \"\"\"\n", | "\"\"\" For Google drive, you can unzip the data by the command below. \"\"\"\n", | ||||
"# !unzip Dataset.zip\n", | |||||
"!unzip Dataset.zip\n", | |||||
"\n", | "\n", | ||||
"\"\"\"\n", | "\"\"\"\n", | ||||
" For Dropbox, we split dataset into five files. \n", | " For Dropbox, we split dataset into five files. \n", | ||||
@@ -82,12 +95,12 @@ | |||||
" For Onedrive, we split dataset into five files. \n", | " For Onedrive, we split dataset into five files. \n", | ||||
" Please download all of them.\n", | " Please download all of them.\n", | ||||
"\"\"\"\n", | "\"\"\"\n", | ||||
"!wget --no-check-certificate \"https://onedrive.live.com/download?cid=10C95EE5FD151BFB&resid=10C95EE5FD151BFB%21106&authkey=ACB6opQR3CG9kmc\" -O Dataset.tar.gz.aa\n", | |||||
"!wget --no-check-certificate \"https://onedrive.live.com/download?cid=93DDDDD552E145DB&resid=93DDDDD552E145DB%21106&authkey=AP6EepjxSdvyV6Y\" -O Dataset.tar.gz.ab\n", | |||||
"!wget --no-check-certificate \"https://onedrive.live.com/download?cid=644545816461BCCC&resid=644545816461BCCC%21106&authkey=ALiefB0kI7Epb0Q\" -O Dataset.tar.gz.ac\n", | |||||
"!wget --no-check-certificate \"https://onedrive.live.com/download?cid=77CEBB3C3C512821&resid=77CEBB3C3C512821%21106&authkey=AAXCx4TTDYC0yjM\" -O Dataset.tar.gz.ad\n", | |||||
"!wget --no-check-certificate \"https://onedrive.live.com/download?cid=383D0E0146A11B02&resid=383D0E0146A11B02%21106&authkey=ALwVc4StVbig6QI\" -O Dataset.tar.gz.ae\n", | |||||
"!cat Dataset.tar.gz.* | tar zxvf -" | |||||
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=10C95EE5FD151BFB&resid=10C95EE5FD151BFB%21106&authkey=ACB6opQR3CG9kmc\" -O Dataset.tar.gz.aa\n", | |||||
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=93DDDDD552E145DB&resid=93DDDDD552E145DB%21106&authkey=AP6EepjxSdvyV6Y\" -O Dataset.tar.gz.ab\n", | |||||
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=644545816461BCCC&resid=644545816461BCCC%21106&authkey=ALiefB0kI7Epb0Q\" -O Dataset.tar.gz.ac\n", | |||||
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=77CEBB3C3C512821&resid=77CEBB3C3C512821%21106&authkey=AAXCx4TTDYC0yjM\" -O Dataset.tar.gz.ad\n", | |||||
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=383D0E0146A11B02&resid=383D0E0146A11B02%21106&authkey=ALwVc4StVbig6QI\" -O Dataset.tar.gz.ae\n", | |||||
"# !cat Dataset.tar.gz.* | tar zxvf -" | |||||
], | ], | ||||
"execution_count": null, | "execution_count": null, | ||||
"outputs": [] | "outputs": [] | ||||