hummingbird
/
Lhy_Machine_Learning
Not watched
2
0
Fork 0
Code
Releases 3 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

update

main
Fafa-DL 3 years ago
parent
624481cbc4
commit
4378b32534
10 changed files with 3368 additions and 86 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -48
      范例/HW04/HW04.ipynb
  2. +9
    -9
      范例/HW07/HW07.ipynb
  3. BIN
      范例/HW07/HW07.pdf
  4. +55
    -29
      范例/HW08/HW08.ipynb
  5. BIN
      范例/HW08/HW08.pdf
  6. BIN
      范例/HW08/hw8_slides.pdf
  7. +3299
    -0
      范例/HW09/HW09.ipynb
  8. BIN
      范例/HW09/HW09.pdf
  9. +1
    -0
      范例/HW10/HW10.ipynb
  10. BIN
      范例/HW10/HW10.pdf

+ 4
- 48
范例/HW04/HW04.ipynb View File

@@ -45,9 +45,8 @@
},
"source": [
"# Download dataset\n",
"- **If all download links fail**\n",
"- **Please follow [here](https://drive.google.com/drive/folders/13T0Pa_WGgQxNkqZk781qhc5T9-zfh19e?usp=sharing)**\n",
"- **Data is [here](https://drive.google.com/file/d/1T0RPnu-Sg5eIPwQPfYysipfcz81MnsYe/view?usp=sharing)**"
"- Please follow [here](https://drive.google.com/drive/folders/13T0Pa_WGgQxNkqZk781qhc5T9-zfh19e?usp=sharing) to download data\n",
"- Data is [here](https://drive.google.com/file/d/1gaFy8RaQVUEXo2n0peCBR5gYKCB-mNHc/view?usp=sharing)"
]
},
{
@@ -56,51 +55,8 @@
"id": "QvpaILXnJIcw"
},
"source": [
"\"\"\"\n",
" For Google drive, You can download data form any link below.\n",
" If a link fails, please use another one.\n",
"\"\"\"\n",
"\"\"\" Download link 1 of Google drive \"\"\"\n",
"# !gdown --id '1T0RPnu-Sg5eIPwQPfYysipfcz81MnsYe' --output Dataset.zip\n",
"\"\"\" Download link 2 of Google drive \"\"\"\n",
"# !gdown --id '1CtHZhJ-mTpNsO-MqvAPIi4Yrt3oSBXYV' --output Dataset.zip\n",
"\"\"\" Download link 3 of Google drive \"\"\"\n",
"# !gdown --id '14hmoMgB1fe6v50biIceKyndyeYABGrRq' --output Dataset.zip\n",
"\"\"\" Download link 4 of Google drive \"\"\"\n",
"# !gdown --id '1e9x-Pjl3n7-9tK9LS_WjiMo2lru4UBH9' --output Dataset.zip\n",
"\"\"\" Download link 5 of Google drive \"\"\"\n",
"# !gdown --id '10TC0g46bcAz_jkiMl65zNmwttT4RiRgY' --output Dataset.zip\n",
"\"\"\" Download link 6 of Google drive \"\"\"\n",
"# !gdown --id '1MUGBvG_JjqO0C2JYHuyV3B0lvaf1kWIm' --output Dataset.zip\n",
"\"\"\" Download link 7 of Google drive \"\"\"\n",
"# !gdown --id '18M91P5DHwILNyOlssZ57AiPOR0OwutOM' --output Dataset.zip\n",
"\"\"\" For all download links fail, Please paste link into 'Paste link here' \"\"\"\n",
"# !gdown --id 'Paste link here' --output Dataset.zip\n",
"\"\"\" For Google drive, you can unzip the data by the command below. \"\"\"\n",
"!unzip Dataset.zip\n",
"\n",
"\"\"\"\n",
" For Dropbox, we split dataset into five files. \n",
" Please download all of them.\n",
"\"\"\"\n",
"# If Dropbox is not work. Please use google drive.\n",
"# !wget https://www.dropbox.com/s/vw324newiku0sz0/Dataset.tar.gz.aa?dl=0\n",
"# !wget https://www.dropbox.com/s/z840g69e7lnkayo/Dataset.tar.gz.ab?dl=0\n",
"# !wget https://www.dropbox.com/s/hl081e1ggonio81/Dataset.tar.gz.ac?dl=0\n",
"# !wget https://www.dropbox.com/s/fh3zd8ow668c4th/Dataset.tar.gz.ad?dl=0\n",
"# !wget https://www.dropbox.com/s/ydzygoy2pv6gw9d/Dataset.tar.gz.ae?dl=0\n",
"# !cat Dataset.tar.gz.* | tar zxvf -\n",
"\n",
"\"\"\"\n",
" For Onedrive, we split dataset into five files. \n",
" Please download all of them.\n",
"\"\"\"\n",
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=10C95EE5FD151BFB&resid=10C95EE5FD151BFB%21106&authkey=ACB6opQR3CG9kmc\" -O Dataset.tar.gz.aa\n",
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=93DDDDD552E145DB&resid=93DDDDD552E145DB%21106&authkey=AP6EepjxSdvyV6Y\" -O Dataset.tar.gz.ab\n",
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=644545816461BCCC&resid=644545816461BCCC%21106&authkey=ALiefB0kI7Epb0Q\" -O Dataset.tar.gz.ac\n",
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=77CEBB3C3C512821&resid=77CEBB3C3C512821%21106&authkey=AAXCx4TTDYC0yjM\" -O Dataset.tar.gz.ad\n",
"# !wget --no-check-certificate \"https://onedrive.live.com/download?cid=383D0E0146A11B02&resid=383D0E0146A11B02%21106&authkey=ALwVc4StVbig6QI\" -O Dataset.tar.gz.ae\n",
"# !cat Dataset.tar.gz.* | tar zxvf -"
"!gdown --id 'paste your own data download link' --output Dataset.zip\n",
"!unzip Dataset.zip"
],
"execution_count": null,
"outputs": []


+ 9
- 9
范例/HW07/HW07.ipynb View File

@@ -166,7 +166,7 @@
"\t torch.backends.cudnn.deterministic = True\n",
"same_seeds(0)"
],
"execution_count": 16,
"execution_count": null,
"outputs": []
},
{
@@ -186,7 +186,7 @@
"\n",
"# Documentation for the toolkit: https://huggingface.co/docs/accelerate/"
],
"execution_count": 17,
"execution_count": null,
"outputs": []
},
{
@@ -258,7 +258,7 @@
"dev_questions, dev_paragraphs = read_data(\"hw7_dev.json\")\n",
"test_questions, test_paragraphs = read_data(\"hw7_test.json\")"
],
"execution_count": 19,
"execution_count": null,
"outputs": []
},
{
@@ -399,7 +399,7 @@
"dev_loader = DataLoader(dev_set, batch_size=1, shuffle=False, pin_memory=True)\n",
"test_loader = DataLoader(test_set, batch_size=1, shuffle=False, pin_memory=True)"
],
"execution_count": 21,
"execution_count": null,
"outputs": []
},
{
@@ -443,7 +443,7 @@
" # Remove spaces in answer (e.g. \"大 金\" --> \"大金\")\n",
" return answer.replace(' ','')"
],
"execution_count": 22,
"execution_count": null,
"outputs": []
},
{
@@ -516,8 +516,8 @@
" with torch.no_grad():\n",
" dev_acc = 0\n",
" for i, data in enumerate(tqdm(dev_loader)):\n",
" output = model(input_ids=data[0].squeeze().to(device), token_type_ids=data[1].squeeze().to(device),\n",
" attention_mask=data[2].squeeze().to(device))\n",
" output = model(input_ids=data[0].squeeze(dim=0).to(device), token_type_ids=data[1].squeeze(dim=0).to(device),\n",
" attention_mask=data[2].squeeze(dim=0).to(device))\n",
" # prediction is correct only if answer text exactly matches\n",
" dev_acc += evaluate(data, output) == dev_questions[i][\"answer_text\"]\n",
" print(f\"Validation | Epoch {epoch + 1} | acc = {dev_acc / len(dev_loader):.3f}\")\n",
@@ -555,8 +555,8 @@
"model.eval()\n",
"with torch.no_grad():\n",
" for data in tqdm(test_loader):\n",
" output = model(input_ids=data[0].squeeze().to(device), token_type_ids=data[1].squeeze().to(device),\n",
" attention_mask=data[2].squeeze().to(device))\n",
" output = model(input_ids=data[0].squeeze(dim=0).to(device), token_type_ids=data[1].squeeze(dim=0).to(device),\n",
" attention_mask=data[2].squeeze(dim=0).to(device))\n",
" result.append(evaluate(data, output))\n",
"\n",
"result_file = \"result.csv\"\n",


BIN
范例/HW07/HW07.pdf View File


+ 55
- 29
范例/HW08/HW08.ipynb View File

@@ -108,13 +108,13 @@
"base_uri": "https://localhost:8080/"
},
"id": "7LexxyPWWjJB",
"outputId": "e8d053f0-1587-4721-9e27-0adf60f7a280"
"outputId": "bac3633f-f293-4477-94e6-61dd92237dd1"
},
"source": [
"# Training progress bar\n",
"!pip install -q qqdm"
],
"execution_count": null,
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
@@ -148,7 +148,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "r_wwI4CSWvc2",
"outputId": "0d5952d0-826e-4a73-a612-fdec5707009a"
"outputId": "70f614de-36d5-4d4e-c58e-28ab04feac55"
},
"source": [
"\n",
@@ -187,19 +187,15 @@
"# 9\n",
"# !gdown --id '1Uc1Y8YYAwj7D0_wd0MeSX3szUiIB1rLU' --output data-bin.tar.gz "
],
"execution_count": null,
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"text": [
"Downloading...\n",
"From: https://drive.google.com/uc?id=15XWO-zI-AKW0igfwSydmwSGa8ENb9wCg\n",
"To: /content/gdrive/My Drive/phucque/data-bin.tar.gz\n",
"1.64GB [00:12, 129MB/s]\n",
"Downloading...\n",
"From: https://drive.google.com/uc?id=1mlKBtfSenGu96ny9YDb10l_cEu-DG-gs\n",
"To: /content/gdrive/My Drive/phucque/testing_labels.npy\n",
"100% 160k/160k [00:00<00:00, 23.3MB/s]\n"
"To: /content/data-bin.tar.gz\n",
"1.64GB [00:16, 98.9MB/s]\n"
],
"name": "stdout"
}
@@ -229,7 +225,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "0K5kmlkuWzhJ",
"outputId": "44d9c8e1-cda9-40ed-d85e-9aef6fc9d825"
"outputId": "16d8d77c-b7e4-417a-c7bd-a6c0b530452e"
},
"source": [
"!tar zxvf data-bin.tar.gz\n",
@@ -237,7 +233,7 @@
"!ls data-bin\n",
"!rm data-bin.tar.gz"
],
"execution_count": null,
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
@@ -245,9 +241,8 @@
"data-bin/\n",
"data-bin/testingset.npy\n",
"data-bin/trainingset.npy\n",
"testing_labels.npy testingset.npy trainingset.npy\n",
"mv: cannot stat 'testing_labels.npy': No such file or directory\n",
"testing_labels.npy testingset.npy trainingset.npy\n"
"testingset.npy\ttrainingset.npy\n",
"testingset.npy\ttrainingset.npy\n"
],
"name": "stdout"
}
@@ -294,7 +289,7 @@
"\n",
"\n"
],
"execution_count": null,
"execution_count": 4,
"outputs": []
},
{
@@ -313,7 +308,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "k7Wd4yiUYzAm",
"outputId": "28aa0306-c4fc-4ba1-ec75-9f611b1b3304"
"outputId": "9ea069b4-6a22-46eb-ae4f-60dc6b9d0bb0"
},
"source": [
"\n",
@@ -323,7 +318,7 @@
"print(train.shape)\n",
"print(test.shape)"
],
"execution_count": null,
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
@@ -366,7 +361,7 @@
"\n",
"same_seeds(19530615)"
],
"execution_count": null,
"execution_count": 6,
"outputs": []
},
{
@@ -629,7 +624,7 @@
"\n",
" return x_reconst\n"
],
"execution_count": null,
"execution_count": 7,
"outputs": []
},
{
@@ -675,7 +670,7 @@
" def __len__(self):\n",
" return len(self.tensors)"
],
"execution_count": null,
"execution_count": 8,
"outputs": []
},
{
@@ -728,7 +723,7 @@
"optimizer = torch.optim.Adam(\n",
" model.parameters(), lr=learning_rate)"
],
"execution_count": null,
"execution_count": 9,
"outputs": []
},
{
@@ -743,7 +738,12 @@
{
"cell_type": "code",
"metadata": {
"id": "JoW1UrrxgI_U"
"id": "JoW1UrrxgI_U",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 490
},
"outputId": "f98d30c4-01cf-4199-84ff-90f87dcce72d"
},
"source": [
"\n",
@@ -789,8 +789,34 @@
"\n",
"\n"
],
"execution_count": null,
"outputs": []
"execution_count": 10,
"outputs": [
{
"output_type": "stream",
"text": [
" \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \n",
" \u001b[99m0/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m - \u001b[0m \u001b[99m - \u001b[0m \n",
"\u001b[1mDescription\u001b[0m 0.0% | |\u001b[K\u001b[F\u001b[K\u001b[F \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \u001b[1mepoch\u001b[0m \u001b[1mloss\u001b[0m \n",
" \u001b[99m1/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m00:00:15<\u001b[93m00:12:25\u001b[0m\u001b[0m \u001b[99m0.07it/s\u001b[0m \u001b[99m1/50\u001b[0m \u001b[99m0.3463\u001b[0m \n",
"\u001b[1mDescription\u001b[0m 2.0% |\u001b[97m█\u001b[0m |\u001b[K\u001b[F\u001b[K\u001b[F \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \u001b[1mepoch\u001b[0m \u001b[1mloss\u001b[0m \n",
" \u001b[99m1/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m00:00:15<\u001b[93m00:12:25\u001b[0m\u001b[0m \u001b[99m0.07it/s\u001b[0m \u001b[99m1/50\u001b[0m \u001b[99m0.3463\u001b[0m \n",
"\u001b[1mDescription\u001b[0m 2.0% |\u001b[97m█\u001b[0m |"
],
"name": "stderr"
},
{
"output_type": "error",
"ename": "KeyboardInterrupt",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-10-6a2514fed04a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mtot_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;31m# ===================backward====================\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mzero_grad\u001b[0;34m(self, set_to_none)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
]
},
{
"cell_type": "markdown",
@@ -830,14 +856,14 @@
"eval_loss = nn.MSELoss(reduction='none')\n",
"\n",
"# load trained model\n",
"checkpoint_path = 'CHECKPOINT.pt'\n",
"checkpoint_path = 'last_model_cnn.pt'\n",
"model = torch.load(checkpoint_path)\n",
"model.eval()\n",
"\n",
"# prediction file \n",
"out_file = 'PREDICTION_FILE.csv'"
],
"execution_count": null,
"execution_count": 11,
"outputs": []
},
{
@@ -849,7 +875,7 @@
" \n",
"anomality = list()\n",
"with torch.no_grad():\n",
" for i, data in enumerate(test_dataloader): \n",
" for i, data in enumerate(test_dataloader): \n",
" if model_type in ['cnn', 'vae', 'resnet']:\n",
" img = data.float().cuda()\n",
" elif model_type in ['fcn']:\n",
@@ -876,7 +902,7 @@
"df.to_csv(out_file, index_label = 'Id')\n",
"\n"
],
"execution_count": null,
"execution_count": 20,
"outputs": []
},
{


BIN
范例/HW08/HW08.pdf View File


BIN
范例/HW08/hw8_slides.pdf View File


+ 3299
- 0
范例/HW09/HW09.ipynb
File diff suppressed because it is too large
View File


BIN
范例/HW09/HW09.pdf View File


+ 1
- 0
范例/HW10/HW10.ipynb
File diff suppressed because it is too large
View File


BIN
范例/HW10/HW10.pdf View File


Write Preview
Loading…
Cancel
Save