|
|
@@ -108,13 +108,13 @@ |
|
|
|
"base_uri": "https://localhost:8080/" |
|
|
|
}, |
|
|
|
"id": "7LexxyPWWjJB", |
|
|
|
"outputId": "e8d053f0-1587-4721-9e27-0adf60f7a280" |
|
|
|
"outputId": "bac3633f-f293-4477-94e6-61dd92237dd1" |
|
|
|
}, |
|
|
|
"source": [ |
|
|
|
"# Training progress bar\n", |
|
|
|
"!pip install -q qqdm" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 1, |
|
|
|
"outputs": [ |
|
|
|
{ |
|
|
|
"output_type": "stream", |
|
|
@@ -148,7 +148,7 @@ |
|
|
|
"base_uri": "https://localhost:8080/" |
|
|
|
}, |
|
|
|
"id": "r_wwI4CSWvc2", |
|
|
|
"outputId": "0d5952d0-826e-4a73-a612-fdec5707009a" |
|
|
|
"outputId": "70f614de-36d5-4d4e-c58e-28ab04feac55" |
|
|
|
}, |
|
|
|
"source": [ |
|
|
|
"\n", |
|
|
@@ -187,19 +187,15 @@ |
|
|
|
"# 9\n", |
|
|
|
"# !gdown --id '1Uc1Y8YYAwj7D0_wd0MeSX3szUiIB1rLU' --output data-bin.tar.gz " |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 2, |
|
|
|
"outputs": [ |
|
|
|
{ |
|
|
|
"output_type": "stream", |
|
|
|
"text": [ |
|
|
|
"Downloading...\n", |
|
|
|
"From: https://drive.google.com/uc?id=15XWO-zI-AKW0igfwSydmwSGa8ENb9wCg\n", |
|
|
|
"To: /content/gdrive/My Drive/phucque/data-bin.tar.gz\n", |
|
|
|
"1.64GB [00:12, 129MB/s]\n", |
|
|
|
"Downloading...\n", |
|
|
|
"From: https://drive.google.com/uc?id=1mlKBtfSenGu96ny9YDb10l_cEu-DG-gs\n", |
|
|
|
"To: /content/gdrive/My Drive/phucque/testing_labels.npy\n", |
|
|
|
"100% 160k/160k [00:00<00:00, 23.3MB/s]\n" |
|
|
|
"To: /content/data-bin.tar.gz\n", |
|
|
|
"1.64GB [00:16, 98.9MB/s]\n" |
|
|
|
], |
|
|
|
"name": "stdout" |
|
|
|
} |
|
|
@@ -229,7 +225,7 @@ |
|
|
|
"base_uri": "https://localhost:8080/" |
|
|
|
}, |
|
|
|
"id": "0K5kmlkuWzhJ", |
|
|
|
"outputId": "44d9c8e1-cda9-40ed-d85e-9aef6fc9d825" |
|
|
|
"outputId": "16d8d77c-b7e4-417a-c7bd-a6c0b530452e" |
|
|
|
}, |
|
|
|
"source": [ |
|
|
|
"!tar zxvf data-bin.tar.gz\n", |
|
|
@@ -237,7 +233,7 @@ |
|
|
|
"!ls data-bin\n", |
|
|
|
"!rm data-bin.tar.gz" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 3, |
|
|
|
"outputs": [ |
|
|
|
{ |
|
|
|
"output_type": "stream", |
|
|
@@ -245,9 +241,8 @@ |
|
|
|
"data-bin/\n", |
|
|
|
"data-bin/testingset.npy\n", |
|
|
|
"data-bin/trainingset.npy\n", |
|
|
|
"testing_labels.npy testingset.npy trainingset.npy\n", |
|
|
|
"mv: cannot stat 'testing_labels.npy': No such file or directory\n", |
|
|
|
"testing_labels.npy testingset.npy trainingset.npy\n" |
|
|
|
"testingset.npy\ttrainingset.npy\n", |
|
|
|
"testingset.npy\ttrainingset.npy\n" |
|
|
|
], |
|
|
|
"name": "stdout" |
|
|
|
} |
|
|
@@ -294,7 +289,7 @@ |
|
|
|
"\n", |
|
|
|
"\n" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 4, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -313,7 +308,7 @@ |
|
|
|
"base_uri": "https://localhost:8080/" |
|
|
|
}, |
|
|
|
"id": "k7Wd4yiUYzAm", |
|
|
|
"outputId": "28aa0306-c4fc-4ba1-ec75-9f611b1b3304" |
|
|
|
"outputId": "9ea069b4-6a22-46eb-ae4f-60dc6b9d0bb0" |
|
|
|
}, |
|
|
|
"source": [ |
|
|
|
"\n", |
|
|
@@ -323,7 +318,7 @@ |
|
|
|
"print(train.shape)\n", |
|
|
|
"print(test.shape)" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 5, |
|
|
|
"outputs": [ |
|
|
|
{ |
|
|
|
"output_type": "stream", |
|
|
@@ -366,7 +361,7 @@ |
|
|
|
"\n", |
|
|
|
"same_seeds(19530615)" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 6, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -629,7 +624,7 @@ |
|
|
|
"\n", |
|
|
|
" return x_reconst\n" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 7, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -675,7 +670,7 @@ |
|
|
|
" def __len__(self):\n", |
|
|
|
" return len(self.tensors)" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 8, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -728,7 +723,7 @@ |
|
|
|
"optimizer = torch.optim.Adam(\n", |
|
|
|
" model.parameters(), lr=learning_rate)" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 9, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -743,7 +738,12 @@ |
|
|
|
{ |
|
|
|
"cell_type": "code", |
|
|
|
"metadata": { |
|
|
|
"id": "JoW1UrrxgI_U" |
|
|
|
"id": "JoW1UrrxgI_U", |
|
|
|
"colab": { |
|
|
|
"base_uri": "https://localhost:8080/", |
|
|
|
"height": 490 |
|
|
|
}, |
|
|
|
"outputId": "f98d30c4-01cf-4199-84ff-90f87dcce72d" |
|
|
|
}, |
|
|
|
"source": [ |
|
|
|
"\n", |
|
|
@@ -789,8 +789,34 @@ |
|
|
|
"\n", |
|
|
|
"\n" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"outputs": [] |
|
|
|
"execution_count": 10, |
|
|
|
"outputs": [ |
|
|
|
{ |
|
|
|
"output_type": "stream", |
|
|
|
"text": [ |
|
|
|
" \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \n", |
|
|
|
" \u001b[99m0/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m - \u001b[0m \u001b[99m - \u001b[0m \n", |
|
|
|
"\u001b[1mDescription\u001b[0m 0.0% | |\u001b[K\u001b[F\u001b[K\u001b[F \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \u001b[1mepoch\u001b[0m \u001b[1mloss\u001b[0m \n", |
|
|
|
" \u001b[99m1/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m00:00:15<\u001b[93m00:12:25\u001b[0m\u001b[0m \u001b[99m0.07it/s\u001b[0m \u001b[99m1/50\u001b[0m \u001b[99m0.3463\u001b[0m \n", |
|
|
|
"\u001b[1mDescription\u001b[0m 2.0% |\u001b[97m█\u001b[0m |\u001b[K\u001b[F\u001b[K\u001b[F \u001b[1mIters\u001b[0m \u001b[1mElapsed Time\u001b[0m \u001b[1mSpeed\u001b[0m \u001b[1mepoch\u001b[0m \u001b[1mloss\u001b[0m \n", |
|
|
|
" \u001b[99m1/\u001b[93m50\u001b[0m\u001b[0m \u001b[99m00:00:15<\u001b[93m00:12:25\u001b[0m\u001b[0m \u001b[99m0.07it/s\u001b[0m \u001b[99m1/50\u001b[0m \u001b[99m0.3463\u001b[0m \n", |
|
|
|
"\u001b[1mDescription\u001b[0m 2.0% |\u001b[97m█\u001b[0m |" |
|
|
|
], |
|
|
|
"name": "stderr" |
|
|
|
}, |
|
|
|
{ |
|
|
|
"output_type": "error", |
|
|
|
"ename": "KeyboardInterrupt", |
|
|
|
"evalue": "ignored", |
|
|
|
"traceback": [ |
|
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", |
|
|
|
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", |
|
|
|
"\u001b[0;32m<ipython-input-10-6a2514fed04a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mtot_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;31m# ===================backward====================\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", |
|
|
|
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mzero_grad\u001b[0;34m(self, set_to_none)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", |
|
|
|
"\u001b[0;31mKeyboardInterrupt\u001b[0m: " |
|
|
|
] |
|
|
|
} |
|
|
|
] |
|
|
|
}, |
|
|
|
{ |
|
|
|
"cell_type": "markdown", |
|
|
@@ -830,14 +856,14 @@ |
|
|
|
"eval_loss = nn.MSELoss(reduction='none')\n", |
|
|
|
"\n", |
|
|
|
"# load trained model\n", |
|
|
|
"checkpoint_path = 'CHECKPOINT.pt'\n", |
|
|
|
"checkpoint_path = 'last_model_cnn.pt'\n", |
|
|
|
"model = torch.load(checkpoint_path)\n", |
|
|
|
"model.eval()\n", |
|
|
|
"\n", |
|
|
|
"# prediction file \n", |
|
|
|
"out_file = 'PREDICTION_FILE.csv'" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 11, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
@@ -849,7 +875,7 @@ |
|
|
|
" \n", |
|
|
|
"anomality = list()\n", |
|
|
|
"with torch.no_grad():\n", |
|
|
|
" for i, data in enumerate(test_dataloader): \n", |
|
|
|
" for i, data in enumerate(test_dataloader): \n", |
|
|
|
" if model_type in ['cnn', 'vae', 'resnet']:\n", |
|
|
|
" img = data.float().cuda()\n", |
|
|
|
" elif model_type in ['fcn']:\n", |
|
|
@@ -876,7 +902,7 @@ |
|
|
|
"df.to_csv(out_file, index_label = 'Id')\n", |
|
|
|
"\n" |
|
|
|
], |
|
|
|
"execution_count": null, |
|
|
|
"execution_count": 20, |
|
|
|
"outputs": [] |
|
|
|
}, |
|
|
|
{ |
|
|
|