整理”

README.md

@@ -1,2 +0,0 @@
-# test
-

dcu/code/README.md

@@ -0,0 +1,2 @@
+# pytorch-cnn-cifar10-dcu
+

dcu/code/main

@@ -0,0 +1,165 @@
+from torchvision.transforms import transforms
+from torchvision.datasets import CIFAR10
+from torch.utils.data import DataLoader
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.optim import Adam
+from torch.autograd import Variable
+import argparse
+import time
+import shutil
+
+from c2net.context import prepare,upload_output
+import argparse
+
+parser = argparse.ArgumentParser(description='忽略超参数不存在的报错问题')
+#添加自定义参数
+parser.add_argument("--test")
+parser.add_argument('--epoch', type=int, default=1)
+parser.add_argument('--card', type=str, default='cuda:0')
+
+args = parser.parse_args()
+args, unknown = parser.parse_known_args()
+
+#初始化导入数据集和预训练模型到容器内
+c2net_context = prepare()
+
+codePath = c2net_context.code_path
+test = codePath + '/pytorch-cnn-cifar10-dcu' + '/test.py'
+
+#获取数据集路径
+cifar_10_python_path = c2net_context.dataset_path+"/"+"cifar-10-python"
+
+#输出结果必须保存在该目录
+outputPath = c2net_context.output_path
+
+#回传结果到openi，只有训练任务才能回传
+upload_output()
+
+class Network(nn.Module):
+    def __init__(self):
+        super(Network, self).__init__()
+
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn1 = nn.BatchNorm2d(12)
+        self.conv2 = nn.Conv2d(in_channels=12, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn2 = nn.BatchNorm2d(12)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv4 = nn.Conv2d(in_channels=12, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn4 = nn.BatchNorm2d(24)
+        self.conv5 = nn.Conv2d(in_channels=24, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn5 = nn.BatchNorm2d(24)
+        self.fc1 = nn.Linear(24 * 10 * 10, 10)
+
+    def forward(self, input):
+        output = F.relu(self.bn1(self.conv1(input)))
+        output = F.relu(self.bn2(self.conv2(output)))
+        output = self.pool(output)
+        output = F.relu(self.bn4(self.conv4(output)))
+        output = F.relu(self.bn5(self.conv5(output)))
+        output = output.view(-1, 24 * 10 * 10)
+        output = self.fc1(output)
+
+        return output
+
+def saveModel():
+    path = outputPath + '/' + 'test.pth'
+    torch.save(model.state_dict(), path)
+    zipfileName = outputPath + '/' + 'test_database'
+    save_zipfile(zipfileName, outputPath)
+
+
+def testAccuracy(card):
+    model.eval()
+    accuracy = 0.0
+    total = 0.0
+
+    # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    device = torch.device(card)
+    model.to(device)
+
+    test_dataloader = Get_dataloader(False)
+    with torch.no_grad():
+        for data in test_dataloader:
+            images, labels = data
+            images = Variable(images.to(device))
+            labels = Variable(labels.to(device))
+            outputs = model(images)
+            _, predicted = torch.max(outputs.data, 1)
+            total += labels.size(0)
+            accuracy += (predicted == labels).sum().item()
+
+    accuracy = (100 * accuracy / total)
+    return (accuracy)
+
+
+def train(num_epochs, card):
+    # best_accuracy = 0.0
+
+    # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+    device = torch.device(card)
+
+    model.to(device)
+
+    train_dataloader = Get_dataloader(True)
+    for epoch in range(num_epochs):
+        running_loss = 0.0
+        running_acc = 0.0
+
+        for i, (images, labels) in enumerate(train_dataloader, 0):
+
+            images = Variable(images.to(device))
+            labels = Variable(labels.to(device))
+            optimizer.zero_grad()
+            outputs = model(images)
+            loss = loss_fn(outputs, labels)
+            loss.backward()
+            optimizer.step()
+
+            running_loss += loss.item()
+            if i % 1000 == 999:
+                print('[%d, %5d] loss: %.3f' %
+                      (epoch + 1, i + 1, running_loss / 1000))
+                running_loss = 0.0
+
+        accuracy = testAccuracy(card)
+        print('For epoch', epoch + 1, 'the test accuracy over the whole test set is %d %%' % (accuracy))
+
+    # saveModel()
+
+
+def Get_dataloader(train):
+    transform_fn = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+    dataset = CIFAR10(root=DATA_ROOT, train=train, transform=transform_fn)
+    data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=0)
+    return data_loader
+
+def save_zipfile(filename, dest):
+    shutil.make_archive(filename, 'zip', dest)
+
+
+DATA_ROOT = cifar_10_python_path
+classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
+
+batch_size = 10
+number_of_labels = 10
+
+model = Network()
+
+loss_fn = nn.CrossEntropyLoss()
+optimizer = Adam(model.parameters(), lr=0.001, weight_decay=0.0001)
+
+if __name__ == "__main__":
+    accuracy = testAccuracy(args.card)
+    print('before training, accuracy for test data is: ', accuracy)
+
+    start = time.perf_counter()
+    train(args.epoch, args.card)
+    end = time.perf_counter()
+    print(f"training completed in {end - start:0.4f} seconds")
+    saveModel()

dcu/code/test-infer.py

@@ -0,0 +1,119 @@
+import torch
+import torch.nn as nn
+from torchvision import models, transforms
+from PIL import Image
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import Response, JSONResponse
+import io
+import uvicorn
+import torch.nn.functional as F
+import os
+import argparse
+
+from c2net.context import prepare
+# import multiprocessing as mp
+
+# mp.set_start_method('spawn', True)
+
+c2net_context = prepare()
+
+
+files = [f for f in os.listdir(c2net_context.pretrain_model_path) if os.path.isdir(c2net_context.pretrain_model_path+ f"/{f}")]
+model_name = ""
+
+for file in files:
+    if (not file.startswith(".")) and len(os.listdir(c2net_context.pretrain_model_path+ f"/{file}")) > 0:
+        model_name = file
+        break
+
+model_path = c2net_context.pretrain_model_path + f"/{model_name}"
+
+pth_file = ""
+for root, dirs, files in os.walk(model_path):
+    for file in files:
+        if file.endswith(".pth"):
+            pth_file = file
+
+custom_model_path = c2net_context.pretrain_model_path+"/"+ model_name + "/" + pth_file
+
+
+batch_size = 10
+number_of_labels = 10
+classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
+
+base_url = os.getenv('OPENI_SELF_URL')
+
+class Network(nn.Module):
+    def __init__(self):
+        super(Network, self).__init__()
+
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn1 = nn.BatchNorm2d(12)
+        self.conv2 = nn.Conv2d(in_channels=12, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn2 = nn.BatchNorm2d(12)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv4 = nn.Conv2d(in_channels=12, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn4 = nn.BatchNorm2d(24)
+        self.conv5 = nn.Conv2d(in_channels=24, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn5 = nn.BatchNorm2d(24)
+        self.fc1 = nn.Linear(24 * 10 * 10, 10)
+
+    def forward(self, input):
+        output = F.relu(self.bn1(self.conv1(input)))
+        output = F.relu(self.bn2(self.conv2(output)))
+        output = self.pool(output)
+        output = F.relu(self.bn4(self.conv4(output)))
+        output = F.relu(self.bn5(self.conv5(output)))
+        output = output.view(-1, 24 * 10 * 10)
+        output = self.fc1(output)
+
+        return output
+
+app = FastAPI()
+
+TIMEOUT_KEEP_ALIVE = 20
+
+model = Network()
+
+transform_fn = transforms.Compose([
+        transforms.Resize(32),
+        transforms.CenterCrop(32),
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+])
+
+def inference_one_image(image, card):
+    image = image.to(card)
+    model_path = custom_model_path
+    model.load_state_dict(torch.load(model_path))
+
+    model.eval()
+    device = torch.device(card)
+    model.to(device)
+
+    output = model(image)
+
+    _, index = torch.max(output, 1)
+    return index
+
+@app.get(os.path.join(base_url,"test"))
+async def api_get():
+    return "This is a GET request."
+
+@app.post(os.path.join(base_url,"infer"))
+async def classification_cnn(file: UploadFile = File(...)) -> Response:
+    file_content = await file.read()
+    image = Image.open(io.BytesIO(file_content))
+    img = transform_fn(image)
+    batch = torch.unsqueeze(img, 0)
+
+    predicted = inference_one_image(batch, "cuda:0")
+
+    if predicted is None:
+        ret = {"error": "predicted is None"}
+    else:
+        ret = {"result": classes[predicted]}
+    return JSONResponse(ret)
+
+if __name__ == '__main__':
+    uvicorn.run(app, host='0.0.0.0', port=int(os.getenv('OPENI_SELF_PORT')), timeout_keep_alive=TIMEOUT_KEEP_ALIVE)

dcu/code/test.py

@@ -0,0 +1,49 @@
+import argparse
+import uvicorn
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import Response, JSONResponse
+import os
+
+from c2net.context import prepare
+import os
+c2net_context = prepare()
+
+files = [f for f in os.listdir(c2net_context.pretrain_model_path) if os.path.isdir(c2net_context.pretrain_model_path+ f"/{f}")]
+model_name = ""
+
+for file in files:
+    if (not file.startswith(".")) and len(os.listdir(c2net_context.pretrain_model_path+ f"/{file}")) > 0:
+        model_name = file
+        break
+
+model_path = c2net_context.pretrain_model_path + f"/{model_name}"
+
+pth_file = ""
+for root, dirs, files in os.walk(model_path):
+    for file in files:
+        if file.endswith(".pth"):
+            pth_file = file
+
+parser = argparse.ArgumentParser(description='忽略超参数不存在的报错问题')
+# #添加自定义参数
+parser.add_argument("--output")
+
+args = parser.parse_args()
+args, unknown = parser.parse_known_args()
+
+TIMEOUT_KEEP_ALIVE = 20
+base_url = os.getenv('OPENI_SELF_URL')
+
+app = FastAPI()
+
+@app.get(os.path.join(base_url,"test"))
+async def api_get():
+    return model_name + "|" + model_path + "|" + pth_file
+
+if __name__ == '__main__':
+    print("this is a test")
+    print(args)
+    print(unknown)
+    print(args.output)
+        
+    uvicorn.run(app, host='0.0.0.0', port=int(os.getenv('OPENI_SELF_PORT')), timeout_keep_alive=TIMEOUT_KEEP_ALIVE)

dcu/code/train.py

@@ -0,0 +1,164 @@
+from torchvision.transforms import transforms
+from torchvision.datasets import CIFAR10
+from torch.utils.data import DataLoader
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.optim import Adam
+from torch.autograd import Variable
+import argparse
+import time
+import shutil
+
+from c2net.context import prepare,upload_output
+
+parser = argparse.ArgumentParser(description='忽略超参数不存在的报错问题')
+#添加自定义参数
+parser.add_argument("--output")
+parser.add_argument('--epoch', type=int, default=1)
+parser.add_argument('--card', type=str, default='cuda:0')
+
+args = parser.parse_args()
+args, unknown = parser.parse_known_args()
+
+#初始化导入数据集和预训练模型到容器内
+c2net_context = prepare()
+
+# codePath = c2net_context.code_path
+# test = codePath + '/pytorch-cnn-cifar10-dcu' + '/test.py'
+
+#获取数据集路径
+cifar_10_python_path = c2net_context.dataset_path+"/"+"jo-161M"
+
+#输出结果必须保存在该目录
+outputPath = c2net_context.output_path
+
+#回传结果到openi，只有训练任务才能回传
+upload_output()
+
+class Network(nn.Module):
+    def __init__(self):
+        super(Network, self).__init__()
+
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn1 = nn.BatchNorm2d(12)
+        self.conv2 = nn.Conv2d(in_channels=12, out_channels=12, kernel_size=5, stride=1, padding=1)
+        self.bn2 = nn.BatchNorm2d(12)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv4 = nn.Conv2d(in_channels=12, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn4 = nn.BatchNorm2d(24)
+        self.conv5 = nn.Conv2d(in_channels=24, out_channels=24, kernel_size=5, stride=1, padding=1)
+        self.bn5 = nn.BatchNorm2d(24)
+        self.fc1 = nn.Linear(24 * 10 * 10, 10)
+
+    def forward(self, input):
+        output = F.relu(self.bn1(self.conv1(input)))
+        output = F.relu(self.bn2(self.conv2(output)))
+        output = self.pool(output)
+        output = F.relu(self.bn4(self.conv4(output)))
+        output = F.relu(self.bn5(self.conv5(output)))
+        output = output.view(-1, 24 * 10 * 10)
+        output = self.fc1(output)
+
+        return output
+
+def saveModel():
+    path = outputPath + '/' + 'test.pth'
+    torch.save(model.state_dict(), path)
+    zipfileName = outputPath + '/' + 'test_database'
+    save_zipfile(zipfileName, outputPath)
+
+
+def testAccuracy(card):
+    model.eval()
+    accuracy = 0.0
+    total = 0.0
+
+    # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    device = torch.device(card)
+    model.to(device)
+
+    test_dataloader = Get_dataloader(False)
+    with torch.no_grad():
+        for data in test_dataloader:
+            images, labels = data
+            images = Variable(images.to(device))
+            labels = Variable(labels.to(device))
+            outputs = model(images)
+            _, predicted = torch.max(outputs.data, 1)
+            total += labels.size(0)
+            accuracy += (predicted == labels).sum().item()
+
+    accuracy = (100 * accuracy / total)
+    return (accuracy)
+
+
+def train(num_epochs, card):
+    # best_accuracy = 0.0
+
+    # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+    device = torch.device(card)
+
+    model.to(device)
+
+    train_dataloader = Get_dataloader(True)
+    for epoch in range(num_epochs):
+        running_loss = 0.0
+        running_acc = 0.0
+
+        for i, (images, labels) in enumerate(train_dataloader, 0):
+
+            images = Variable(images.to(device))
+            labels = Variable(labels.to(device))
+            optimizer.zero_grad()
+            outputs = model(images)
+            loss = loss_fn(outputs, labels)
+            loss.backward()
+            optimizer.step()
+
+            running_loss += loss.item()
+            if i % 1000 == 999:
+                print('[%d, %5d] loss: %.3f' %
+                      (epoch + 1, i + 1, running_loss / 1000))
+                running_loss = 0.0
+
+        accuracy = testAccuracy(card)
+        print('For epoch', epoch + 1, 'the test accuracy over the whole test set is %d %%' % (accuracy))
+
+    # saveModel()
+
+
+def Get_dataloader(train):
+    transform_fn = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+    dataset = CIFAR10(root=DATA_ROOT, train=train, transform=transform_fn)
+    data_loader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True, num_workers=0)
+    return data_loader
+
+def save_zipfile(filename, dest):
+    shutil.make_archive(filename, 'zip', dest)
+
+
+DATA_ROOT = cifar_10_python_path
+classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
+
+batch_size = 10
+number_of_labels = 10
+
+model = Network()
+
+loss_fn = nn.CrossEntropyLoss()
+optimizer = Adam(model.parameters(), lr=0.001, weight_decay=0.0001)
+
+if __name__ == "__main__":
+    accuracy = testAccuracy(args.card)
+    print('before training, accuracy for test data is: ', accuracy)
+
+    start = time.perf_counter()
+    train(args.epoch, args.card)
+    end = time.perf_counter()
+    print(f"training completed in {end - start:0.4f} seconds")
+    saveModel()