@@ -0,0 +1,149 @@ | |||
""" | |||
######################## train lenet example ######################## | |||
train lenet and get network model files(.ckpt) | |||
""" | |||
#!/usr/bin/python | |||
#coding=utf-8 | |||
import os | |||
import argparse | |||
import moxing as mox | |||
from config import mnist_cfg as cfg | |||
from dataset import create_dataset | |||
from dataset_distributed import create_dataset_parallel | |||
from lenet import LeNet5 | |||
import json | |||
import mindspore.nn as nn | |||
from mindspore import context | |||
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor | |||
from mindspore.train import Model | |||
from mindspore.nn.metrics import Accuracy | |||
from mindspore import load_checkpoint, load_param_into_net | |||
from mindspore.context import ParallelMode | |||
from mindspore.communication.management import init, get_rank | |||
import time | |||
### Copy multiple datasets from obs to training image ### | |||
def MultiObsToEnv(multi_data_url, data_dir): | |||
#--multi_data_url is json data, need to do json parsing for multi_data_url | |||
multi_data_json = json.loads(multi_data_url) | |||
for i in range(len(multi_data_json)): | |||
path = data_dir + "/" + multi_data_json[i]["dataset_name"] | |||
file_path = data_dir + "/" + os.path.splitext(multi_data_json[i]["dataset_name"])[0] | |||
if not os.path.exists(file_path): | |||
os.makedirs(file_path) | |||
try: | |||
mox.file.copy_parallel(multi_data_json[i]["dataset_url"], path) | |||
print("Successfully Download {} to {}".format(multi_data_json[i]["dataset_url"],path)) | |||
#unzip dataset | |||
os.system("unzip -d %s %s" % (file_path, path)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format( | |||
multi_data_json[i]["dataset_url"], path) + str(e)) | |||
#Set a cache file to determine whether the data has been copied to obs. | |||
#If this file exists during multi-card training, there is no need to copy the dataset multiple times. | |||
f = open("/cache/download_input.txt", 'w') | |||
f.close() | |||
try: | |||
if os.path.exists("/cache/download_input.txt"): | |||
print("download_input succeed") | |||
except Exception as e: | |||
print("download_input failed") | |||
return | |||
def DownloadFromQizhi(multi_data_url, data_dir): | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) | |||
if device_num > 1: | |||
# set device_id and init for multi-card training | |||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) | |||
context.reset_auto_parallel_context() | |||
context.set_auto_parallel_context(device_num = device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, parameter_broadcast=True) | |||
init() | |||
#Copying obs data does not need to be executed multiple times, just let the 0th card copy the data | |||
local_rank=int(os.getenv('RANK_ID')) | |||
if local_rank%8==0: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
#If the cache file does not exist, it means that the copy data has not been completed, | |||
#and Wait for 0th card to finish copying data | |||
while not os.path.exists("/cache/download_input.txt"): | |||
time.sleep(1) | |||
return | |||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example') | |||
### --multi_data_url,--ckpt_url,--device_target,These 4 parameters must be defined first in a multi-dataset, | |||
### otherwise an error will be reported. | |||
### There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
### because they are predefined in the background, you only need to define them in your code. | |||
parser.add_argument('--multi_data_url', | |||
help='dataset path in obs') | |||
parser.add_argument('--ckpt_url', | |||
help='pre_train_model path in obs') | |||
parser.add_argument( | |||
'--device_target', | |||
type=str, | |||
default="Ascend", | |||
choices=['Ascend', 'CPU'], | |||
help='device where the code will be implemented (default: Ascend),if to use the CPU on the Qizhi platform:device_target=CPU') | |||
parser.add_argument('--epoch_size', | |||
type=int, | |||
default=5, | |||
help='Training epochs.') | |||
if __name__ == "__main__": | |||
args, unknown = parser.parse_known_args() | |||
data_dir = '/cache/dataset' | |||
train_dir = '/cache/output' | |||
if not os.path.exists(data_dir): | |||
os.makedirs(data_dir) | |||
if not os.path.exists(train_dir): | |||
os.makedirs(train_dir) | |||
###Initialize and copy data to training image | |||
DownloadFromQizhi(args.multi_data_url, data_dir) | |||
###The dataset path is used here:data_dir + "/MNIST_Data" +"/train" | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) | |||
if device_num > 1: | |||
ds_train = create_dataset_parallel(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) | |||
if ds_train.get_dataset_size() == 0: | |||
raise ValueError( | |||
"Please check dataset size > 0 and batch_size <= dataset size") | |||
network = LeNet5(cfg.num_classes) | |||
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") | |||
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) | |||
time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) | |||
if args.device_target != "Ascend": | |||
model = Model(network,net_loss,net_opt,metrics={"accuracy": Accuracy()}) | |||
else: | |||
model = Model(network, net_loss,net_opt,metrics={"accuracy": Accuracy()},amp_level="O2") | |||
config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, | |||
keep_checkpoint_max=cfg.keep_checkpoint_max) | |||
#Note that this method saves the model file on each card. You need to specify the save path on each card. | |||
# In this example, get_rank() is added to distinguish different paths. | |||
if device_num == 1: | |||
outputDirectory = train_dir + "/" | |||
if device_num > 1: | |||
outputDirectory = train_dir + "/" + str(get_rank()) + "/" | |||
ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", | |||
directory=outputDirectory, | |||
config=config_ck) | |||
print("============== Starting Training ==============") | |||
epoch_size = cfg['epoch_size'] | |||
if (args.epoch_size): | |||
epoch_size = args.epoch_size | |||
print('epoch_size is: ', epoch_size) | |||
model.train(epoch_size, | |||
ds_train, | |||
callbacks=[time_cb, ckpoint_cb, | |||
LossMonitor()]) | |||
@@ -0,0 +1,197 @@ | |||
""" | |||
######################## multi-dataset train lenet example ######################## | |||
This example is a multi-dataset training tutorial. If it is a single dataset, please refer to the single dataset | |||
training tutorial train.py. This example cannot be used for a single dataset! | |||
""" | |||
""" | |||
######################## Instructions for using the training environment ######################## | |||
1、(1)The structure of the dataset uploaded for multi-dataset training in this example | |||
MNISTData.zip | |||
├── test | |||
└── train | |||
checkpoint_lenet-1_1875.zip | |||
├── checkpoint_lenet-1_1875.ckpt | |||
(2)The dataset structure in the training image for multiple datasets in this example | |||
workroot | |||
├── MNISTData | |||
| ├── test | |||
| └── train | |||
└── checkpoint_lenet-1_1875 | |||
├── checkpoint_lenet-1_1875.ckpt | |||
2、Multi-dataset training requires predefined functions | |||
(1)Copy multi-dataset from obs to training image | |||
function MultiObsToEnv(multi_data_url, data_dir) | |||
(2)Copy the output to obs | |||
function EnvToObs(train_dir, obs_train_url) | |||
(2)Download the input from Qizhi And Init | |||
function DownloadFromQizhi(multi_data_url, data_dir) | |||
(2)Upload the output to Qizhi | |||
function UploadToQizhi(train_dir, obs_train_url) | |||
3、4 parameters need to be defined | |||
--data_url is the first dataset you selected on the Qizhi platform | |||
--multi_data_url is the multi-dataset you selected on the Qizhi platform | |||
--data_url,--multi_data_url,--train_url,--device_target,These 4 parameters must be defined first in a multi-dataset task, | |||
otherwise an error will be reported. | |||
There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
because they are predefined in the background, you only need to define them in your code | |||
4、How the dataset is used | |||
Multi-datasets use multi_data_url as input, data_dir + dataset name + file or folder name in the dataset as the | |||
calling path of the dataset in the training image. | |||
For example, the calling path of the train folder in the MNIST_Data dataset in this example is | |||
data_dir + "/MNIST_Data" +"/train" | |||
For details, please refer to the following sample code. | |||
""" | |||
import os | |||
import argparse | |||
import moxing as mox | |||
from config import mnist_cfg as cfg | |||
from dataset import create_dataset | |||
from dataset_distributed import create_dataset_parallel | |||
from lenet import LeNet5 | |||
import json | |||
import mindspore.nn as nn | |||
from mindspore import context | |||
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor | |||
from mindspore.train import Model | |||
from mindspore.nn.metrics import Accuracy | |||
from mindspore import load_checkpoint, load_param_into_net | |||
from mindspore.context import ParallelMode | |||
from mindspore.communication.management import init, get_rank | |||
import time | |||
### Copy multiple datasets from obs to training image ### | |||
def MultiObsToEnv(multi_data_url, data_dir): | |||
#--multi_data_url is json data, need to do json parsing for multi_data_url | |||
multi_data_json = json.loads(multi_data_url) | |||
for i in range(len(multi_data_json)): | |||
path = data_dir + "/" + multi_data_json[i]["dataset_name"] | |||
file_path = data_dir + "/" + os.path.splitext(multi_data_json[i]["dataset_name"])[0] | |||
if not os.path.exists(file_path): | |||
os.makedirs(file_path) | |||
try: | |||
mox.file.copy_parallel(multi_data_json[i]["dataset_url"], path) | |||
print("Successfully Download {} to {}".format(multi_data_json[i]["dataset_url"],path)) | |||
#unzip dataset | |||
os.system("unzip -d %s %s" % (file_path, path)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format( | |||
multi_data_json[i]["dataset_url"], path) + str(e)) | |||
#Set a cache file to determine whether the data has been copied to obs. | |||
#If this file exists during multi-card training, there is no need to copy the dataset multiple times. | |||
f = open("/cache/download_input.txt", 'w') | |||
f.close() | |||
try: | |||
if os.path.exists("/cache/download_input.txt"): | |||
print("download_input succeed") | |||
except Exception as e: | |||
print("download_input failed") | |||
return | |||
def DownloadFromQizhi(multi_data_url, data_dir): | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) | |||
if device_num > 1: | |||
# set device_id and init for multi-card training | |||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) | |||
context.reset_auto_parallel_context() | |||
context.set_auto_parallel_context(device_num = device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, parameter_broadcast=True) | |||
init() | |||
#Copying obs data does not need to be executed multiple times, just let the 0th card copy the data | |||
local_rank=int(os.getenv('RANK_ID')) | |||
if local_rank%8==0: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
#If the cache file does not exist, it means that the copy data has not been completed, | |||
#and Wait for 0th card to finish copying data | |||
while not os.path.exists("/cache/download_input.txt"): | |||
time.sleep(1) | |||
return | |||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example') | |||
### --multi_data_url,--ckpt_url,--device_target,These 4 parameters must be defined first in a multi-dataset, | |||
### otherwise an error will be reported. | |||
### There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
### because they are predefined in the background, you only need to define them in your code. | |||
parser.add_argument('--multi_data_url', | |||
help='dataset path in obs') | |||
parser.add_argument('--ckpt_url', | |||
help='pre_train_model path in obs') | |||
parser.add_argument( | |||
'--device_target', | |||
type=str, | |||
default="Ascend", | |||
choices=['Ascend', 'CPU'], | |||
help='device where the code will be implemented (default: Ascend),if to use the CPU on the Qizhi platform:device_target=CPU') | |||
parser.add_argument('--epoch_size', | |||
type=int, | |||
default=5, | |||
help='Training epochs.') | |||
if __name__ == "__main__": | |||
args, unknown = parser.parse_known_args() | |||
data_dir = '/cache/dataset' | |||
train_dir = '/cache/output' | |||
if not os.path.exists(data_dir): | |||
os.makedirs(data_dir) | |||
if not os.path.exists(train_dir): | |||
os.makedirs(train_dir) | |||
###Initialize and copy data to training image | |||
DownloadFromQizhi(args.multi_data_url, data_dir) | |||
###The dataset path is used here:data_dir + "/MNIST_Data" +"/train" | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) | |||
if device_num > 1: | |||
ds_train = create_dataset_parallel(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) | |||
if ds_train.get_dataset_size() == 0: | |||
raise ValueError( | |||
"Please check dataset size > 0 and batch_size <= dataset size") | |||
network = LeNet5(cfg.num_classes) | |||
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") | |||
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) | |||
time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) | |||
###The dataset path is used here:data_dir + "/checkpoint_lenet-1_1875"+"/checkpoint_lenet-1_1875.ckpt" | |||
load_param_into_net(network, load_checkpoint(os.path.join(data_dir + "/checkpoint_lenet-1_1875", | |||
"checkpoint_lenet-1_1875.ckpt"))) | |||
if args.device_target != "Ascend": | |||
model = Model(network,net_loss,net_opt,metrics={"accuracy": Accuracy()}) | |||
else: | |||
model = Model(network, net_loss,net_opt,metrics={"accuracy": Accuracy()},amp_level="O2") | |||
config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, | |||
keep_checkpoint_max=cfg.keep_checkpoint_max) | |||
#Note that this method saves the model file on each card. You need to specify the save path on each card. | |||
# In this example, get_rank() is added to distinguish different paths. | |||
if device_num == 1: | |||
outputDirectory = train_dir + "/" | |||
if device_num > 1: | |||
outputDirectory = train_dir + "/" + str(get_rank()) + "/" | |||
ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", | |||
directory=outputDirectory, | |||
config=config_ck) | |||
print("============== Starting Training ==============") | |||
epoch_size = cfg['epoch_size'] | |||
if (args.epoch_size): | |||
epoch_size = args.epoch_size | |||
print('epoch_size is: ', epoch_size) | |||
model.train(epoch_size, | |||
ds_train, | |||
callbacks=[time_cb, ckpoint_cb, | |||
LossMonitor()]) | |||
@@ -0,0 +1,233 @@ | |||
""" | |||
######################## single-dataset train lenet example ######################## | |||
This example is a single-dataset training tutorial. If it is a multi-dataset, please refer to the multi-dataset training | |||
tutorial train_for_multidataset.py. This example cannot be used for multi-datasets! | |||
######################## Instructions for using the training environment ######################## | |||
The image of the debugging environment and the image of the training environment are two different images, | |||
and the working local directories are different. In the training task, you need to pay attention to the following points. | |||
1、(1)The structure of the dataset uploaded for single dataset training in this example | |||
MNISTData.zip | |||
├── test | |||
└── train | |||
2、Single dataset training requires predefined functions | |||
(1)Copy single dataset from obs to training image | |||
function ObsToEnv(obs_data_url, data_dir) | |||
(2)Copy the output to obs | |||
function EnvToObs(train_dir, obs_train_url) | |||
(3)Download the input from Qizhi And Init | |||
function DownloadFromQizhi(obs_data_url, data_dir) | |||
(4)Upload the output to Qizhi | |||
function UploadToQizhi(train_dir, obs_train_url) | |||
(5)Copy ckpt file from obs to training image. | |||
function ObsUrlToEnv(obs_ckpt_url, ckpt_url) | |||
3、3 parameters need to be defined | |||
--data_url is the dataset you selected on the Qizhi platform | |||
--data_url,--train_url,--device_target,These 3 parameters must be defined first in a single dataset task, | |||
otherwise an error will be reported. | |||
There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
because they are predefined in the background, you only need to define them in your code. | |||
4、How the dataset is used | |||
A single dataset uses data_url as the input, and data_dir (ie:'/cache/data') as the calling method | |||
of the dataset in the image. | |||
For details, please refer to the following sample code. | |||
5、How to load the checkpoint file | |||
The checkpoint file is loaded by the ckpt_url parameter | |||
""" | |||
import os | |||
import argparse | |||
import moxing as mox | |||
from config import mnist_cfg as cfg | |||
from dataset import create_dataset | |||
from dataset_distributed import create_dataset_parallel | |||
from lenet import LeNet5 | |||
import mindspore.nn as nn | |||
from mindspore import context | |||
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor | |||
from mindspore import load_checkpoint, load_param_into_net | |||
from mindspore.train import Model | |||
from mindspore.nn.metrics import Accuracy | |||
from mindspore.context import ParallelMode | |||
from mindspore.communication.management import init, get_rank | |||
import mindspore.ops as ops | |||
import time | |||
import json | |||
#from upload import UploadOutput | |||
### Copy single dataset from obs to training image### | |||
def ObsToEnv(obs_data_url, data_dir): | |||
try: | |||
mox.file.copy_parallel(obs_data_url, data_dir) | |||
print("Successfully Download {} to {}".format(obs_data_url, data_dir)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format(obs_data_url, data_dir) + str(e)) | |||
#Set a cache file to determine whether the data has been copied to obs. | |||
#If this file exists during multi-card training, there is no need to copy the dataset multiple times. | |||
f = open("/cache/download_input.txt", 'w') | |||
f.close() | |||
try: | |||
if os.path.exists("/cache/download_input.txt"): | |||
print("download_input succeed") | |||
except Exception as e: | |||
print("download_input failed") | |||
return | |||
### Copy ckpt file from obs to training image### | |||
### To operate on folders, use mox.file.copy_parallel. If copying a file. | |||
### Please use mox.file.copy to operate the file, this operation is to operate the file | |||
def ObsUrlToEnv(obs_ckpt_url, ckpt_url): | |||
try: | |||
mox.file.copy(obs_ckpt_url, ckpt_url) | |||
print("Successfully Download {} to {}".format(obs_ckpt_url,ckpt_url)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format(obs_ckpt_url, ckpt_url) + str(e)) | |||
return | |||
### Copy multiple datasets from obs to training image ### | |||
def MultiObsToEnv(multi_data_url, data_dir): | |||
#--multi_data_url is json data, need to do json parsing for multi_data_url | |||
multi_data_json = json.loads(multi_data_url) | |||
for i in range(len(multi_data_json)): | |||
path = data_dir + "/" + multi_data_json[i]["dataset_name"] | |||
file_path = data_dir + "/" + os.path.splitext(multi_data_json[i]["dataset_name"])[0] | |||
if not os.path.exists(file_path): | |||
os.makedirs(file_path) | |||
try: | |||
mox.file.copy_parallel(multi_data_json[i]["dataset_url"], path) | |||
print("Successfully Download {} to {}".format(multi_data_json[i]["dataset_url"],path)) | |||
#unzip dataset | |||
os.system("unzip -d %s %s" % (file_path, path)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format( | |||
multi_data_json[i]["dataset_url"], path) + str(e)) | |||
#Set a cache file to determine whether the data has been copied to obs. | |||
#If this file exists during multi-card training, there is no need to copy the dataset multiple times. | |||
f = open("/cache/download_input.txt", 'w') | |||
f.close() | |||
try: | |||
if os.path.exists("/cache/download_input.txt"): | |||
print("download_input succeed") | |||
except Exception as e: | |||
print("download_input failed") | |||
return | |||
def DownloadFromQizhi(multi_data_url, data_dir): | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) | |||
if device_num > 1: | |||
# set device_id and init for multi-card training | |||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) | |||
context.reset_auto_parallel_context() | |||
context.set_auto_parallel_context(device_num = device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, parameter_broadcast=True) | |||
init() | |||
#Copying obs data does not need to be executed multiple times, just let the 0th card copy the data | |||
local_rank=int(os.getenv('RANK_ID')) | |||
if local_rank%8==0: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
#If the cache file does not exist, it means that the copy data has not been completed, | |||
#and Wait for 0th card to finish copying data | |||
while not os.path.exists("/cache/download_input.txt"): | |||
time.sleep(1) | |||
return | |||
### --data_url,--train_url,--device_target,These 3 parameters must be defined first in a single dataset, | |||
### otherwise an error will be reported. | |||
###There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
###because they are predefined in the background, you only need to define them in your code. | |||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example') | |||
parser.add_argument('--multi_data_url', | |||
help='dataset path in obs') | |||
parser.add_argument('--ckpt_url', | |||
help='pre_train_model path in obs') | |||
parser.add_argument( | |||
'--device_target', | |||
type=str, | |||
default="Ascend", | |||
choices=['Ascend', 'CPU'], | |||
help='device where the code will be implemented (default: Ascend),if to use the CPU on the Qizhi platform:device_target=CPU') | |||
parser.add_argument('--epoch_size', | |||
type=int, | |||
default=5, | |||
help='Training epochs.') | |||
if __name__ == "__main__": | |||
args, unknown = parser.parse_known_args() | |||
data_dir = '/cache/dataset' | |||
train_dir = '/cache/output' | |||
ckpt_url = '/cache/checkpoint.ckpt' | |||
if not os.path.exists(data_dir): | |||
os.makedirs(data_dir) | |||
if not os.path.exists(train_dir): | |||
os.makedirs(train_dir) | |||
###Initialize and copy data to training image | |||
###Copy ckpt file from obs to training image | |||
ObsUrlToEnv(args.ckpt_url, ckpt_url) | |||
###Copy data from obs to training image | |||
DownloadFromQizhi(args.multi_data_url, data_dir) | |||
###The dataset path is used here:data_dir +"/train" | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
ds_train = create_dataset(os.path.join(data_dir+ "/MNISTData", "train"), cfg.batch_size) | |||
if device_num > 1: | |||
ds_train = create_dataset_parallel(os.path.join(data_dir+ "/MNISTData", "train"), cfg.batch_size) | |||
if ds_train.get_dataset_size() == 0: | |||
raise ValueError("Please check dataset size > 0 and batch_size <= dataset size") | |||
network = LeNet5(cfg.num_classes) | |||
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") | |||
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) | |||
time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) | |||
###The ckpt path is used here:ckpt_url | |||
print('-------ckpt_url is:', args.ckpt_url) | |||
load_param_into_net(network, load_checkpoint(ckpt_url)) | |||
if args.device_target != "Ascend": | |||
model = Model(network, | |||
net_loss, | |||
net_opt, | |||
metrics={"accuracy": Accuracy()}) | |||
else: | |||
model = Model(network, | |||
net_loss, | |||
net_opt, | |||
metrics={"accuracy": Accuracy()}, | |||
amp_level="O2") | |||
config_ck = CheckpointConfig( | |||
save_checkpoint_steps=cfg.save_checkpoint_steps, | |||
keep_checkpoint_max=cfg.keep_checkpoint_max) | |||
#Note that this method saves the model file on each card. You need to specify the save path on each card. | |||
# In this example, get_rank() is added to distinguish different paths. | |||
if device_num == 1: | |||
outputDirectory = train_dir + "/" | |||
if device_num > 1: | |||
outputDirectory = train_dir + "/" + str(get_rank()) + "/" | |||
ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", | |||
directory=outputDirectory, | |||
config=config_ck) | |||
print("============== Starting Training ==============") | |||
epoch_size = cfg['epoch_size'] | |||
if (args.epoch_size): | |||
epoch_size = args.epoch_size | |||
print('epoch_size is: ', epoch_size) | |||
model.train(epoch_size, | |||
ds_train, | |||
callbacks=[time_cb, ckpoint_cb, | |||
LossMonitor()]) |
@@ -0,0 +1,114 @@ | |||
""" | |||
######################## train lenet example ######################## | |||
train lenet and get network model files(.ckpt) | |||
""" | |||
#!/usr/bin/python | |||
#coding=utf-8 | |||
import os | |||
import argparse | |||
import moxing as mox | |||
from config import mnist_cfg as cfg | |||
from dataset import create_dataset | |||
from dataset_distributed import create_dataset_parallel | |||
from lenet import LeNet5 | |||
import json | |||
import mindspore.nn as nn | |||
from mindspore import context | |||
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor | |||
from mindspore.train import Model | |||
from mindspore.nn.metrics import Accuracy | |||
from mindspore import load_checkpoint, load_param_into_net | |||
from mindspore.context import ParallelMode | |||
from mindspore.communication.management import init, get_rank | |||
import time | |||
### Copy multiple datasets from obs to training image ### | |||
def MultiObsToEnv(multi_data_url, data_dir): | |||
#--multi_data_url is json data, need to do json parsing for multi_data_url | |||
multi_data_json = json.loads(multi_data_url) | |||
for i in range(len(multi_data_json)): | |||
path = data_dir + "/" + multi_data_json[i]["dataset_name"] | |||
file_path = data_dir + "/" + os.path.splitext(multi_data_json[i]["dataset_name"])[0] | |||
if not os.path.exists(file_path): | |||
os.makedirs(file_path) | |||
try: | |||
mox.file.copy_parallel(multi_data_json[i]["dataset_url"], path) | |||
print("Successfully Download {} to {}".format(multi_data_json[i]["dataset_url"],path)) | |||
#unzip dataset | |||
os.system("unzip -d %s %s" % (file_path, path)) | |||
except Exception as e: | |||
print('moxing download {} to {} failed: '.format( | |||
multi_data_json[i]["dataset_url"], path) + str(e)) | |||
#Set a cache file to determine whether the data has been copied to obs. | |||
#If this file exists during multi-card training, there is no need to copy the dataset multiple times. | |||
f = open("/cache/download_input.txt", 'w') | |||
f.close() | |||
try: | |||
if os.path.exists("/cache/download_input.txt"): | |||
print("download_input succeed") | |||
except Exception as e: | |||
print("download_input failed") | |||
return | |||
def DownloadFromQizhi(multi_data_url, data_dir): | |||
device_num = int(os.getenv('RANK_SIZE')) | |||
if device_num == 1: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) | |||
if device_num > 1: | |||
# set device_id and init for multi-card training | |||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) | |||
context.reset_auto_parallel_context() | |||
context.set_auto_parallel_context(device_num = device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True, parameter_broadcast=True) | |||
init() | |||
#Copying obs data does not need to be executed multiple times, just let the 0th card copy the data | |||
local_rank=int(os.getenv('RANK_ID')) | |||
if local_rank%8==0: | |||
MultiObsToEnv(multi_data_url,data_dir) | |||
#If the cache file does not exist, it means that the copy data has not been completed, | |||
#and Wait for 0th card to finish copying data | |||
while not os.path.exists("/cache/download_input.txt"): | |||
time.sleep(1) | |||
return | |||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example') | |||
### --multi_data_url,--ckpt_url,--device_target,These 4 parameters must be defined first in a multi-dataset, | |||
### otherwise an error will be reported. | |||
### There is no need to add these parameters to the running parameters of the Qizhi platform, | |||
### because they are predefined in the background, you only need to define them in your code. | |||
parser.add_argument('--multi_data_url', | |||
help='dataset path in obs') | |||
parser.add_argument('--ckpt_url', | |||
help='pre_train_model path in obs') | |||
parser.add_argument( | |||
'--device_target', | |||
type=str, | |||
default="Ascend", | |||
choices=['Ascend', 'CPU'], | |||
help='device where the code will be implemented (default: Ascend),if to use the CPU on the Qizhi platform:device_target=CPU') | |||
parser.add_argument('--epoch_size', | |||
type=int, | |||
default=5, | |||
help='Training epochs.') | |||
if __name__ == "__main__": | |||
args, unknown = parser.parse_known_args() | |||
data_dir = '/cache/dataset' | |||
train_dir = '/cache/output' | |||
if not os.path.exists(data_dir): | |||
os.makedirs(data_dir) | |||
if not os.path.exists(train_dir): | |||
os.makedirs(train_dir) | |||
###Initialize and copy data to training image | |||
DownloadFromQizhi(args.multi_data_url, data_dir) | |||
print("--------start ls:") | |||
os.system("cd /cache/dataset; ls -al") | |||
print("--------end ls-----------") | |||
@@ -0,0 +1,33 @@ | |||
# Copyright 2020 Huawei Technologies Co., Ltd | |||
# | |||
# Licensed under the Apache License, Version 2.0 (the "License"); | |||
# you may not use this file except in compliance with the License. | |||
# You may obtain a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
# ============================================================================ | |||
""" | |||
network config setting, will be used in train.py | |||
""" | |||
from easydict import EasyDict as edict | |||
mnist_cfg = edict({ | |||
'num_classes': 10, | |||
'lr': 0.01, | |||
'momentum': 0.9, | |||
'epoch_size': 10, | |||
'batch_size': 32, | |||
'buffer_size': 1000, | |||
'image_height': 32, | |||
'image_width': 32, | |||
'save_checkpoint_steps': 1875, | |||
'keep_checkpoint_max': 150, | |||
'air_name': "lenet", | |||
}) |
@@ -0,0 +1,60 @@ | |||
# Copyright 2020 Huawei Technologies Co., Ltd | |||
# | |||
# Licensed under the Apache License, Version 2.0 (the "License"); | |||
# you may not use this file except in compliance with the License. | |||
# You may obtain a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
# ============================================================================ | |||
""" | |||
Produce the dataset | |||
""" | |||
import mindspore.dataset as ds | |||
import mindspore.dataset.vision.c_transforms as CV | |||
import mindspore.dataset.transforms.c_transforms as C | |||
from mindspore.dataset.vision import Inter | |||
from mindspore.common import dtype as mstype | |||
def create_dataset(data_path, batch_size=32, repeat_size=1, | |||
num_parallel_workers=1): | |||
""" | |||
create dataset for train or test | |||
""" | |||
# define dataset | |||
mnist_ds = ds.MnistDataset(data_path) | |||
resize_height, resize_width = 32, 32 | |||
rescale = 1.0 / 255.0 | |||
shift = 0.0 | |||
rescale_nml = 1 / 0.3081 | |||
shift_nml = -1 * 0.1307 / 0.3081 | |||
# define map operations | |||
resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # Bilinear mode | |||
rescale_nml_op = CV.Rescale(rescale_nml, shift_nml) | |||
rescale_op = CV.Rescale(rescale, shift) | |||
hwc2chw_op = CV.HWC2CHW() | |||
type_cast_op = C.TypeCast(mstype.int32) | |||
# apply map operations on images | |||
mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=resize_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=rescale_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=rescale_nml_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=hwc2chw_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
# apply DatasetOps | |||
buffer_size = 10000 | |||
mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 10000 as in LeNet train script | |||
mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) | |||
mnist_ds = mnist_ds.repeat(repeat_size) | |||
return mnist_ds |
@@ -0,0 +1,55 @@ | |||
""" | |||
Produce the dataset: | |||
与单机不同的是,在数据集接口需要传入num_shards和shard_id参数,分别对应卡的数量和逻辑序号,建议通过HCCL接口获取: | |||
get_rank:获取当前设备在集群中的ID。 | |||
get_group_size:获取集群数量。 | |||
""" | |||
import mindspore.dataset as ds | |||
import mindspore.dataset.vision.c_transforms as CV | |||
import mindspore.dataset.transforms.c_transforms as C | |||
from mindspore.dataset.vision import Inter | |||
from mindspore.common import dtype as mstype | |||
from mindspore.communication.management import init, get_rank, get_group_size | |||
def create_dataset_parallel(data_path, batch_size=32, repeat_size=1, | |||
num_parallel_workers=1, shard_id=0, num_shards=8): | |||
""" | |||
create dataset for train or test | |||
""" | |||
resize_height, resize_width = 32, 32 | |||
rescale = 1.0 / 255.0 | |||
shift = 0.0 | |||
rescale_nml = 1 / 0.3081 | |||
shift_nml = -1 * 0.1307 / 0.3081 | |||
# get shard_id and num_shards.Get the ID of the current device in the cluster And Get the number of clusters. | |||
shard_id = get_rank() | |||
num_shards = get_group_size() | |||
# define dataset | |||
mnist_ds = ds.MnistDataset(data_path, num_shards=num_shards, shard_id=shard_id) | |||
# define map operations | |||
resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # Bilinear mode | |||
rescale_nml_op = CV.Rescale(rescale_nml, shift_nml) | |||
rescale_op = CV.Rescale(rescale, shift) | |||
hwc2chw_op = CV.HWC2CHW() | |||
type_cast_op = C.TypeCast(mstype.int32) | |||
# apply map operations on images | |||
mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=resize_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=rescale_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=rescale_nml_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
mnist_ds = mnist_ds.map(operations=hwc2chw_op, input_columns="image", num_parallel_workers=num_parallel_workers) | |||
# apply DatasetOps | |||
buffer_size = 10000 | |||
mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 10000 as in LeNet train script | |||
mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) | |||
mnist_ds = mnist_ds.repeat(repeat_size) | |||
return mnist_ds |
@@ -0,0 +1,60 @@ | |||
# Copyright 2020 Huawei Technologies Co., Ltd | |||
# | |||
# Licensed under the Apache License, Version 2.0 (the "License"); | |||
# you may not use this file except in compliance with the License. | |||
# You may obtain a copy of the License at | |||
# | |||
# http://www.apache.org/licenses/LICENSE-2.0 | |||
# | |||
# Unless required by applicable law or agreed to in writing, software | |||
# distributed under the License is distributed on an "AS IS" BASIS, | |||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
# See the License for the specific language governing permissions and | |||
# limitations under the License. | |||
# ============================================================================ | |||
"""LeNet.""" | |||
import mindspore.nn as nn | |||
from mindspore.common.initializer import Normal | |||
class LeNet5(nn.Cell): | |||
""" | |||
Lenet network | |||
Args: | |||
num_class (int): Number of classes. Default: 10. | |||
num_channel (int): Number of channels. Default: 1. | |||
Returns: | |||
Tensor, output tensor | |||
Examples: | |||
>>> LeNet(num_class=10) | |||
""" | |||
def __init__(self, num_class=10, num_channel=1, include_top=True): | |||
super(LeNet5, self).__init__() | |||
self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid') | |||
self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') | |||
self.relu = nn.ReLU() | |||
self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) | |||
self.include_top = include_top | |||
if self.include_top: | |||
self.flatten = nn.Flatten() | |||
self.fc1 = nn.Dense(16 * 5 * 5, 120, weight_init=Normal(0.02)) | |||
self.fc2 = nn.Dense(120, 84, weight_init=Normal(0.02)) | |||
self.fc3 = nn.Dense(84, num_class, weight_init=Normal(0.02)) | |||
def construct(self, x): | |||
x = self.conv1(x) | |||
x = self.relu(x) | |||
x = self.max_pool2d(x) | |||
x = self.conv2(x) | |||
x = self.relu(x) | |||
x = self.max_pool2d(x) | |||
if not self.include_top: | |||
return x | |||
x = self.flatten(x) | |||
x = self.relu(self.fc1(x)) | |||
x = self.relu(self.fc2(x)) | |||
x = self.fc3(x) | |||
return x |