|
|
@@ -423,7 +423,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//prepare command |
|
|
|
preTrainModelPath := getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName) |
|
|
|
|
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, form.CkptName) |
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, form.CkptName, "") |
|
|
|
if err != nil { |
|
|
|
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
@@ -680,7 +680,8 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
|
|
|
|
//prepare command |
|
|
|
preTrainModelPath := getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName) |
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, form.CkptName) |
|
|
|
modelRemoteObsUrl := "s3:///grampus/jobs/" + jobName + "/output/models.zip" |
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, form.CkptName, modelRemoteObsUrl) |
|
|
|
if err != nil { |
|
|
|
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
@@ -967,7 +968,7 @@ func GrampusGetLog(ctx *context.Context) { |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, pretrainModelPath, pretrainModelFileName string) (string, error) { |
|
|
|
func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, pretrainModelPath, pretrainModelFileName, modelRemoteObsUrl string) (string, error) { |
|
|
|
var command string |
|
|
|
|
|
|
|
workDir := grampus.NpuWorkDir |
|
|
@@ -1024,6 +1025,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo |
|
|
|
|
|
|
|
var commandCode string |
|
|
|
if processorType == grampus.ProcessorTypeNPU { |
|
|
|
paramCode += " --obs_url=" + modelRemoteObsUrl |
|
|
|
commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" |
|
|
|
} else if processorType == grampus.ProcessorTypeGPU { |
|
|
|
if pretrainModelFileName != "" { |
|
|
|