From 518e47e96816c8ea41ba1217c644d42a516f0f55 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 11 Oct 2022 19:58:14 +0800 Subject: [PATCH 1/2] code opt --- models/cloudbrain.go | 2 ++ modules/grampus/grampus.go | 11 +++++++++++ routers/repo/grampus.go | 11 ++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index d088a9127..6797fe332 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1490,6 +1490,8 @@ type GrampusTasks struct { ReplicaNum int `json:"replicaNum"` Datasets []GrampusDataset `json:"datasets"` Models []GrampusDataset `json:"models"` + Code GrampusDataset `json:"code"` + BootFile string `json:"bootFile"` } type GrampusDataset struct { diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index d72a7b10e..7bacb46d3 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -1,6 +1,7 @@ package grampus import ( + "code.gitea.io/gitea/modules/cloudbrain" "encoding/json" "strings" @@ -73,6 +74,7 @@ type GenerateTrainJobReq struct { PreTrainModelPath string PreTrainModelUrl string Spec *models.Specification + CodeName string } func getEndPoint() string { @@ -102,6 +104,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error centerID, centerName := getCentersParamter(ctx, req) var datasetGrampus, modelGrampus []models.GrampusDataset + var codeGrampus models.GrampusDataset if ProcessorTypeNPU == req.ProcessType { datasetGrampus = getDatasetGrampus(req.DatasetInfos) if len(req.ModelName) != 0 { @@ -114,6 +117,12 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }, } } + codeGrampus = models.GrampusDataset{ + Name: req.CodeName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ObjectKey: req.CodeObsPath + cloudbrain.DefaultBranchName + ".zip", + } } jobResult, err := createJob(models.CreateGrampusJobRequest{ @@ -130,6 +139,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error ReplicaNum: 1, Datasets: datasetGrampus, Models: modelGrampus, + Code: codeGrampus, + BootFile: req.BootFile, }, }, }) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index abf64281a..bc61b0b55 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -713,6 +713,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain DatasetNames: datasetNames, DatasetInfos: datasetInfos, Spec: spec, + CodeName: strings.ToLower(repo.Name), } if form.ModelName != "" { //使用预训练模型训练 req.ModelName = form.ModelName @@ -976,8 +977,8 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - command += commandDownload + //commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" + //command += commandDownload } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -986,8 +987,8 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - command += commandUnzip + //commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" + //command += commandUnzip } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand @@ -1024,7 +1025,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo var commandCode string if processorType == grampus.ProcessorTypeNPU { - commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";" + commandCode = "/bin/bash /home/work/run_train_for_openi.sh /home/work/openi.py /tmp/log/train.log" + paramCode + ";" } else if processorType == grampus.ProcessorTypeGPU { if pretrainModelFileName != "" { paramCode += " --ckpt_url" + "=" + workDir + "pretrainmodel/" + pretrainModelFileName From cfda54463ea4fa65cdb45c544a0093f8e32e9602 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 17 Oct 2022 17:06:50 +0800 Subject: [PATCH 2/2] del unneed code --- routers/repo/grampus.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 63176184e..b78bdebd3 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -978,8 +978,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo command += "pwd;cd " + workDir + fmt.Sprintf(grampus.CommandPrepareScript, setting.Grampus.SyncScriptProject, setting.Grampus.SyncScriptProject) //download code & dataset if processorType == grampus.ProcessorTypeNPU { - //commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + ";" - //command += commandDownload + //no need to download code & dataset by internet } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " '" + dataRemotePath + "' '" + datasetName + "'" commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload) @@ -988,8 +987,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //unzip code & dataset if processorType == grampus.ProcessorTypeNPU { - //commandUnzip := "cd " + workDir + "code;unzip -q master.zip;" - //command += commandUnzip + //no need to process } else if processorType == grampus.ProcessorTypeGPU { unZipDatasetCommand := generateDatasetUnzipCommand(datasetName) commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand