diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1992baf54..762760f90 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -75,6 +75,15 @@ type C2NetSqInfos struct { C2NetSqInfo []*C2NetSequenceInfo `json:"sequence"` } +type C2NetObsDatasetInfo struct { + Name string `json:"name"` + Uuid string `json:"uuid"` +} + +type C2NetObsDatasets struct { + C2NetObsDataset []*C2NetObsDatasetInfo `json:"datasets"` +} + type StFlavorInfos struct { FlavorInfo []*FlavorInfo `json:"flavor_info"` } @@ -583,15 +592,17 @@ var ( //grampus config Grampus = struct { - Env string - Host string - UserName string - Password string - SpecialPools string - C2NetSequence string + Env string + Host string + UserName string + Password string + SpecialPools string + C2NetSequence string + ExistedDatasets string }{} - C2NetInfos *C2NetSqInfos + C2NetInfos *C2NetSqInfos + ExistedDatasets *C2NetObsDatasets //elk config ElkUrl string @@ -1464,7 +1475,7 @@ func NewContext() { FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("") - ModelArtsMultiNode=sec.Key("MULTI_NODE").MustString("") + ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("") sec = Cfg.Section("elk") ElkUrl = sec.Key("ELKURL").MustString("") @@ -1558,6 +1569,13 @@ func getGrampusConfig() { log.Error("Unmarshal(C2NetSequence) failed:%v", err) } } + + Grampus.ExistedDatasets = sec.Key("EXISTED_DATASETS").MustString("") + if Grampus.ExistedDatasets != "" { + if err := json.Unmarshal([]byte(Grampus.ExistedDatasets), &ExistedDatasets); err != nil { + log.Error("Unmarshal(ExistedDatasets) failed:%v", err) + } + } } func SetRadarMapConfig() { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 6fc77a454..4f3421c98 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -320,7 +320,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain } //prepare command - command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name) + command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name, "") if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) @@ -502,7 +502,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain } //prepare command - command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name) + command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name, uuid) if err != nil { log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) @@ -749,7 +749,7 @@ func GrampusGetLog(ctx *context.Context) { return } -func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) { +func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, uuid string) (string, error) { var command string workDir := grampus.NpuWorkDir @@ -761,6 +761,15 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //download code & dataset if processorType == grampus.ProcessorTypeNPU { commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';" + // todo: check obs attachment, use moxing copy + if setting.ExistedDatasets != nil { + for _, dataset := range setting.ExistedDatasets.C2NetObsDataset { + if uuid == dataset.Uuid { + commandDownload = "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " uuid:" + uuid + " '" + datasetName + "';" + break + } + } + } command += commandDownload } else if processorType == grampus.ProcessorTypeGPU { commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"