special dataset

modules/setting/setting.go

@@ -75,6 +75,15 @@ type C2NetSqInfos struct {
 	C2NetSqInfo []*C2NetSequenceInfo `json:"sequence"`
 }
 
+type C2NetObsDatasetInfo struct {
+	Name string `json:"name"`
+	Uuid string `json:"uuid"`
+}
+
+type C2NetObsDatasets struct {
+	C2NetObsDataset []*C2NetObsDatasetInfo `json:"datasets"`
+}
+
 type StFlavorInfos struct {
 	FlavorInfo []*FlavorInfo `json:"flavor_info"`
 }
@@ -583,15 +592,17 @@ var (
 
 	//grampus config
 	Grampus = struct {
-		Env           string
-		Host          string
-		UserName      string
-		Password      string
-		SpecialPools  string
-		C2NetSequence string
+		Env             string
+		Host            string
+		UserName        string
+		Password        string
+		SpecialPools    string
+		C2NetSequence   string
+		ExistedDatasets string
 	}{}
 
-	C2NetInfos *C2NetSqInfos
+	C2NetInfos      *C2NetSqInfos
+	ExistedDatasets *C2NetObsDatasets
 
 	//elk config
 	ElkUrl              string
@@ -1464,7 +1475,7 @@ func NewContext() {
 	FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("")
 	TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
 	ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("")
-	ModelArtsMultiNode=sec.Key("MULTI_NODE").MustString("")
+	ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("")
 
 	sec = Cfg.Section("elk")
 	ElkUrl = sec.Key("ELKURL").MustString("")
@@ -1558,6 +1569,13 @@ func getGrampusConfig() {
 			log.Error("Unmarshal(C2NetSequence) failed:%v", err)
 		}
 	}
+
+	Grampus.ExistedDatasets = sec.Key("EXISTED_DATASETS").MustString("")
+	if Grampus.ExistedDatasets != "" {
+		if err := json.Unmarshal([]byte(Grampus.ExistedDatasets), &ExistedDatasets); err != nil {
+			log.Error("Unmarshal(ExistedDatasets) failed:%v", err)
+		}
+	}
 }
 
 func SetRadarMapConfig() {

routers/repo/grampus.go

@@ -320,7 +320,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	}
 
 	//prepare command
-	command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name)
+	command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name, "")
 	if err != nil {
 		log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
@@ -502,7 +502,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	}
 
 	//prepare command
-	command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name)
+	command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name, uuid)
 	if err != nil {
 		log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
@@ -749,7 +749,7 @@ func GrampusGetLog(ctx *context.Context) {
 	return
 }
 
-func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) {
+func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, uuid string) (string, error) {
 	var command string
 
 	workDir := grampus.NpuWorkDir
@@ -761,6 +761,15 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 	//download code & dataset
 	if processorType == grampus.ProcessorTypeNPU {
 		commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"
+		// todo: check obs attachment, use moxing copy
+		if setting.ExistedDatasets != nil {
+			for _, dataset := range setting.ExistedDatasets.C2NetObsDataset {
+				if uuid == dataset.Uuid {
+					commandDownload = "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " uuid:" + uuid + " '" + datasetName + "';"
+					break
+				}
+			}
+		}
 		command += commandDownload
 	} else if processorType == grampus.ProcessorTypeGPU {
 		commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"