Merge pull request 'dataMount' (#3395) from dataMount into V20221214

Reviewed-on: https://openi.pcl.ac.cn/OpenI/aiforge/pulls/3395
Reviewed-by: ychao_1983 <ychao_1983@sina.com>

models/cloudbrain.go

@@ -1061,6 +1061,9 @@ type UserImageConfig struct {
 	CreateVersion bool        `json:"create_version"`
 	Flavor        Flavor      `json:"flavor"`
 	PoolID        string      `json:"pool_id"`
+	ShareAddr     string      `json:"nas_share_addr"`
+	MountPath     string      `json:"nas_mount_path"`
+	NasType       string      `json:"nas_type"`
 }
 
 type CreateTrainJobParams struct {
@@ -1084,13 +1087,18 @@ type Config struct {
 	CreateVersion bool   `json:"create_version"`
 	Flavor        Flavor `json:"flavor"`
 	PoolID        string `json:"pool_id"`
+	ShareAddr     string `json:"nas_share_addr"`
+	MountPath     string `json:"nas_mount_path"`
+	NasType       string `json:"nas_type"`
 }
+
 type CreateInferenceJobParams struct {
 	JobName     string    `json:"job_name"`
 	Description string    `json:"job_desc"`
 	InfConfig   InfConfig `json:"config"`
 	WorkspaceID string    `json:"workspace_id"`
 }
+
 type CreateInfUserImageParams struct {
 	JobName     string             `json:"job_name"`
 	Description string             `json:"job_desc"`
@@ -1148,6 +1156,9 @@ type TrainJobVersionConfig struct {
 	Flavor        Flavor      `json:"flavor"`
 	PoolID        string      `json:"pool_id"`
 	PreVersionId  int64       `json:"pre_version_id"`
+	ShareAddr     string      `json:"nas_share_addr"`
+	MountPath     string      `json:"nas_mount_path"`
+	NasType       string      `json:"nas_type"`
 }
 
 type TrainJobVersionUserImageConfig struct {
@@ -1163,6 +1174,9 @@ type TrainJobVersionUserImageConfig struct {
 	PreVersionId  int64       `json:"pre_version_id"`
 	UserImageUrl  string      `json:"user_image_url"`
 	UserCommand   string      `json:"user_command"`
+	ShareAddr     string      `json:"nas_share_addr"`
+	MountPath     string      `json:"nas_mount_path"`
+	NasType       string      `json:"nas_type"`
 }
 
 type CreateConfigParams struct {
@@ -1178,6 +1192,7 @@ type CreateConfigParams struct {
 	LogUrl        string      `json:"log_url"`
 	Flavor        Flavor      `json:"flavor"`
 	PoolID        string      `json:"pool_id"`
+	Volumes       []Volumes   `json:"volumes"`
 }
 
 type Parameter struct {

modules/modelarts/modelarts.go

@@ -281,7 +281,7 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin
 	return nil
 }
 
-func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string,autoStopDurationInMs int64) (string, error) {
+func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string, autoStopDurationInMs int64) (string, error) {
 	if poolInfos == nil {
 		json.Unmarshal([]byte(setting.PoolInfos), &poolInfos)
 	}
@@ -379,6 +379,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
 				Parameter:    req.Parameters,
 				UserImageUrl: req.UserImageUrl,
 				UserCommand:  req.UserCommand,
+				ShareAddr:    setting.ModelArtsShareAddr,
+				MountPath:    setting.ModelArtsMountPath,
+				NasType:      setting.ModelArtsNasType,
 			},
 		})
 	} else {
@@ -399,6 +402,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
 					Code: req.Spec.SourceSpecId,
 				},
 				Parameter: req.Parameters,
+				ShareAddr: setting.ModelArtsShareAddr,
+				MountPath: setting.ModelArtsMountPath,
+				NasType:   setting.ModelArtsNasType,
 			},
 		})
 	}
@@ -517,6 +523,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
 				PreVersionId: req.PreVersionId,
 				UserImageUrl: req.UserImageUrl,
 				UserCommand:  req.UserCommand,
+				ShareAddr:    setting.ModelArtsShareAddr,
+				MountPath:    setting.ModelArtsMountPath,
+				NasType:      setting.ModelArtsNasType,
 			},
 		}, jobId)
 	} else {
@@ -536,6 +545,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
 				},
 				Parameter:    req.Parameters,
 				PreVersionId: req.PreVersionId,
+				ShareAddr:    setting.ModelArtsShareAddr,
+				MountPath:    setting.ModelArtsMountPath,
+				NasType:      setting.ModelArtsNasType,
 			},
 		}, jobId)
 	}
@@ -972,14 +984,14 @@ func getJupyterBaseUrl(url string) string {
 }
 
 func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) {
-    log.Info("jupyter url:"+jupyterUrl)
+	log.Info("jupyter url:" + jupyterUrl)
 	var cookies []*http.Cookie
 	const retryTimes = 10
 	for i := 0; i < retryTimes; i++ {
 		res, err := http.Get(jupyterUrl)
 		if err != nil {
-			log.Error("browser jupyterUrl failed.",err)
-			if i==retryTimes-1{
+			log.Error("browser jupyterUrl failed.", err)
+			if i == retryTimes-1 {
 				return cookies, ""
 			}
 

modules/modelarts/resty.go

@@ -497,7 +497,7 @@ sendjob:
 	}
 
 	req, _ := json.Marshal(createJobParams)
-	log.Info("%s", req)
+	log.Info("postapi json: %s", req)
 
 	if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
 		retry++
@@ -543,6 +543,8 @@ func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.Create
 	var result models.CreateTrainJobResult
 
 	retry := 0
+	req, _ := json.Marshal(createJobParams)
+	log.Info("postapi json: %s", req)
 
 sendjob:
 	res, err := client.R().

modules/setting/setting.go

@@ -585,6 +585,9 @@ var (
 	TrainJobFLAVORINFOS   string
 	ModelArtsSpecialPools string
 	ModelArtsMultiNode    string
+	ModelArtsShareAddr    string
+	ModelArtsMountPath    string
+	ModelArtsNasType      string
 	//kanban
 	IsCloudbrainTimingEnabled bool
 
@@ -1557,6 +1560,9 @@ func NewContext() {
 	TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
 	ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("")
 	ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("")
+	ModelArtsShareAddr = sec.Key("ModelArts_Share_Addr").MustString("192.168.0.30:/")
+	ModelArtsMountPath = sec.Key("ModelArts_Mount_Path").MustString("/cache/sfs")
+	ModelArtsNasType = sec.Key("ModelArts_Nas_Type").MustString("nfs")
 
 	sec = Cfg.Section("elk")
 	ElkUrl = sec.Key("ELKURL").MustString("")