From 669fc61dd9c959be4b52600e5cf16d246f7788e7 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 17 Jan 2022 15:03:29 +0800 Subject: [PATCH] get child types --- models/cloudbrain.go | 53 +++++++++++++------------ modules/auth/cloudbrain.go | 19 ++++----- modules/cloudbrain/cloudbrain.go | 68 ++++++++++++++++---------------- routers/repo/cloudbrain.go | 59 +++++++++++++++++++++------ routers/routes/routes.go | 1 + 5 files changed, 119 insertions(+), 81 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 5bc70e081..11e77e49f 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -77,29 +77,30 @@ const ( ) type Cloudbrain struct { - ID int64 `xorm:"pk autoincr"` - JobID string `xorm:"INDEX NOT NULL"` - JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` - JobName string - Status string - UserID int64 - RepoID int64 - SubTaskName string - ContainerID string - ContainerIp string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Duration int64 - TrainJobDuration string - Image string //GPU镜像名称 - GpuQueue string //GPU类型即GPU队列 - ResourceSpecId int //GPU规格id - DeletedAt time.Time `xorm:"deleted"` - CanDebug bool `xorm:"-"` - CanDel bool `xorm:"-"` - CanModify bool `xorm:"-"` - Type int - BenchmarkType string `xorm:"DEFAULT ''"` + ID int64 `xorm:"pk autoincr"` + JobID string `xorm:"INDEX NOT NULL"` + JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` + JobName string + Status string + UserID int64 + RepoID int64 + SubTaskName string + ContainerID string + ContainerIp string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + Duration int64 + TrainJobDuration string + Image string //GPU镜像名称 + GpuQueue string //GPU类型即GPU队列 + ResourceSpecId int //GPU规格id + DeletedAt time.Time `xorm:"deleted"` + CanDebug bool `xorm:"-"` + CanDel bool `xorm:"-"` + CanModify bool `xorm:"-"` + Type int + BenchmarkTypeID int + BenchmarkChildTypeID int VersionID int64 //版本id VersionName string `xorm:"INDEX"` //当前版本 @@ -387,14 +388,14 @@ type BenchmarkTypes struct { type BenchmarkType struct { Id int `json:"id"` - First string `json:"first"` //一级算法类型名称 + First string `json:"first"` //一级算法类型名称 Second []*BenchmarkDataset `json:"second"` } type BenchmarkDataset struct { Id int `json:"id"` - Value string `json:"value"` //二级算法类型名称 - Attachment string `json:"attachment"` //数据集的uuid + Value string `json:"value"` //二级算法类型名称 + Attachment string `json:"attachment"` //数据集的uuid } type GpuInfos struct { diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index c5b5c1f47..277588f40 100755 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -6,15 +6,16 @@ import ( ) type CreateCloudBrainForm struct { - JobName string `form:"job_name" binding:"Required"` - Image string `form:"image" binding:"Required"` - Command string `form:"command" binding:"Required"` - Attachment string `form:"attachment" binding:"Required"` - JobType string `form:"job_type" binding:"Required"` - BenchmarkCategory string `form:"get_benchmark_category"` - GpuType string `form:"gpu_type"` - ResourceSpecId int `form:"resource_spec_id" binding:"Required"` - BenchmarkType string `form:"benchmark_type"` + JobName string `form:"job_name" binding:"Required"` + Image string `form:"image" binding:"Required"` + Command string `form:"command" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + JobType string `form:"job_type" binding:"Required"` + BenchmarkCategory string `form:"get_benchmark_category"` + GpuType string `form:"gpu_type"` + ResourceSpecId int `form:"resource_spec_id" binding:"Required"` + BenchmarkTypeID int `form:"benchmark_types_id"` + BenchmarkChildTypeID int `form:"benchmark_child_types_id"` } type CommitImageCloudBrainForm struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 4898cf56c..2140e3b34 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -14,7 +14,7 @@ import ( ) const ( - Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple; + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple; service ssh stop; jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` @@ -105,7 +105,7 @@ func AdminOrJobCreaterRight(ctx *context.Context) { } -func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, benchmarkType string, resourceSpecId int) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + @@ -204,20 +204,21 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, var jobID = jobResult.Payload["jobId"].(string) err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: string(models.JobWaiting), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: jobID, - JobName: jobName, - SubTaskName: SubTaskName, - JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, - Image: image, - GpuQueue: gpuQueue, - ResourceSpecId: resourceSpecId, - ComputeResource: models.GPUResource, - BenchmarkType: benchmarkType, + Status: string(models.JobWaiting), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobID, + JobName: jobName, + SubTaskName: SubTaskName, + JobType: jobType, + Type: models.TypeCloudBrainOne, + Uuid: uuid, + Image: image, + GpuQueue: gpuQueue, + ResourceSpecId: resourceSpecId, + ComputeResource: models.GPUResource, + BenchmarkTypeID: benchmarkTypeID, + BenchmarkChildTypeID: benchmarkChildTypeID, }) if err != nil { @@ -274,7 +275,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string Volumes: []models.Volume{ { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, CodeMountPath + "/"), + Path: storage.GetMinioPath(jobName, CodeMountPath+"/"), MountPath: CodeMountPath, ReadOnly: false, }, @@ -288,28 +289,28 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, ModelMountPath + "/"), + Path: storage.GetMinioPath(jobName, ModelMountPath+"/"), MountPath: ModelMountPath, ReadOnly: false, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, BenchMarkMountPath + "/"), + Path: storage.GetMinioPath(jobName, BenchMarkMountPath+"/"), MountPath: BenchMarkMountPath, ReadOnly: true, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath + "/"), + Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath+"/"), MountPath: Snn4imagenetMountPath, ReadOnly: true, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, BrainScoreMountPath + "/"), + Path: storage.GetMinioPath(jobName, BrainScoreMountPath+"/"), MountPath: BrainScoreMountPath, ReadOnly: true, }, @@ -327,20 +328,19 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string var jobID = jobResult.Payload["jobId"].(string) newTask := &models.Cloudbrain{ - Status: string(models.JobWaiting), - UserID: task.UserID, - RepoID: task.RepoID, - JobID: jobID, - JobName: task.JobName, - SubTaskName: task.SubTaskName, - JobType: task.JobType, - Type: task.Type, - Uuid: task.Uuid, - Image: task.Image, - GpuQueue: task.GpuQueue, - ResourceSpecId: task.ResourceSpecId, + Status: string(models.JobWaiting), + UserID: task.UserID, + RepoID: task.RepoID, + JobID: jobID, + JobName: task.JobName, + SubTaskName: task.SubTaskName, + JobType: task.JobType, + Type: task.Type, + Uuid: task.Uuid, + Image: task.Image, + GpuQueue: task.GpuQueue, + ResourceSpecId: task.ResourceSpecId, ComputeResource: task.ComputeResource, - BenchmarkType: task.BenchmarkType, } err = models.RestartCloudbrain(task, newTask) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 2681f8bfe..1c4fee707 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -248,7 +248,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, "", resourceSpecId) + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, 0, 0, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) @@ -974,6 +974,48 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ctx.HTML(200, tplCloudBrainBenchmarkIndex) } +func GetChildTypes(ctx *context.Context) { + benchmarkTypeID := ctx.QueryInt("benchmark_type_id") + resultCode := "-1" + errMsg := "" + childTypes := "" + // + + for { + if benchmarkTypes == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkTypes), &benchmarkTypes); err != nil { + log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", setting.BenchmarkTypes, err, ctx.Data["MsgID"]) + errMsg = "system error" + break + } + } + + var isExist bool + + for id, _ := range benchmarkTypes.BenchmarkType { + if id == benchmarkTypeID { + isExist = true + //childTypesInfo = benchmarkType.Second + break + } + } + + if !isExist { + log.Error("no such benchmark_type_id", ctx.Data["MsgID"]) + errMsg = "system error" + break + } + + break + } + + ctx.JSON(200, map[string]string{ + "result_code": resultCode, + "error_msg": errMsg, + "child_types": childTypes, + }) +} + func CloudBrainBenchmarkNew(ctx *context.Context) { err := cloudBrainNewDataPrepare(ctx) if err != nil { @@ -988,26 +1030,19 @@ func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainF jobName := form.JobName image := form.Image uuid := form.Attachment - jobType := form.JobType gpuQueue := form.GpuType command := cloudbrain.CommandBenchmark codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId - benchmarkType := form.BenchmarkType + benchmarkTypeID := form.BenchmarkTypeID + benchmarkChildTypeID := form.BenchmarkChildTypeID if !jobNamePattern.MatchString(jobName) { ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form) return } - if jobType != string(models.JobTypeBenchmark) { - log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("jobtype error", tplCloudBrainBenchmarkNew, &form) - return - } - - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeBenchmark)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) @@ -1076,7 +1111,7 @@ func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainF err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, benchmarkType, resourceSpecId) + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 8302cd7b6..a76328fd7 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -990,6 +990,7 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainBenchmarkNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainBenchmarkCreate) + m.Get("/get_child_types", reqRepoCloudBrainWriter, repo.GetChildTypes) }) }, context.RepoRef()) m.Group("/modelmanage", func() {