diff --git a/custom/public/img/ranking_list.jpg b/custom/public/img/ranking_list.jpg new file mode 100644 index 000000000..94610215b Binary files /dev/null and b/custom/public/img/ranking_list.jpg differ diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7270a9144..82c4c6b83 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -78,28 +78,30 @@ const ( ) type Cloudbrain struct { - ID int64 `xorm:"pk autoincr"` - JobID string `xorm:"INDEX NOT NULL"` - JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` - JobName string - Status string - UserID int64 - RepoID int64 - SubTaskName string - ContainerID string - ContainerIp string - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Duration int64 - TrainJobDuration string - Image string //GPU镜像名称 - GpuQueue string //GPU类型即GPU队列 - ResourceSpecId int //GPU规格id - DeletedAt time.Time `xorm:"deleted"` - CanDebug bool `xorm:"-"` - CanDel bool `xorm:"-"` - CanModify bool `xorm:"-"` - Type int + ID int64 `xorm:"pk autoincr"` + JobID string `xorm:"INDEX NOT NULL"` + JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` + JobName string + Status string + UserID int64 + RepoID int64 + SubTaskName string + ContainerID string + ContainerIp string + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + Duration int64 + TrainJobDuration string + Image string //GPU镜像名称 + GpuQueue string //GPU类型即GPU队列 + ResourceSpecId int //GPU规格id + DeletedAt time.Time `xorm:"deleted"` + CanDebug bool `xorm:"-"` + CanDel bool `xorm:"-"` + CanModify bool `xorm:"-"` + Type int + BenchmarkTypeID int + BenchmarkChildTypeID int VersionID int64 //版本id VersionName string `xorm:"INDEX"` //当前版本 @@ -214,7 +216,7 @@ type CloudbrainsOptions struct { CloudbrainIDs []int64 // JobStatus CloudbrainStatus Type int - JobTypes []string + JobTypes []string VersionName string IsLatestVersion string JobTypeNot bool @@ -387,6 +389,24 @@ type Category struct { Value string `json:"value"` } +type BenchmarkTypes struct { + BenchmarkType []*BenchmarkType `json:"type"` +} + +type BenchmarkType struct { + Id int `json:"id"` + First string `json:"first"` //一级算法类型名称 + Second []*BenchmarkDataset `json:"second"` +} + +type BenchmarkDataset struct { + Id int `json:"id"` + Value string `json:"value"` //二级算法类型名称 + Attachment string `json:"attachment"` //数据集的uuid + Owner string `json:"owner"` //评估脚本所在仓库的拥有者 + RepoName string `json:"repo_name"` //评估脚本所在仓库的名称 +} + type GpuInfos struct { GpuInfo []*GpuInfo `json:"gpu_type"` } @@ -442,6 +462,59 @@ type CommitImageResult struct { Payload map[string]interface{} `json:"payload"` } +type GetJobLogParams struct { + Size string `json:"size"` + Sort string `json:"sort"` + QueryInfo QueryInfo `json:"query"` +} + +type QueryInfo struct { + MatchInfo MatchInfo `json:"match"` +} + +type MatchInfo struct { + PodName string `json:"kubernetes.pod.name"` +} + +type GetJobLogResult struct { + ScrollID string `json:"_scroll_id"` + Took int `json:"took"` + TimedOut bool `json:"timed_out"` + Shards struct { + Total int `json:"total"` + Successful int `json:"successful"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + } `json:"_shards"` + Hits struct { + Hits []Hits `json:"hits"` + } `json:"hits"` +} + +type Hits struct { + Index string `json:"_index"` + Type string `json:"_type"` + ID string `json:"_id"` + Source struct { + Message string `json:"message"` + } `json:"_source"` + Sort []int `json:"sort"` +} + +type GetAllJobLogParams struct { + Scroll string `json:"scroll"` + ScrollID string `json:"scroll_id"` +} + +type DeleteJobLogTokenParams struct { + ScrollID string `json:"scroll_id"` +} + +type DeleteJobLogTokenResult struct { + Succeeded bool `json:"succeeded"` + NumFreed int `json:"num_freed"` +} + type CloudBrainResult struct { Code string `json:"code"` Msg string `json:"msg"` @@ -1221,8 +1294,8 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudbrainCountByUserID(userID int64) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) +func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) return int(count), err } diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go index d598b495c..0d7ef1b02 100755 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -6,14 +6,19 @@ import ( ) type CreateCloudBrainForm struct { - JobName string `form:"job_name" binding:"Required"` - Image string `form:"image" binding:"Required"` - Command string `form:"command" binding:"Required"` - Attachment string `form:"attachment" binding:"Required"` - JobType string `form:"job_type" binding:"Required"` - BenchmarkCategory string `form:"get_benchmark_category"` - GpuType string `form:"gpu_type"` - ResourceSpecId int `form:"resource_spec_id" binding:"Required"` + JobName string `form:"job_name" binding:"Required"` + Image string `form:"image" binding:"Required"` + Command string `form:"command" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + JobType string `form:"job_type" binding:"Required"` + BenchmarkCategory string `form:"get_benchmark_category"` + GpuType string `form:"gpu_type"` + TrainUrl string `form:"train_url"` + TestUrl string `form:"test_url"` + Description string `form:"description"` + ResourceSpecId int `form:"resource_spec_id" binding:"Required"` + BenchmarkTypeID int `form:"benchmark_types_id"` + BenchmarkChildTypeID int `form:"benchmark_child_types_id"` } type CommitImageCloudBrainForm struct { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 74dcbe7b0..f15443b30 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -14,11 +14,16 @@ import ( ) const ( - Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple; + service ssh stop; + jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` + //CommandBenchmark = `echo "start benchmark";python /code/test.py;echo "end benchmark"` + CommandBenchmark = `echo "start benchmark";cd /benchmark && bash run_bk.sh;echo "end benchmark"` CodeMountPath = "/code" DataSetMountPath = "/dataset" ModelMountPath = "/model" BenchMarkMountPath = "/benchmark" + BenchMarkResourceID = 1 Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" @@ -102,7 +107,7 @@ func AdminOrJobCreaterRight(ctx *context.Context) { } -func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + @@ -201,19 +206,22 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, var jobID = jobResult.Payload["jobId"].(string) err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: string(models.JobWaiting), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: jobID, - JobName: jobName, - SubTaskName: SubTaskName, - JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, - Image: image, - GpuQueue: gpuQueue, - ResourceSpecId: resourceSpecId, - ComputeResource: models.GPUResource, + Status: string(models.JobWaiting), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobID, + JobName: jobName, + SubTaskName: SubTaskName, + JobType: jobType, + Type: models.TypeCloudBrainOne, + Uuid: uuid, + Image: image, + GpuQueue: gpuQueue, + ResourceSpecId: resourceSpecId, + ComputeResource: models.GPUResource, + BenchmarkTypeID: benchmarkTypeID, + BenchmarkChildTypeID: benchmarkChildTypeID, + Description: description, }) if err != nil { @@ -270,7 +278,7 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string Volumes: []models.Volume{ { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, CodeMountPath + "/"), + Path: storage.GetMinioPath(jobName, CodeMountPath+"/"), MountPath: CodeMountPath, ReadOnly: false, }, @@ -284,28 +292,28 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, ModelMountPath + "/"), + Path: storage.GetMinioPath(jobName, ModelMountPath+"/"), MountPath: ModelMountPath, ReadOnly: false, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, BenchMarkMountPath + "/"), + Path: storage.GetMinioPath(jobName, BenchMarkMountPath+"/"), MountPath: BenchMarkMountPath, ReadOnly: true, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath + "/"), + Path: storage.GetMinioPath(jobName, Snn4imagenetMountPath+"/"), MountPath: Snn4imagenetMountPath, ReadOnly: true, }, }, { HostPath: models.StHostPath{ - Path: storage.GetMinioPath(jobName, BrainScoreMountPath + "/"), + Path: storage.GetMinioPath(jobName, BrainScoreMountPath+"/"), MountPath: BrainScoreMountPath, ReadOnly: true, }, @@ -323,18 +331,18 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newJobID *string var jobID = jobResult.Payload["jobId"].(string) newTask := &models.Cloudbrain{ - Status: string(models.JobWaiting), - UserID: task.UserID, - RepoID: task.RepoID, - JobID: jobID, - JobName: task.JobName, - SubTaskName: task.SubTaskName, - JobType: task.JobType, - Type: task.Type, - Uuid: task.Uuid, - Image: task.Image, - GpuQueue: task.GpuQueue, - ResourceSpecId: task.ResourceSpecId, + Status: string(models.JobWaiting), + UserID: task.UserID, + RepoID: task.RepoID, + JobID: jobID, + JobName: task.JobName, + SubTaskName: task.SubTaskName, + JobType: task.JobType, + Type: task.Type, + Uuid: task.Uuid, + Image: task.Image, + GpuQueue: task.GpuQueue, + ResourceSpecId: task.ResourceSpecId, ComputeResource: task.ComputeResource, } diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index 4e30ea0e4..46b7c991b 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -2,7 +2,10 @@ package cloudbrain import ( "encoding/json" + "errors" "fmt" + "net/http" + "strconv" "strings" "code.gitea.io/gitea/modules/log" @@ -23,6 +26,8 @@ const ( JobHasBeenStopped = "S410" Public = "public" Custom = "custom" + LogPageSize = 500 + LogPageTokenExpired = "5m" ) func getRestyClient() *resty.Client { @@ -270,3 +275,99 @@ sendjob: return nil } + +func GetJobLog(jobID string) (*models.GetJobLogResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetJobLogResult + req := models.GetJobLogParams{ + Size: strconv.Itoa(LogPageSize), + Sort: "log.offset", + QueryInfo: models.QueryInfo{ + MatchInfo: models.MatchInfo{ + PodName: jobID + "-task1-0", + }, + }, + } + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(req). + SetResult(&result). + Post(HOST + "es/_search?_source=message&scroll=" + LogPageTokenExpired) + + if err != nil { + log.Error("GetJobLog failed: %v", err) + return &result, fmt.Errorf("resty GetJobLog: %v, %s", err, res.String()) + } + + if !strings.Contains(res.Status(), strconv.Itoa(http.StatusOK)) { + log.Error("res.Status(): %s, response: %s", res.Status(), res.String()) + return &result, errors.New(res.String()) + } + + return &result, nil +} + +func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) { + checkSetting() + client := getRestyClient() + var result models.GetJobLogResult + req := models.GetAllJobLogParams{ + Scroll: LogPageTokenExpired, + ScrollID: scrollID, + } + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(req). + SetResult(&result). + Post(HOST + "es/_search/scroll") + + if err != nil { + log.Error("GetJobAllLog failed: %v", err) + return &result, fmt.Errorf("resty GetJobAllLog: %v, %s", err, res.String()) + } + + if !strings.Contains(res.Status(), strconv.Itoa(http.StatusOK)) { + log.Error("res.Status(): %s, response: %s", res.Status(), res.String()) + return &result, errors.New(res.String()) + } + + return &result, nil +} + +func DeleteJobLogToken(scrollID string) (error) { + checkSetting() + client := getRestyClient() + var result models.DeleteJobLogTokenResult + req := models.DeleteJobLogTokenParams{ + ScrollID: scrollID, + } + + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetBody(req). + SetResult(&result). + Delete(HOST + "es/_search/scroll") + + if err != nil { + log.Error("DeleteJobLogToken failed: %v", err) + return fmt.Errorf("resty DeleteJobLogToken: %v, %s", err, res.String()) + } + + if !strings.Contains(res.Status(), strconv.Itoa(http.StatusOK)) { + log.Error("res.Status(): %s, response: %s", res.Status(), res.String()) + return errors.New(res.String()) + } + + if !result.Succeeded { + log.Error("DeleteJobLogToken failed") + return errors.New("DeleteJobLogToken failed") + } + + return nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e7ab0b7d2..92ed32ef2 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -462,11 +462,15 @@ var ( MaxDuration int64 //benchmark config - IsBenchmarkEnabled bool - BenchmarkOwner string - BenchmarkName string - BenchmarkServerHost string - BenchmarkCategory string + IsBenchmarkEnabled bool + BenchmarkOwner string + BenchmarkName string + BenchmarkServerHost string + BenchmarkCategory string + BenchmarkTypes string + BenchmarkGpuTypes string + BenchmarkResourceSpecs string + BenchmarkMaxDuration int64 //snn4imagenet config IsSnn4imagenetEnabled bool @@ -1270,6 +1274,10 @@ func NewContext() { BenchmarkName = sec.Key("NAME").MustString("") BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") + BenchmarkTypes = sec.Key("TYPES").MustString("") + BenchmarkGpuTypes = sec.Key("GPU_TYPES").MustString("") + BenchmarkResourceSpecs = sec.Key("RESOURCE_SPECS").MustString("") + BenchmarkMaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a292685cc..739d515bc 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -933,6 +933,14 @@ modelarts.train_job_para_admin=train_job_para_admin modelarts.train_job_para.edit=train_job_para.edit modelarts.train_job_para.connfirm=train_job_para.connfirm + +modelarts.evaluate_job=Model Evaluation +modelarts.evaluate_job.new_job=New Model Evaluation +cloudbrain.benchmark.evaluate_type=Evaluation Type +cloudbrain.benchmark.evaluate_child_type=Child Type +cloudbrain.benchmark.evaluate_mirror=Mirror +cloudbrain.benchmark.evaluate_train=Train Script +cloudbrain.benchmark.evaluate_test=Test Script modelarts.infer_job_model = Model modelarts.infer_job_model_file = Model File modelarts.infer_job = Inference Job diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 01cae4e2c..935b6112b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -941,6 +941,14 @@ modelarts.back=返回 modelarts.train_job_para_admin=任务参数管理 modelarts.train_job_para.edit=编辑 modelarts.train_job_para.connfirm=确定 +modelarts.evaluate_job=评测任务 +modelarts.evaluate_job.new_job=新建评测任务 +cloudbrain.benchmark.evaluate_type=评测类型 +cloudbrain.benchmark.evaluate_child_type=子类型 +cloudbrain.benchmark.evaluate_mirror=镜像 +cloudbrain.benchmark.evaluate_train=训练程序 +cloudbrain.benchmark.evaluate_test=测试程序 + modelarts.infer_job_model = 模型名称 modelarts.infer_job_model_file = 模型文件 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2a4092510..21da7cf42 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -879,6 +879,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqAnyRepoReader()) m.Group("/cloudbrain", func() { m.Get("/:jobid", repo.GetCloudbrainTask) + m.Get("/:jobid/log", repo.CloudbrainGetLog) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelarts", func() { m.Group("/notebook", func() { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index bfba5236b..dd468783d 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -8,6 +8,7 @@ package repo import ( "code.gitea.io/gitea/modules/log" "net/http" + "sort" "time" "code.gitea.io/gitea/models" @@ -91,3 +92,59 @@ func GetCloudbrainTask(ctx *context.APIContext) { }) } + +func CloudbrainGetLog(ctx *context.Context) { + jobID := ctx.Params(":jobid") + _, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("GetCloudbrainByJobID failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + + var hits []models.Hits + result, err := cloudbrain.GetJobLog(jobID) + if err != nil{ + log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"]) + ctx.ServerError(err.Error(), err) + return + } + hits = result.Hits.Hits + + //if the size equal page_size, then take the scroll_id to get all log and delete the scroll_id(the num of scroll_id is limited) + if len(result.Hits.Hits) >= cloudbrain.LogPageSize { + for { + resultNext, err := cloudbrain.GetJobAllLog(result.ScrollID) + if err != nil{ + log.Error("GetJobAllLog failed: %v", err, ctx.Data["MsgID"]) + } else { + for _, hit := range resultNext.Hits.Hits { + hits = append(hits, hit) + } + } + + if len(resultNext.Hits.Hits) < cloudbrain.LogPageSize { + log.Info("get all log already") + break + } + } + } + + cloudbrain.DeleteJobLogToken(result.ScrollID) + + sort.Slice(hits, func(i, j int) bool { + return hits[i].Sort[0] < hits[j].Sort[0] + }) + + var content string + for _, log := range hits { + content += log.Source.Message + "\n" + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "Content": content, + }) + + return +} diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 679d05305..c73a93c1a 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -6,11 +6,12 @@ package repo import ( - "code.gitea.io/gitea/modules/util" "net/http" "strconv" "strings" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index ca1d08101..6621081fd 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -28,15 +28,21 @@ import ( ) const ( - tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" tplCloudBrainNew base.TplName = "repo/cloudbrain/new" tplCloudBrainShow base.TplName = "repo/cloudbrain/show" tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index" + + tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index" + tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" + tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" ) var ( - gpuInfos *models.GpuInfos - categories *models.Categories + gpuInfos *models.GpuInfos + categories *models.Categories + benchmarkTypes *models.BenchmarkTypes + benchmarkGpuInfos *models.GpuInfos + benchmarkResourceSpecs *models.ResourceSpecs ) var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) @@ -124,11 +130,28 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } ctx.Data["benchmark_categories"] = categories.Category + if benchmarkTypes == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkTypes), &benchmarkTypes); err != nil { + log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", setting.BenchmarkTypes, err, ctx.Data["MsgID"]) + } + } + ctx.Data["benchmark_types"] = benchmarkTypes.BenchmarkType + if gpuInfos == nil { json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) } ctx.Data["gpu_types"] = gpuInfos.GpuInfo + if benchmarkGpuInfos == nil { + json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) + } + ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo + + if benchmarkResourceSpecs == nil { + json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs) + } + ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec + if cloudbrain.ResourceSpecs == nil { json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) } @@ -155,9 +178,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true jobName := form.JobName image := form.Image - command := form.Command uuid := form.Attachment jobType := form.JobType + command := cloudbrain.Command gpuQueue := form.GpuType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId @@ -174,7 +197,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) @@ -238,12 +261,14 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, resourceSpecId) + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, + 0, 0, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") } @@ -276,7 +301,7 @@ func CloudBrainRestart(ctx *context.Context) { break } - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) resultCode = "-1" @@ -310,7 +335,15 @@ func CloudBrainRestart(ctx *context.Context) { }) } +func CloudBrainBenchMarkShow(ctx *context.Context) { + cloudBrainShow(ctx, tplCloudBrainBenchmarkShow) +} + func CloudBrainShow(ctx *context.Context) { + cloudBrainShow(ctx, tplCloudBrainShow) +} + +func cloudBrainShow(ctx *context.Context, tpName base.TplName) { ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") @@ -327,6 +360,8 @@ func CloudBrainShow(ctx *context.Context) { if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB") + spec := "GPU数：" + strconv.Itoa(jobRes.Resource.NvidiaComGpu) + "，CPU数：" + strconv.Itoa(jobRes.Resource.CPU) + "，内存(MB):" + jobRes.Resource.Memory + ctx.Data["resource_spec"] = spec taskRoles := jobRes.TaskRoles if jobRes.JobStatus.State != string(models.JobFailed) { taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) @@ -351,11 +386,28 @@ func CloudBrainShow(ctx *context.Context) { } ctx.Data["result"] = jobRes + } else { + log.Info("error:" + err.Error()) + } + user, err := models.GetUserByID(task.UserID) + if err == nil { + task.User = user } + var duration int64 + if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.CreatedUnix) + } else { + duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + } + ctx.Data["duration"] = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000) ctx.Data["task"] = task ctx.Data["jobID"] = jobID - ctx.HTML(200, tplCloudBrainShow) + ctx.Data["jobName"] = task.JobName + version_list_task := make([]*models.Cloudbrain, 0) + version_list_task = append(version_list_task, task) + ctx.Data["version_list_task"] = version_list_task + ctx.HTML(200, tpName) } func CloudBrainDebug(ctx *context.Context) { @@ -393,7 +445,7 @@ func CloudBrainStop(ctx *context.Context) { task := ctx.Cloudbrain for { - if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded){ log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) resultCode = "-1" errorMsg = "system error" @@ -510,22 +562,31 @@ func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { } func CloudBrainDel(ctx *context.Context) { + if err := deleteCloudbrainJob(ctx); err != nil { + log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) + ctx.ServerError(err.Error(), err) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") +} + +func deleteCloudbrainJob(ctx *context.Context) error { task := ctx.Cloudbrain - if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) { + if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) { log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) - ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) - return + return errors.New("the job has not been stopped") } err := models.DeleteJob(task) if err != nil { - ctx.ServerError("DeleteJob failed", err) - return + log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"]) + return err } deleteJobStorage(task.JobName, models.TypeCloudBrainOne) - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") + + return nil } func CloudBrainShowModels(ctx *context.Context) { @@ -632,6 +693,12 @@ func CloudBrainDownloadModel(ctx *context.Context) { } func GetRate(ctx *context.Context) { + isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll") + if isObjectDetcionAll { + ctx.Redirect(setting.BenchmarkServerHost + "?username=admin") + return + } + var jobID = ctx.Params(":jobid") job, err := models.GetCloudbrainByJobID(jobID) if err != nil { @@ -640,6 +707,7 @@ func GetRate(ctx *context.Context) { } if job.JobType == string(models.JobTypeBenchmark) { + log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name) ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name) } else if job.JobType == string(models.JobTypeSnn4imagenet) { ctx.Redirect(setting.Snn4imagenetServerHost) @@ -855,7 +923,14 @@ func SyncCloudbrainStatus() { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } - if task.Duration >= setting.MaxDuration { + var maxDuration int64 + if task.JobType == string(models.JobTypeBenchmark) { + maxDuration = setting.BenchmarkMaxDuration + } else { + maxDuration = setting.MaxDuration + } + + if task.Duration >= maxDuration { log.Info("begin to stop job(%s), because of the duration", task.JobName) err = cloudbrain.StopJob(task.JobID) if err != nil { @@ -923,3 +998,328 @@ func SyncCloudbrainStatus() { return } + +func CloudBrainBenchmarkIndex(ctx *context.Context) { + MustEnableCloudbrain(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + var jobTypes []string + jobTypes = append(jobTypes, string(models.JobTypeBenchmark)) + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + Type: models.TypeCloudBrainOne, + JobTypes: jobTypes, + }) + if err != nil { + ctx.ServerError("Get debugjob faild:", err) + return + } + + for i, task := range ciTasks { + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) + ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + var duration int64 + if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix) + } else { + duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix) + } + ciTasks[i].TrainJobDuration = util.AddZero(duration/3600000) + ":" + util.AddZero(duration%3600000/60000) + ":" + util.AddZero(duration%60000/1000) + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + + ctx.Data["Page"] = pager + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = ciTasks + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) + ctx.Data["RepoIsEmpty"] = repo.IsEmpty + ctx.HTML(200, tplCloudBrainBenchmarkIndex) +} + +func GetChildTypes(ctx *context.Context) { + benchmarkTypeID := ctx.QueryInt("benchmark_type_id") + re := make(map[string]interface{}) + for { + if benchmarkTypes == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkTypes), &benchmarkTypes); err != nil { + log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", setting.BenchmarkTypes, err, ctx.Data["MsgID"]) + re["errMsg"] = "system error" + break + } + } + var isExist bool + for _, benchmarkType := range benchmarkTypes.BenchmarkType { + if benchmarkTypeID == benchmarkType.Id { + isExist = true + re["child_types"] = benchmarkType.Second + re["result_code"] = "0" + break + } + } + if !isExist { + re["result_code"] = "1" + log.Error("no such benchmark_type_id", ctx.Data["MsgID"]) + re["errMsg"] = "system error" + break + } + break + } + ctx.JSON(200, re) +} + +func CloudBrainBenchmarkNew(ctx *context.Context) { + err := cloudBrainNewDataPrepare(ctx) + if err != nil { + ctx.ServerError("get new cloudbrain info failed", err) + return + } + ctx.HTML(200, tplCloudBrainBenchmarkNew) +} + +func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int) (*models.BenchmarkDataset, error) { + var childInfo *models.BenchmarkDataset + if benchmarkTypes == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkTypes), &benchmarkTypes); err != nil { + log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", setting.BenchmarkTypes, err) + return childInfo, err + } + } + + var isExist bool + for _, benchmarkType := range benchmarkTypes.BenchmarkType { + if benchmarkType.Id == benchmarkTypeID { + for _, childType := range benchmarkType.Second { + if childType.Id == benchmarkChildTypeID { + childInfo = childType + isExist = true + break + } + } + break + } + } + + if !isExist { + log.Error("no such benchmark_type_id&benchmark_child_type_id") + return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id") + } + + return childInfo, nil +} + +func getBenchmarkGpuQueue(gpuQueue string) (string, error) { + queue := "" + if benchmarkGpuInfos == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil { + log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err) + return queue, err + } + } + + var isExist bool + for _, gpuInfo := range benchmarkGpuInfos.GpuInfo { + if gpuQueue == gpuInfo.Queue { + isExist = true + queue = gpuQueue + break + } + } + + if !isExist { + log.Error("no such gpuQueue, %s", gpuQueue) + return queue, errors.New("no such gpuQueue") + } + + return queue, nil +} + +func getBenchmarkResourceSpec(resourceSpecID int) (int, error) { + var id int + if benchmarkResourceSpecs == nil { + if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil { + log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err) + return id, err + } + } + + var isExist bool + for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec { + if resourceSpecID == resourceSpec.Id { + isExist = true + id = resourceSpecID + break + } + } + + if !isExist { + log.Error("no such resourceSpecID, %d", resourceSpecID) + return id, errors.New("no such resourceSpec") + } + + return id, nil +} + +func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { + ctx.Data["PageIsCloudBrain"] = true + jobName := form.JobName + image := form.Image + gpuQueue := form.GpuType + command := cloudbrain.CommandBenchmark + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + resourceSpecId := cloudbrain.BenchMarkResourceID + benchmarkTypeID := form.BenchmarkTypeID + benchmarkChildTypeID := form.BenchmarkChildTypeID + + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form) + return + } + + childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID) + if err != nil { + log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form) + return + } + + _, err = getBenchmarkGpuQueue(gpuQueue) + if err != nil { + log.Error("getBenchmarkGpuQueue failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("gpu queue error", tplCloudBrainBenchmarkNew, &form) + return + } + + _, err = getBenchmarkResourceSpec(resourceSpecId) + if err != nil { + log.Error("getBenchmarkResourceSpec failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("resource spec error", tplCloudBrainBenchmarkNew, &form) + return + } + + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeBenchmark)) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainBenchmarkNew, &form) + return + } + } + + _, err = models.GetCloudbrainByName(jobName) + if err == nil { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form) + return + } else { + if !models.IsErrJobNotExist(err) { + log.Error("GetCloudbrainByName failed, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + return + } + } + repo := ctx.Repo.Repository + os.RemoveAll(codePath) + if err := downloadCode(repo, codePath); err != nil { + log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + return + } + + if _, err := os.Stat(codePath + "/train.py"); err != nil { + if os.IsNotExist(err) { + // file does not exist + log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form) + } else { + log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + } + return + } else if _, err := os.Stat(codePath + "/test.py"); err != nil { + if os.IsNotExist(err) { + // file does not exist + log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form) + } else { + log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + } + return + } + + if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { + log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + return + } + + benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath + var gpuType string + for _, gpuInfo := range gpuInfos.GpuInfo { + if gpuInfo.Queue == gpuQueue { + gpuType = gpuInfo.Value + } + } + + if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, gpuType); err != nil { + log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"]) + //cloudBrainNewDataPrepare(ctx) + //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + //return + } + + if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil { + log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) + //cloudBrainNewDataPrepare(ctx) + //ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) + //return + } + + err = cloudbrain.GenerateTask(ctx, jobName, image, command, childInfo.Attachment, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, + benchmarkTypeID, benchmarkChildTypeID, resourceSpecId) + if err != nil { + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") +} + +func BenchmarkDel(ctx *context.Context) { + if err := deleteCloudbrainJob(ctx); err != nil { + log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"]) + ctx.ServerError(err.Error(), err) + return + } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") +} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 0198856a5..cc4e0840b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -64,7 +64,7 @@ func DebugJobIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) + jobTypes = append(jobTypes, string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, @@ -387,7 +387,7 @@ func TrainJobIndex(ctx *context.Context) { RepoID: repo.ID, Type: models.TypeCloudBrainTwo, JobTypeNot: false, - JobTypes: jobTypes, + JobTypes: jobTypes, IsLatestVersion: modelarts.IsLatestVersion, }) if err != nil { @@ -1321,10 +1321,10 @@ func TrainJobShow(ctx *context.Context) { Page: page, PageSize: setting.UI.IssuePagingNum, }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, JobTypes: jobTypes, - JobID: jobID, + JobID: jobID, }) if err != nil { @@ -1438,10 +1438,10 @@ func TrainJobDel(ctx *context.Context) { var jobTypes []string jobTypes = append(jobTypes, string(models.JobTypeTrain)) VersionListTasks, _, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, JobTypes: jobTypes, - JobID: jobID, + JobID: jobID, }) if err != nil { ctx.ServerError("get VersionListTasks failed", err) @@ -1747,8 +1747,8 @@ func InferenceJobIndex(ctx *context.Context) { Page: page, PageSize: setting.UI.IssuePagingNum, }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, JobTypes: jobTypes, }) if err != nil { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 62ed21eb4..76b8e9eaf 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -979,6 +979,19 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) + + m.Group("/benchmark", func() { + m.Get("", reqRepoCloudBrainReader, repo.CloudBrainBenchmarkIndex) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.CloudBrainBenchMarkShow) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.BenchmarkDel) + m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) + }) + m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainBenchmarkNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainBenchmarkCreate) + m.Get("/get_child_types", repo.GetChildTypes) + }) }, context.RepoRef()) m.Group("/modelmanage", func() { m.Post("/create_model", reqRepoModelManageWriter, repo.SaveModel) diff --git a/templates/repo/cloudbrain/benchmark/index.tmpl b/templates/repo/cloudbrain/benchmark/index.tmpl new file mode 100644 index 000000000..35f6eaef6 --- /dev/null +++ b/templates/repo/cloudbrain/benchmark/index.tmpl @@ -0,0 +1,387 @@ + +{{template "base/head" .}} + + + + +
+
+
+
+
+
+
+
+
+ + +
+ +
+ {{template "repo/header" .}} + +
+ {{template "base/alert" .}} +
+
+
+ {{$.i18n.Tr "repo.modelarts.notebook"}} + {{$.i18n.Tr "repo.modelarts.train_job"}} + {{$.i18n.Tr "repo.modelarts.infer_job"}} + {{$.i18n.Tr "repo.modelarts.evaluate_job"}} +
+
+
+ 基准测试排行榜 + {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{$.i18n.Tr "repo.modelarts.evaluate_job.new_job"}} + {{else}} + {{$.i18n.Tr "repo.modelarts.evaluate_job.new_job"}} + {{end}} +
+
+ {{if eq 0 (len .Tasks)}} +
+
+
未创建过评测任务
+
+ {{if $.RepoIsEmpty}} +
代码版本：您还没有初始化代码仓库，请先创建代码版本；
+ {{end}} +
使用说明：可以参考启智AI协作平台小白训练营课程。
+
+
+ {{else}} + +
+
+
+ +
+ + +
+
+
+ {{$.i18n.Tr "repo.cloudbrain_task"}} +
+
+ {{$.i18n.Tr "repo.modelarts.status"}} +
+
+ {{$.i18n.Tr "repo.modelarts.createtime"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_status_runtime"}} +
+
+ {{$.i18n.Tr "repo.modelarts.computing_resources"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_creator"}} +
+
+ {{$.i18n.Tr "repo.cloudbrain_operate"}} +
+
+
+ + {{range .Tasks}} +
+
+ + +
+ + + {{.JobName}} + +
+ +
+ + {{.Status}} + +
+ +
+ {{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}} +
+ +
+ {{.TrainJobDuration}} +
+ +
+ {{.ComputeResource}} +
+ +
+ {{if .User.Name}} + + {{else}} + + {{end}} +
+ +
+
+ +
+ {{$.CsrfTokenHtml}} + {{if .CanDel}} + + + {{$.i18n.Tr "repo.stop"}} + + + {{else}} + + {{$.i18n.Tr "repo.stop"}} + + {{end}} +
+ + + + 评分 + + + + +
+ + {{$.CsrfTokenHtml}} + {{if .CanDel}} + + {{$.i18n.Tr "repo.delete"}} + + {{else}} + + {{$.i18n.Tr "repo.delete"}} + + + {{end}} +
+
+
+
+
+ {{end}} + + + + {{template "base/paginate" .}} +
+ +
+
+
+ {{end}} + +
+ +
+ + + + + +
+
+
+ 删除任务 +
+ +
+

你确认删除该任务么？此任务一旦删除不可恢复。

+
+
+
+ 取消操作 +
+
+ 确定操作 +
+
+
+
+ + +{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl new file mode 100755 index 000000000..4ad5cfd35 --- /dev/null +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -0,0 +1,243 @@ +{{template "base/head" .}} + + +
+
+
+
+
+
+
+
+
+
+ {{template "repo/header" .}} +
+ {{template "base/alert" .}} +

+ {{.i18n.Tr "repo.modelarts.evaluate_job.new_job"}} +

+
+ +
+ {{.CsrfTokenHtml}} + + + +
+ + +
+
+ + +
+ +
+ + +
+ + +
+
+ +   + +
+
+ + +
+
+ +
+ +       + + + + {{range .images}} + + {{end}} + {{range .public_images}} + + {{end}} + +
+ + + +
+ + +
+ +
+ + + 查看样例 +
+ +
+ + + 查看样例 +
+ + +
+ + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+ + + +
+
+
+
+{{template "base/footer" .}} + + \ No newline at end of file diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl new file mode 100755 index 000000000..329f31476 --- /dev/null +++ b/templates/repo/cloudbrain/benchmark/show.tmpl @@ -0,0 +1,437 @@ +{{template "base/head" .}} + +
+
+
+
+
+
+
+
+
+
+{{template "repo/header" .}} +
+

+
+ + {{.i18n.Tr "repo.cloudbrain"}} + +
/
+ + {{$.i18n.Tr "repo.modelarts.evaluate_job"}} + +
/
+
{{.jobName}}
+
+

+ {{range $k ,$v := .version_list_task}} +
+
+
+
+ + + +
+ {{TimeSinceUnix1 .CreatedUnix}} + + {{$.i18n.Tr "repo.modelarts.status"}}： + {{.Status}} + + {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： + {{$.duration}} + +
+ + +
+
+
+
+
+
+ {{$.i18n.Tr "repo.modelarts.train_job.config"}} + {{$.i18n.Tr "repo.modelarts.log"}} +
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{$.i18n.Tr "repo.cloudbrain_task"}} + +
+ {{.JobName}} +
+
+ {{$.i18n.Tr "repo.modelarts.status"}} + +
+ {{.Status}} +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.start_time"}} + +
+ {{TimeSinceUnix1 .CreatedUnix}} +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.dura_time"}} + +
+ {{$.duration}} +
+
+ 镜像 + +
+ {{.Image}} +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 训练程序 + +
+ train.py +
+
+ 测试程序 + +
+ test.py +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.description"}} + +
+ {{.Description}} +
+
+ {{$.i18n.Tr "repo.modelarts.train_job.standard"}} + +
+ {{$.resource_spec}} +
+
+ 创建者 + +
+ {{.User.Name}} +
+
+
+
+
+ +
+
+
+
+ +
+ + + 

+                            

+                                        
+                        

+                        
+                    

+                    
+                

+            

+        

+        {{end}} {{template "base/paginate" .}}
+    

+    
+    

+        

+            

+                 删除任务
+            

+    
+            

+                
你确认删除该任务么？此任务一旦删除不可恢复。

+            

+            

+                

+                     取消操作
+                

+                

+                     确定操作
+                

+            

+        

+    

+    
+    
+

+{{template "base/footer" .}}
+
+
\ No newline at end of file
diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl
index e39e7cb22..0735c593b 100755
--- a/templates/repo/cloudbrain/new.tmpl
+++ b/templates/repo/cloudbrain/new.tmpl
@@ -150,13 +150,10 @@
                         
                     
 
-                    

+