From 8930280c8c502d3187a15390ea8340cddc824980 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 21 Jul 2022 16:17:39 +0800 Subject: [PATCH 01/14] update --- models/cloudbrain.go | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 85338d2bc..7b7693eb6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2014,3 +2014,110 @@ func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) { return datasetInfos, datasetNames, nil } + +var ( + SpecsMapInitFlag = false + CloudbrainDebugResourceSpecsMap map[int]*ResourceSpec + CloudbrainTrainResourceSpecsMap map[int]*ResourceSpec + CloudbrainBenchmarkResourceSpecsMap map[int]*ResourceSpec + ModelArtsDebugResourceSpecsMap map[string]*FlavorInfo + ModelArtsTrainResourceSpecsMap map[string]*FlavorInfo +) + +type ModelArtsFlavor struct { + Info []struct { + Code string `json:"code"` + Value string `json:"value"` + UnitPrice int64 `json:"unitPrice"` + } `json:"flavor"` +} + +func InitResourceSpecMap() { + if CloudbrainDebugResourceSpecsMap == nil || len(CloudbrainDebugResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.ResourceSpecs), &t) + CloudbrainDebugResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainDebugResourceSpecsMap[spec.Id] = spec + } + } + if CloudbrainTrainResourceSpecsMap == nil || len(CloudbrainTrainResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.TrainResourceSpecs), &t) + CloudbrainTrainResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainTrainResourceSpecsMap[spec.Id] = spec + } + } + if CloudbrainBenchmarkResourceSpecsMap == nil || len(CloudbrainBenchmarkResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &t) + CloudbrainBenchmarkResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainBenchmarkResourceSpecsMap[spec.Id] = spec + } + } + if ModelArtsDebugResourceSpecsMap == nil || len(ModelArtsDebugResourceSpecsMap) == 0 { + t := FlavorInfos{} + json.Unmarshal([]byte(setting.FlavorInfos), &t) + ModelArtsDebugResourceSpecsMap = make(map[string]*FlavorInfo, len(t.FlavorInfo)) + for _, spec := range t.FlavorInfo { + ModelArtsDebugResourceSpecsMap[spec.Value] = spec + } + } + if ModelArtsTrainResourceSpecsMap == nil || len(ModelArtsTrainResourceSpecsMap) == 0 { + t := ModelArtsFlavor{} + json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &t) + ModelArtsTrainResourceSpecsMap = make(map[string]*FlavorInfo, len(t.Info)) + for _, spec := range t.Info { + f := &FlavorInfo{ + Value: spec.Code, + Desc: spec.Value, + } + ModelArtsTrainResourceSpecsMap[spec.Value] = f + } + } + SpecsMapInitFlag = true +} + +type ResourceAndFlavor struct { + ResourceSpec *ResourceSpec + FlavorInfo *FlavorInfo +} + +func NewResourceAndFlavor(resourceSpec *ResourceSpec, flavorInfo *FlavorInfo) *ResourceAndFlavor { + return &ResourceAndFlavor{ + ResourceSpec: resourceSpec, + FlavorInfo: flavorInfo, + } +} + +func GetCloudbrainResourceSpec(jobType string, clusterType int, resourceSpecId int, flavorCode string) *ResourceAndFlavor { + if !SpecsMapInitFlag { + InitResourceSpecMap() + } + if clusterType == TypeCloudBrainOne { + switch jobType { + case string(JobTypeDebug): + return NewResourceAndFlavor(CloudbrainDebugResourceSpecsMap[resourceSpecId], nil) + case string(JobTypeTrain): + return NewResourceAndFlavor(CloudbrainTrainResourceSpecsMap[resourceSpecId], nil) + case string(JobTypeBenchmark): + return NewResourceAndFlavor(CloudbrainBenchmarkResourceSpecsMap[resourceSpecId], nil) + + } + } else if clusterType == TypeCloudBrainTwo { + switch jobType { + case string(JobTypeDebug): + return NewResourceAndFlavor(nil, ModelArtsDebugResourceSpecsMap[flavorCode]) + case string(JobTypeTrain): + return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) + case string(JobTypeInference): + return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) + + } + } + + return nil + +} From cd725d6ab6dd5832b8a24275dd92642025dad7c7 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 22 Jul 2022 11:40:02 +0800 Subject: [PATCH 02/14] update --- models/cloudbrain.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7b7693eb6..1b877c568 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2121,3 +2121,21 @@ func GetCloudbrainResourceSpec(jobType string, clusterType int, resourceSpecId i return nil } +func GetCloudbrainTaskCardNum(task Cloudbrain) int { + spec := GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) + if spec == nil { + return 1 + } + if task.Type == TypeCloudBrainOne { + if spec.ResourceSpec == nil { + return 1 + } + return spec.ResourceSpec.GpuNum + } else if task.Type == TypeCloudBrainTwo { + if spec.FlavorInfo == nil { + return 1 + } + return spec.FlavorInfo.UnitPrice + } + return 1 +} From 5fb14f452c1d2a84a2e68e1ca7e9f33bbd57d339 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 22 Jul 2022 18:06:44 +0800 Subject: [PATCH 03/14] GetCloudbrainCardNumAndType --- models/cloudbrain.go | 80 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1b877c568..13649fe93 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2022,6 +2022,9 @@ var ( CloudbrainBenchmarkResourceSpecsMap map[int]*ResourceSpec ModelArtsDebugResourceSpecsMap map[string]*FlavorInfo ModelArtsTrainResourceSpecsMap map[string]*FlavorInfo + GpuInfosMapInitFlag = false + CloudbrainDebugGpuInfosMap map[string]*GpuInfo + CloudbrainTrainGpuInfosMap map[string]*GpuInfo ) type ModelArtsFlavor struct { @@ -2080,6 +2083,26 @@ func InitResourceSpecMap() { SpecsMapInitFlag = true } +func InitGpuInfoMap() { + if CloudbrainDebugGpuInfosMap == nil || len(CloudbrainDebugGpuInfosMap) == 0 { + t := GpuInfos{} + json.Unmarshal([]byte(setting.GpuTypes), &t) + CloudbrainDebugGpuInfosMap = make(map[int]*GpuInfo, len(t.GpuInfo)) + for _, GpuInfo := range t.GpuInfo { + CloudbrainDebugGpuInfosMap[GpuInfo.Queue] = GpuInfo + } + } + if CloudbrainTrainGpuInfosMap == nil || len(CloudbrainTrainGpuInfosMap) == 0 { + t := GpuInfos{} + json.Unmarshal([]byte(setting.TrainGpuTypes), &t) + CloudbrainTrainGpuInfosMap = make(map[int]*GpuInfo, len(t.GpuInfo)) + for _, GpuInfo := range t.GpuInfo { + CloudbrainTrainGpuInfosMap[GpuInfo.Queue] = GpuInfo + } + } + GpuInfosMapInitFlag = true +} + type ResourceAndFlavor struct { ResourceSpec *ResourceSpec FlavorInfo *FlavorInfo @@ -2121,21 +2144,52 @@ func GetCloudbrainResourceSpec(jobType string, clusterType int, resourceSpecId i return nil } -func GetCloudbrainTaskCardNum(task Cloudbrain) int { - spec := GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) - if spec == nil { - return 1 + +// func GetCloudbrainTaskCardNum(task Cloudbrain) int { +// spec := GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) +// if spec == nil { +// return 0 +// } +// if task.Type == TypeCloudBrainOne { +// if spec.ResourceSpec == nil { +// return 0 +// } +// return spec.ResourceSpec.GpuNum +// } else if task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net { +// return getCardnumByFlavorname(task.FlavorName) +// } +// return 0 +// } + +func getCardNumAndTypeByFlavorname(FlavorName string) (int, string) { + var beginIndex = strings.Index(FlavorName, ":") + var lastIndex = strings.LastIndex(FlavorName, ":") + var endIndex = strings.Index(FlavorName, "*") + cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex:endIndex])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return 0, "" + } + cardType := strings.TrimSpace(FlavorName[endIndex:lastIndex]) + return cardNum, cardType +} + +func GetCloudbrainCardNumAndType(task Cloudbrain) (int, string) { + if !SpecsMapInitFlag { + InitResourceSpecMap() + } + if !GpuInfosMapInitFlag { + InitGpuInfoMap() } if task.Type == TypeCloudBrainOne { - if spec.ResourceSpec == nil { - return 1 - } - return spec.ResourceSpec.GpuNum - } else if task.Type == TypeCloudBrainTwo { - if spec.FlavorInfo == nil { - return 1 + switch task.JobType { + case string(JobTypeDebug): + return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId].GpuNum, CloudbrainDebugGpuInfosMap[task.GpuQueue].Value + case string(JobTypeTrain): + return CloudbrainTrainResourceSpecsMap[task.ResourceSpecId].GpuNum, CloudbrainTrainGpuInfosMap[task.GpuQueue].Value } - return spec.FlavorInfo.UnitPrice + } else if task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net { + return getCardNumAndTypeByFlavorname(task.FlavorName) } - return 1 + return 0, "" } From 7f0abd5c869f8214387c2e9d9a795208722a19eb Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 25 Jul 2022 15:16:35 +0800 Subject: [PATCH 04/14] fix-2454 --- models/cloudbrain.go | 186 ++++++++------------ models/cloudbrain_static.go | 5 + routers/api/v1/repo/cloudbrain_dashboard.go | 60 +++++-- 3 files changed, 120 insertions(+), 131 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 13649fe93..dc316e943 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1844,7 +1844,8 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { } if (opts.IsLatestVersion) != "" { - cond = cond.And(builder.Or(builder.And(builder.Eq{"cloudbrain.is_latest_version": opts.IsLatestVersion}, builder.Eq{"cloudbrain.job_type": "TRAIN"}), builder.Neq{"cloudbrain.job_type": "TRAIN"})) + cond = cond.And(builder.Or(builder.And(builder.Eq{"cloudbrain.is_latest_version": opts.IsLatestVersion}, + builder.Eq{"cloudbrain.job_type": "TRAIN"}), builder.Neq{"cloudbrain.job_type": "TRAIN"})) } if len(opts.CloudbrainIDs) > 0 { @@ -1882,7 +1883,8 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { } else { lowerKeyWord := strings.ToLower(opts.Keyword) - cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) + cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, + builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) count, err = sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Join("left", "`user`", condition).Count(new(CloudbrainInfo)) @@ -1960,7 +1962,8 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time").Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", + "created_unix", "start_time", "end_time").Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } @@ -2020,22 +2023,12 @@ var ( CloudbrainDebugResourceSpecsMap map[int]*ResourceSpec CloudbrainTrainResourceSpecsMap map[int]*ResourceSpec CloudbrainBenchmarkResourceSpecsMap map[int]*ResourceSpec - ModelArtsDebugResourceSpecsMap map[string]*FlavorInfo - ModelArtsTrainResourceSpecsMap map[string]*FlavorInfo GpuInfosMapInitFlag = false CloudbrainDebugGpuInfosMap map[string]*GpuInfo CloudbrainTrainGpuInfosMap map[string]*GpuInfo ) -type ModelArtsFlavor struct { - Info []struct { - Code string `json:"code"` - Value string `json:"value"` - UnitPrice int64 `json:"unitPrice"` - } `json:"flavor"` -} - -func InitResourceSpecMap() { +func InitCloudbrainOneResourceSpecMap() { if CloudbrainDebugResourceSpecsMap == nil || len(CloudbrainDebugResourceSpecsMap) == 0 { t := ResourceSpecs{} json.Unmarshal([]byte(setting.ResourceSpecs), &t) @@ -2060,34 +2053,14 @@ func InitResourceSpecMap() { CloudbrainBenchmarkResourceSpecsMap[spec.Id] = spec } } - if ModelArtsDebugResourceSpecsMap == nil || len(ModelArtsDebugResourceSpecsMap) == 0 { - t := FlavorInfos{} - json.Unmarshal([]byte(setting.FlavorInfos), &t) - ModelArtsDebugResourceSpecsMap = make(map[string]*FlavorInfo, len(t.FlavorInfo)) - for _, spec := range t.FlavorInfo { - ModelArtsDebugResourceSpecsMap[spec.Value] = spec - } - } - if ModelArtsTrainResourceSpecsMap == nil || len(ModelArtsTrainResourceSpecsMap) == 0 { - t := ModelArtsFlavor{} - json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &t) - ModelArtsTrainResourceSpecsMap = make(map[string]*FlavorInfo, len(t.Info)) - for _, spec := range t.Info { - f := &FlavorInfo{ - Value: spec.Code, - Desc: spec.Value, - } - ModelArtsTrainResourceSpecsMap[spec.Value] = f - } - } SpecsMapInitFlag = true } -func InitGpuInfoMap() { +func InitCloudbrainOneGpuInfoMap() { if CloudbrainDebugGpuInfosMap == nil || len(CloudbrainDebugGpuInfosMap) == 0 { t := GpuInfos{} json.Unmarshal([]byte(setting.GpuTypes), &t) - CloudbrainDebugGpuInfosMap = make(map[int]*GpuInfo, len(t.GpuInfo)) + CloudbrainDebugGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) for _, GpuInfo := range t.GpuInfo { CloudbrainDebugGpuInfosMap[GpuInfo.Queue] = GpuInfo } @@ -2095,7 +2068,7 @@ func InitGpuInfoMap() { if CloudbrainTrainGpuInfosMap == nil || len(CloudbrainTrainGpuInfosMap) == 0 { t := GpuInfos{} json.Unmarshal([]byte(setting.TrainGpuTypes), &t) - CloudbrainTrainGpuInfosMap = make(map[int]*GpuInfo, len(t.GpuInfo)) + CloudbrainTrainGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) for _, GpuInfo := range t.GpuInfo { CloudbrainTrainGpuInfosMap[GpuInfo.Queue] = GpuInfo } @@ -2103,93 +2076,82 @@ func InitGpuInfoMap() { GpuInfosMapInitFlag = true } -type ResourceAndFlavor struct { - ResourceSpec *ResourceSpec - FlavorInfo *FlavorInfo -} - -func NewResourceAndFlavor(resourceSpec *ResourceSpec, flavorInfo *FlavorInfo) *ResourceAndFlavor { - return &ResourceAndFlavor{ - ResourceSpec: resourceSpec, - FlavorInfo: flavorInfo, +func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { + if FlavorName == "" { + return 0, "", nil + } else { + var beginIndex = strings.Index(FlavorName, ":") + var lastIndex = strings.LastIndex(FlavorName, ":") + var endIndex = strings.Index(FlavorName, "*") + cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return 0, "", err + } + cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) + return cardNum, cardType, err } } -func GetCloudbrainResourceSpec(jobType string, clusterType int, resourceSpecId int, flavorCode string) *ResourceAndFlavor { +func GetCloudbrainCardNumAndType(task Cloudbrain) (int, string, error) { if !SpecsMapInitFlag { - InitResourceSpecMap() + InitCloudbrainOneResourceSpecMap() } - if clusterType == TypeCloudBrainOne { - switch jobType { - case string(JobTypeDebug): - return NewResourceAndFlavor(CloudbrainDebugResourceSpecsMap[resourceSpecId], nil) - case string(JobTypeTrain): - return NewResourceAndFlavor(CloudbrainTrainResourceSpecsMap[resourceSpecId], nil) - case string(JobTypeBenchmark): - return NewResourceAndFlavor(CloudbrainBenchmarkResourceSpecsMap[resourceSpecId], nil) + if !GpuInfosMapInitFlag { + InitCloudbrainOneGpuInfoMap() + } + FlavorName, err := GetCloudbrainFlavorName(task) + if err != nil { + return 0, "", nil + } + return getCardNumAndTypeByFlavorname(FlavorName) +} +func GetCloudbrainFlavorName(task Cloudbrain) (string, error) { + if task.Type == TypeCloudBrainOne { + ResourceSpec, GpuInfo, err := GetCloudBrainOneResourceSpec(task) + if err != nil { + log.Info("GetCloudBrainOneResourceSpec err:", err) + return "", err + } else { + if ResourceSpec == nil || GpuInfo == nil { + err := errors.New("ResourceSpec or GpuInfo is nil") + return "", err + } else { + CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(ResourceSpec.GpuNum) + "*Nvidia-" + GpuInfo.Value + + " | CPU:" + strconv.Itoa(ResourceSpec.CpuNum) + "核" + strconv.Itoa(ResourceSpec.MemMiB) + "MB" + return CloudbrainOneFlavorName, nil + } } - } else if clusterType == TypeCloudBrainTwo { - switch jobType { - case string(JobTypeDebug): - return NewResourceAndFlavor(nil, ModelArtsDebugResourceSpecsMap[flavorCode]) - case string(JobTypeTrain): - return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) - case string(JobTypeInference): - return NewResourceAndFlavor(nil, ModelArtsTrainResourceSpecsMap[flavorCode]) - + } else if (task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net) && task.FlavorName != "" { + return task.FlavorName, nil + } else if task.Type == TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { + index := strings.LastIndex(task.FlavorCode, ".") + cardNum, err := strconv.Atoi(strings.TrimSpace(task.FlavorCode[index+1 : len(task.FlavorCode)])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return "", err } + CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(task.FlavorCode[index+1:len(task.FlavorCode)]) + + "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + + "核" + strconv.Itoa(cardNum*256) + "GB" + return CloudbrainTwoFlavorName, nil } - return nil - + return "", nil } -// func GetCloudbrainTaskCardNum(task Cloudbrain) int { -// spec := GetCloudbrainResourceSpec(task.JobType, task.Type, task.ResourceSpecId, task.FlavorCode) -// if spec == nil { -// return 0 -// } -// if task.Type == TypeCloudBrainOne { -// if spec.ResourceSpec == nil { -// return 0 -// } -// return spec.ResourceSpec.GpuNum -// } else if task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net { -// return getCardnumByFlavorname(task.FlavorName) -// } -// return 0 -// } - -func getCardNumAndTypeByFlavorname(FlavorName string) (int, string) { - var beginIndex = strings.Index(FlavorName, ":") - var lastIndex = strings.LastIndex(FlavorName, ":") - var endIndex = strings.Index(FlavorName, "*") - cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex:endIndex])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return 0, "" - } - cardType := strings.TrimSpace(FlavorName[endIndex:lastIndex]) - return cardNum, cardType -} - -func GetCloudbrainCardNumAndType(task Cloudbrain) (int, string) { - if !SpecsMapInitFlag { - InitResourceSpecMap() - } - if !GpuInfosMapInitFlag { - InitGpuInfoMap() - } - if task.Type == TypeCloudBrainOne { - switch task.JobType { - case string(JobTypeDebug): - return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId].GpuNum, CloudbrainDebugGpuInfosMap[task.GpuQueue].Value - case string(JobTypeTrain): - return CloudbrainTrainResourceSpecsMap[task.ResourceSpecId].GpuNum, CloudbrainTrainGpuInfosMap[task.GpuQueue].Value +func GetCloudBrainOneResourceSpec(task Cloudbrain) (*ResourceSpec, *GpuInfo, error) { + if task.ResourceSpecId >= 0 { + if task.JobType == string(JobTypeTrain) { + return CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], CloudbrainTrainGpuInfosMap[task.GpuQueue], nil + } else if task.JobType == string(JobTypeDebug) { + return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], CloudbrainDebugGpuInfosMap[task.GpuQueue], nil + } else { + return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], CloudbrainDebugGpuInfosMap[task.GpuQueue], nil } - } else if task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net { - return getCardNumAndTypeByFlavorname(task.FlavorName) + } else { + err := errors.New("ResourceSpecId is null") + return nil, nil, err } - return 0, "" } diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 03cd7d2bc..86143d995 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -29,6 +29,11 @@ type TaskDetail struct { RepoAlias string `json:"RepoAlias"` RepoID int64 `json:"RepoID"` IsDelete bool `json:"IsDelete"` + CardNum int `json:"CardNum"` + CardType string `json:"CardType"` + CardDuration string `json:"CardDuration"` + AiCenter string `json:"AiCenter"` + FlavorName string `json:"FlavorName"` } func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index cc125c97f..aaa378113 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -751,27 +751,13 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias } - if ciTasks[i].Cloudbrain.Status == string(models.JobWaiting) { - WaitTimeInt := time.Now().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() - taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - taskDetail.WaitTime = "00:00:00" - } - } else if ciTasks[i].Cloudbrain.Status == string(models.JobStopped) && ciTasks[i].Cloudbrain.StartTime.AsTime().Unix() == 0 { - WaitTimeInt := ciTasks[i].Cloudbrain.EndTime.AsTime().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() - taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - taskDetail.WaitTime = "00:00:00" + taskDetail.CardNum, taskDetail.CardType, _ = models.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) - } - } else { - WaitTimeInt := ciTasks[i].Cloudbrain.StartTime.AsTime().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() - taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - taskDetail.WaitTime = "00:00:00" - } - } + taskDetail.CardDuration = models.ConvertDurationToStr(int64(taskDetail.CardNum) * ciTasks[i].Cloudbrain.Duration) + taskDetail.AiCenter = getCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) + taskDetail.FlavorName, _ = models.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) + taskDetail.WaitTime = getCloudbrainWaitTime(ciTasks[i].Cloudbrain) if ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainTwo || (ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainOne && ciTasks[i].Cloudbrain.JobType == "TRAIN") { taskDetail.JobID = ciTasks[i].Cloudbrain.JobID } @@ -797,6 +783,42 @@ func GetCloudbrainsDetailData(ctx *context.Context) { }) } +func getCloudbrainWaitTime(task models.Cloudbrain) string { + var WaitTime string + if task.Status == string(models.JobWaiting) { + WaitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + } + } else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 { + WaitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + + } + } else { + WaitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + } + } + return WaitTime +} + +func getCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { + if task.Type == models.TypeCloudBrainOne { + return ctx.Tr("repo.cloudbrain1") + } else if task.Type == models.TypeCloudBrainTwo { + return ctx.Tr("repo.cloudbrain2") + } else if task.Type == models.TypeC2Net { + return task.AiCenter + } + return "" +} + func GetCloudbrainsCreateHoursData(ctx *context.Context) { recordCloudbrain, err := models.GetRecordBeginTime() if err != nil { From d9f2a9658a8628f0c8a2a9536bcd9c9299d4e85a Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 25 Jul 2022 15:53:03 +0800 Subject: [PATCH 05/14] fix-2454 --- models/cloudbrain.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index dc316e943..351d64a69 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2083,13 +2083,16 @@ func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { var beginIndex = strings.Index(FlavorName, ":") var lastIndex = strings.LastIndex(FlavorName, ":") var endIndex = strings.Index(FlavorName, "*") - cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return 0, "", err + if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { + cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return 0, "", err + } + cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) + return cardNum, cardType, err } - cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) - return cardNum, cardType, err + return 0, "", nil } } @@ -2109,9 +2112,9 @@ func GetCloudbrainCardNumAndType(task Cloudbrain) (int, string, error) { func GetCloudbrainFlavorName(task Cloudbrain) (string, error) { if task.Type == TypeCloudBrainOne { - ResourceSpec, GpuInfo, err := GetCloudBrainOneResourceSpec(task) + ResourceSpec, GpuInfo, err := getCloudBrainOneResourceSpec(task) if err != nil { - log.Info("GetCloudBrainOneResourceSpec err:", err) + log.Info("getCloudBrainOneResourceSpec err:", err) return "", err } else { if ResourceSpec == nil || GpuInfo == nil { @@ -2141,7 +2144,7 @@ func GetCloudbrainFlavorName(task Cloudbrain) (string, error) { return "", nil } -func GetCloudBrainOneResourceSpec(task Cloudbrain) (*ResourceSpec, *GpuInfo, error) { +func getCloudBrainOneResourceSpec(task Cloudbrain) (*ResourceSpec, *GpuInfo, error) { if task.ResourceSpecId >= 0 { if task.JobType == string(JobTypeTrain) { return CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], CloudbrainTrainGpuInfosMap[task.GpuQueue], nil From 9f55b0c082e7add5dfad7be9f0b4932dfc26c5d5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 25 Jul 2022 16:41:36 +0800 Subject: [PATCH 06/14] =?UTF-8?q?fix-2357=E2=80=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 ++ routers/admin/cloudbrains.go | 5 +++++ routers/api/v1/repo/cloudbrain_dashboard.go | 3 ++- routers/repo/cloudbrain.go | 21 +++++++++++++++++- routers/user/home.go | 24 +++++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 351d64a69..e54596a6b 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -137,6 +137,8 @@ type Cloudbrain struct { Type int BenchmarkTypeID int BenchmarkChildTypeID int + CardType string + Cluster string VersionID int64 //版本id VersionName string `xorm:"INDEX"` //当前版本 diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 8cfe10795..6f7ae12c0 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -10,6 +10,7 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -89,6 +90,10 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) + _, cardType, _ := models.GetCloudbrainCardNumAndType(task.Cloudbrain) + ciTasks[i].Cloudbrain.CardType = cardType + ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index aaa378113..512e412e6 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/repo" "github.com/360EntSecGroup-Skylar/excelize/v2" ) @@ -754,7 +755,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.CardNum, taskDetail.CardType, _ = models.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) taskDetail.CardDuration = models.ConvertDurationToStr(int64(taskDetail.CardNum) * ciTasks[i].Cloudbrain.Duration) - taskDetail.AiCenter = getCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) + taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) taskDetail.FlavorName, _ = models.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) taskDetail.WaitTime = getCloudbrainWaitTime(ciTasks[i].Cloudbrain) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5a3d0a6f8..21b3de75b 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,6 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/grampus" "encoding/json" "errors" "fmt" @@ -16,6 +15,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/grampus" + "code.gitea.io/gitea/modules/timeutil" "github.com/unknwon/i18n" @@ -2309,3 +2310,21 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { } return benchmarkTypesMap[lang] } +func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { + if task.Type == models.TypeCloudBrainOne { + return ctx.Tr("repo.cloudbrain1") + } else if task.Type == models.TypeCloudBrainTwo { + return ctx.Tr("repo.cloudbrain2") + } else if task.Type == models.TypeC2Net { + return task.AiCenter + } + return "" +} +func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { + if task.Type == models.TypeCloudBrainOne || task.Type == models.TypeCloudBrainTwo { + return ctx.Tr("cloudbrain.resource_cluster_openi") + } else if task.Type == models.TypeC2Net { + return ctx.Tr("cloudbrain.resource_cluster_c2net") + } + return "" +} diff --git a/routers/user/home.go b/routers/user/home.go index ab64e707f..e219cffc6 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -834,6 +834,11 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + ciTasks[i].Cloudbrain.AiCenter = GetCloudbrainAiCenter(task.Cloudbrain, ctx) + _, cardType, _ := models.GetCloudbrainCardNumAndType(task.Cloudbrain) + ciTasks[i].Cloudbrain.CardType = cardType + ciTasks[i].Cloudbrain.Cluster = GetCloudbrainCluster(task.Cloudbrain, ctx) + } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) @@ -857,3 +862,22 @@ func getTotalPage(total int64, pageSize int) int { return int(total)/pageSize + another } + +func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { + if task.Type == models.TypeCloudBrainOne { + return ctx.Tr("repo.cloudbrain1") + } else if task.Type == models.TypeCloudBrainTwo { + return ctx.Tr("repo.cloudbrain2") + } else if task.Type == models.TypeC2Net { + return task.AiCenter + } + return "" +} +func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { + if task.Type == models.TypeCloudBrainOne || task.Type == models.TypeCloudBrainTwo { + return ctx.Tr("cloudbrain.resource_cluster_openi") + } else if task.Type == models.TypeC2Net { + return ctx.Tr("cloudbrain.resource_cluster_c2net") + } + return "" +} From 974dd5340f9c4967a090b02805be8fd87cb7ceaf Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 25 Jul 2022 18:18:31 +0800 Subject: [PATCH 07/14] fix-bug --- models/cloudbrain.go | 83 --------------- options/locale/locale_zh-CN.ini | 2 + routers/admin/cloudbrains.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 76 +++++-------- routers/repo/cloudbrain.go | 112 ++++++++++++++++++++ routers/user/home.go | 26 +---- 6 files changed, 144 insertions(+), 157 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e54596a6b..4fc2f9cc3 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2077,86 +2077,3 @@ func InitCloudbrainOneGpuInfoMap() { } GpuInfosMapInitFlag = true } - -func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { - if FlavorName == "" { - return 0, "", nil - } else { - var beginIndex = strings.Index(FlavorName, ":") - var lastIndex = strings.LastIndex(FlavorName, ":") - var endIndex = strings.Index(FlavorName, "*") - if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { - cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return 0, "", err - } - cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) - return cardNum, cardType, err - } - return 0, "", nil - } -} - -func GetCloudbrainCardNumAndType(task Cloudbrain) (int, string, error) { - if !SpecsMapInitFlag { - InitCloudbrainOneResourceSpecMap() - } - if !GpuInfosMapInitFlag { - InitCloudbrainOneGpuInfoMap() - } - FlavorName, err := GetCloudbrainFlavorName(task) - if err != nil { - return 0, "", nil - } - return getCardNumAndTypeByFlavorname(FlavorName) -} - -func GetCloudbrainFlavorName(task Cloudbrain) (string, error) { - if task.Type == TypeCloudBrainOne { - ResourceSpec, GpuInfo, err := getCloudBrainOneResourceSpec(task) - if err != nil { - log.Info("getCloudBrainOneResourceSpec err:", err) - return "", err - } else { - if ResourceSpec == nil || GpuInfo == nil { - err := errors.New("ResourceSpec or GpuInfo is nil") - return "", err - } else { - CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(ResourceSpec.GpuNum) + "*Nvidia-" + GpuInfo.Value + - " | CPU:" + strconv.Itoa(ResourceSpec.CpuNum) + "核" + strconv.Itoa(ResourceSpec.MemMiB) + "MB" - return CloudbrainOneFlavorName, nil - } - } - } else if (task.Type == TypeCloudBrainTwo || task.Type == TypeC2Net) && task.FlavorName != "" { - return task.FlavorName, nil - } else if task.Type == TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { - index := strings.LastIndex(task.FlavorCode, ".") - cardNum, err := strconv.Atoi(strings.TrimSpace(task.FlavorCode[index+1 : len(task.FlavorCode)])) - if err != nil { - log.Error("strconv.Atoi failed: %v", err) - return "", err - } - CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(task.FlavorCode[index+1:len(task.FlavorCode)]) + - "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + - "核" + strconv.Itoa(cardNum*256) + "GB" - return CloudbrainTwoFlavorName, nil - } - - return "", nil -} - -func getCloudBrainOneResourceSpec(task Cloudbrain) (*ResourceSpec, *GpuInfo, error) { - if task.ResourceSpecId >= 0 { - if task.JobType == string(JobTypeTrain) { - return CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], CloudbrainTrainGpuInfosMap[task.GpuQueue], nil - } else if task.JobType == string(JobTypeDebug) { - return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], CloudbrainDebugGpuInfosMap[task.GpuQueue], nil - } else { - return CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], CloudbrainDebugGpuInfosMap[task.GpuQueue], nil - } - } else { - err := errors.New("ResourceSpecId is null") - return nil, nil, err - } -} diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index b9acb6ec0..cb7982749 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3069,6 +3069,8 @@ select_dataset = 选择数据集 specification = 规格 select_specification = 选择资源规格 description = 描述 +card_duration = 运行卡时 +card_type = 卡类型 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 dataset_path_rule = 数据集位置存储在环境变量data_url中，训练输出路径存储在环境变量train_url中。 diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 6f7ae12c0..a3f9ba929 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -91,7 +91,7 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) - _, cardType, _ := models.GetCloudbrainCardNumAndType(task.Cloudbrain) + _, cardType, _ := repo.GetCloudbrainCardNumAndType(task.Cloudbrain) ciTasks[i].Cloudbrain.CardType = cardType ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 512e412e6..1053dca01 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -752,13 +752,12 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias } - taskDetail.CardNum, taskDetail.CardType, _ = models.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) - - taskDetail.CardDuration = models.ConvertDurationToStr(int64(taskDetail.CardNum) * ciTasks[i].Cloudbrain.Duration) + taskDetail.CardNum, taskDetail.CardType, _ = repo.GetCloudbrainCardNumAndType(ciTasks[i].Cloudbrain) + taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) taskDetail.AiCenter = repo.GetCloudbrainAiCenter(ciTasks[i].Cloudbrain, ctx) - taskDetail.FlavorName, _ = models.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) + taskDetail.FlavorName, _ = repo.GetCloudbrainFlavorName(ciTasks[i].Cloudbrain) - taskDetail.WaitTime = getCloudbrainWaitTime(ciTasks[i].Cloudbrain) + taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) if ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainTwo || (ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainOne && ciTasks[i].Cloudbrain.JobType == "TRAIN") { taskDetail.JobID = ciTasks[i].Cloudbrain.JobID } @@ -784,31 +783,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { }) } -func getCloudbrainWaitTime(task models.Cloudbrain) string { - var WaitTime string - if task.Status == string(models.JobWaiting) { - WaitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" - } - } else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 { - WaitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" - - } - } else { - WaitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() - WaitTime = models.ConvertDurationToStr(WaitTimeInt) - if WaitTimeInt < 0 { - WaitTime = "00:00:00" - } - } - return WaitTime -} - func getCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { if task.Type == models.TypeCloudBrainOne { return ctx.Tr("repo.cloudbrain1") @@ -1254,18 +1228,23 @@ func allCloudbrainHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_type"), "C1": ctx.Tr("repo.modelarts.status"), "D1": ctx.Tr("repo.cloudbrain_task_type"), "E1": ctx.Tr("repo.modelarts.createtime"), "F1": ctx.Tr("repo.modelarts.train_job.wait_time"), "G1": ctx.Tr("repo.modelarts.train_job.dura_time"), - "H1": ctx.Tr("repo.modelarts.train_job.start_time"), - "I1": ctx.Tr("repo.modelarts.train_job.end_time"), "J1": ctx.Tr("repo.modelarts.computing_resources"), - "K1": ctx.Tr("repo.cloudbrain_creator"), "L1": ctx.Tr("repo.repo_name"), "M1": ctx.Tr("repo.cloudbrain_task_name"), "N1": ctx.Tr("repo.modelarts.deletetime")} + "H1": ctx.Tr("cloudbrain.card_duration"), + "I1": ctx.Tr("repo.modelarts.train_job.start_time"), "J1": ctx.Tr("repo.modelarts.train_job.end_time"), + "K1": ctx.Tr("repo.modelarts.computing_resources"), "L1": ctx.Tr("cloudbrain.resource_specification"), + "M1": ctx.Tr("repo.grampus.train_job.ai_center"), "N1": ctx.Tr("cloudbrain.card_type"), + "O1": ctx.Tr("repo.cloudbrain_creator"), "P1": ctx.Tr("repo.repo_name"), "Q1": ctx.Tr("repo.cloudbrain_task_name"), + "R1": ctx.Tr("repo.modelarts.deletetime")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): getCloudbrainType(rs, ctx), getCellName("C", row): rs.Status, getCellName("D", row): rs.JobType, - getCellName("E", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("F", row): getBrainWaitTime(rs), - getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): getBrainStartTime(rs), - getCellName("I", row): getBrainEndTime(rs), - getCellName("J", row): rs.ComputeResource, getCellName("K", row): rs.Name, getCellName("L", row): getBrainRepo(rs), - getCellName("M", row): rs.JobName, getCellName("N", row): getBrainDeleteTime(rs), + getCellName("E", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("F", row): repo.GetCloudbrainWaitTime(rs.Cloudbrain), + getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): repo.GetCloudbrainCardDuration(rs.Cloudbrain), + getCellName("I", row): getBrainStartTime(rs), + getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainFlavorName(rs), + getCellName("M", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("N", row): getCloudbrainCardType(rs), + getCellName("O", row): rs.Name, getCellName("P", row): getBrainRepo(rs), + getCellName("Q", row): rs.JobName, getCellName("R", row): getBrainDeleteTime(rs), } } func getBrainRepo(rs *models.CloudbrainInfo) string { @@ -1292,19 +1271,6 @@ func getBrainEndTime(rs *models.CloudbrainInfo) string { } } -func getBrainWaitTime(rs *models.CloudbrainInfo) string { - var waitTime int64 - if rs.Cloudbrain.Status == string(models.JobWaiting) { - waitTime = time.Now().Unix() - rs.Cloudbrain.CreatedUnix.AsTime().Unix() - } else { - waitTime = int64(rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix) - } - if waitTime <= 0 { - return "00:00:00" - } else { - return models.ConvertDurationToStr(waitTime) - } -} func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string { if rs.Cloudbrain.Type == models.TypeCloudBrainOne { return ctx.Tr("repo.cloudbrain1") @@ -1316,6 +1282,14 @@ func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string { return ctx.Tr("repo.cloudbrain_untype") } } +func getCloudbrainCardType(rs *models.CloudbrainInfo) string { + _, cardType, _ := repo.GetCloudbrainCardNumAndType(rs.Cloudbrain) + return cardType +} +func getCloudbrainFlavorName(rs *models.CloudbrainInfo) string { + flavorName, _ := repo.GetCloudbrainFlavorName(rs.Cloudbrain) + return flavorName +} func getBrainDeleteTime(rs *models.CloudbrainInfo) string { nilTime := time.Time{} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 21b3de75b..7684b3429 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2328,3 +2328,115 @@ func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { } return "" } +func GetCloudbrainCardDuration(task models.Cloudbrain) string { + CardNum, _, _ := GetCloudbrainCardNumAndType(task) + CardDuration := models.ConvertDurationToStr(int64(CardNum) * task.Duration) + return CardDuration +} +func GetCloudbrainWaitTime(task models.Cloudbrain) string { + var WaitTime string + if task.Status == string(models.JobWaiting) { + WaitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + } + } else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 { + WaitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + + } + } else { + WaitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix() + WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + WaitTime = "00:00:00" + } + } + return WaitTime +} + +func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) { + if !models.SpecsMapInitFlag { + models.InitCloudbrainOneResourceSpecMap() + } + if !models.GpuInfosMapInitFlag { + models.InitCloudbrainOneGpuInfoMap() + } + FlavorName, err := GetCloudbrainFlavorName(task) + if err != nil { + return 0, "", nil + } + return getCardNumAndTypeByFlavorname(FlavorName) +} + +func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) { + if FlavorName == "" { + return 0, "", nil + } else { + var beginIndex = strings.Index(FlavorName, ":") + var lastIndex = strings.LastIndex(FlavorName, ":") + var endIndex = strings.Index(FlavorName, "*") + if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) { + cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return 0, "", err + } + cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex]) + return cardNum, cardType, err + } + return 0, "", nil + } +} + +func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { + if task.Type == models.TypeCloudBrainOne { + ResourceSpec, GpuInfo, err := getCloudBrainOneResourceSpec(task) + if err != nil { + log.Info("getCloudBrainOneResourceSpec err:", err) + return "", err + } else { + if ResourceSpec == nil || GpuInfo == nil { + err := errors.New("ResourceSpec or GpuInfo is nil") + return "", err + } else { + CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(ResourceSpec.GpuNum) + "*Nvidia-" + GpuInfo.Value + + " | CPU:" + strconv.Itoa(ResourceSpec.CpuNum) + "核" + strconv.Itoa(ResourceSpec.MemMiB) + "MB" + return CloudbrainOneFlavorName, nil + } + } + } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net) && task.FlavorName != "" { + return task.FlavorName, nil + } else if task.Type == models.TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { + index := strings.LastIndex(task.FlavorCode, ".") + cardNum, err := strconv.Atoi(strings.TrimSpace(task.FlavorCode[index+1 : len(task.FlavorCode)])) + if err != nil { + log.Error("strconv.Atoi failed: %v", err) + return "", err + } + CloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(task.FlavorCode[index+1:len(task.FlavorCode)]) + + "*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) + + "核" + strconv.Itoa(cardNum*256) + "GB" + return CloudbrainTwoFlavorName, nil + } + + return "", nil +} + +func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) { + if task.ResourceSpecId >= 0 { + if task.JobType == string(models.JobTypeTrain) { + return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[task.GpuQueue], nil + } else if task.JobType == string(models.JobTypeDebug) { + return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[task.GpuQueue], nil + } else { + return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[task.GpuQueue], nil + } + } else { + err := errors.New("ResourceSpecId is null") + return nil, nil, err + } +} diff --git a/routers/user/home.go b/routers/user/home.go index e219cffc6..25b1c518e 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/repo" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" @@ -834,10 +835,10 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - ciTasks[i].Cloudbrain.AiCenter = GetCloudbrainAiCenter(task.Cloudbrain, ctx) - _, cardType, _ := models.GetCloudbrainCardNumAndType(task.Cloudbrain) + ciTasks[i].Cloudbrain.AiCenter = repo.GetCloudbrainAiCenter(task.Cloudbrain, ctx) + _, cardType, _ := repo.GetCloudbrainCardNumAndType(task.Cloudbrain) ciTasks[i].Cloudbrain.CardType = cardType - ciTasks[i].Cloudbrain.Cluster = GetCloudbrainCluster(task.Cloudbrain, ctx) + ciTasks[i].Cloudbrain.Cluster = repo.GetCloudbrainCluster(task.Cloudbrain, ctx) } @@ -862,22 +863,3 @@ func getTotalPage(total int64, pageSize int) int { return int(total)/pageSize + another } - -func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne { - return ctx.Tr("repo.cloudbrain1") - } else if task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("repo.cloudbrain2") - } else if task.Type == models.TypeC2Net { - return task.AiCenter - } - return "" -} -func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string { - if task.Type == models.TypeCloudBrainOne || task.Type == models.TypeCloudBrainTwo { - return ctx.Tr("cloudbrain.resource_cluster_openi") - } else if task.Type == models.TypeC2Net { - return ctx.Tr("cloudbrain.resource_cluster_c2net") - } - return "" -} From 40e4a17d13d0147827fdb8c4446dad657236f4af Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 26 Jul 2022 11:07:47 +0800 Subject: [PATCH 08/14] fix-bug --- routers/api/v1/repo/cloudbrain_dashboard.go | 8 ++++---- routers/repo/cloudbrain.go | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 1053dca01..5c4d53946 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1230,8 +1230,8 @@ func allCloudbrainHeader(ctx *context.Context) map[string]string { "E1": ctx.Tr("repo.modelarts.createtime"), "F1": ctx.Tr("repo.modelarts.train_job.wait_time"), "G1": ctx.Tr("repo.modelarts.train_job.dura_time"), "H1": ctx.Tr("cloudbrain.card_duration"), "I1": ctx.Tr("repo.modelarts.train_job.start_time"), "J1": ctx.Tr("repo.modelarts.train_job.end_time"), - "K1": ctx.Tr("repo.modelarts.computing_resources"), "L1": ctx.Tr("cloudbrain.resource_specification"), - "M1": ctx.Tr("repo.grampus.train_job.ai_center"), "N1": ctx.Tr("cloudbrain.card_type"), + "K1": ctx.Tr("repo.modelarts.computing_resources"), "L1": ctx.Tr("cloudbrain.card_type"), + "M1": ctx.Tr("repo.grampus.train_job.ai_center"), "N1": ctx.Tr("cloudbrain.resource_specification"), "O1": ctx.Tr("repo.cloudbrain_creator"), "P1": ctx.Tr("repo.repo_name"), "Q1": ctx.Tr("repo.cloudbrain_task_name"), "R1": ctx.Tr("repo.modelarts.deletetime")} @@ -1241,8 +1241,8 @@ func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Contex getCellName("E", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("F", row): repo.GetCloudbrainWaitTime(rs.Cloudbrain), getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): repo.GetCloudbrainCardDuration(rs.Cloudbrain), getCellName("I", row): getBrainStartTime(rs), - getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainFlavorName(rs), - getCellName("M", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("N", row): getCloudbrainCardType(rs), + getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainCardType(rs), + getCellName("M", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("N", row): getCloudbrainFlavorName(rs), getCellName("O", row): rs.Name, getCellName("P", row): getBrainRepo(rs), getCellName("Q", row): rs.JobName, getCellName("R", row): getBrainDeleteTime(rs), } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 7684b3429..a5a470cb1 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2409,7 +2409,8 @@ func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { } } } else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net) && task.FlavorName != "" { - return task.FlavorName, nil + ReplaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":") + return ReplaceFlavorName, nil } else if task.Type == models.TypeCloudBrainTwo && task.FlavorName == "" && task.FlavorCode != "" { index := strings.LastIndex(task.FlavorCode, ".") cardNum, err := strconv.Atoi(strings.TrimSpace(task.FlavorCode[index+1 : len(task.FlavorCode)])) @@ -2427,13 +2428,17 @@ func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) { } func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) { + GpuQueueDefault := "openidebug" + if task.GpuQueue != "" { + GpuQueueDefault = task.GpuQueue + } if task.ResourceSpecId >= 0 { if task.JobType == string(models.JobTypeTrain) { - return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[task.GpuQueue], nil + return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[GpuQueueDefault], nil } else if task.JobType == string(models.JobTypeDebug) { - return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[task.GpuQueue], nil + return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[GpuQueueDefault], nil } else { - return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[task.GpuQueue], nil + return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[GpuQueueDefault], nil } } else { err := errors.New("ResourceSpecId is null") From 909dfea677cfe62b441bd89cfaa9c611eb2d9836 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 26 Jul 2022 12:03:50 +0800 Subject: [PATCH 09/14] fix-2454 --- models/cloudbrain.go | 27 +++++++++++++++++++++++++++ routers/repo/cloudbrain.go | 10 +++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 4d81e65a7..200ae83ad 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2097,10 +2097,13 @@ var ( SpecsMapInitFlag = false CloudbrainDebugResourceSpecsMap map[int]*ResourceSpec CloudbrainTrainResourceSpecsMap map[int]*ResourceSpec + CloudbrainInferenceResourceSpecsMap map[int]*ResourceSpec CloudbrainBenchmarkResourceSpecsMap map[int]*ResourceSpec GpuInfosMapInitFlag = false CloudbrainDebugGpuInfosMap map[string]*GpuInfo CloudbrainTrainGpuInfosMap map[string]*GpuInfo + CloudbrainInferenceGpuInfosMap map[string]*GpuInfo + CloudbrainBenchmarkGpuInfosMap map[string]*GpuInfo ) func InitCloudbrainOneResourceSpecMap() { @@ -2120,6 +2123,14 @@ func InitCloudbrainOneResourceSpecMap() { CloudbrainTrainResourceSpecsMap[spec.Id] = spec } } + if CloudbrainInferenceResourceSpecsMap == nil || len(CloudbrainInferenceResourceSpecsMap) == 0 { + t := ResourceSpecs{} + json.Unmarshal([]byte(setting.InferenceResourceSpecs), &t) + CloudbrainInferenceResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) + for _, spec := range t.ResourceSpec { + CloudbrainInferenceResourceSpecsMap[spec.Id] = spec + } + } if CloudbrainBenchmarkResourceSpecsMap == nil || len(CloudbrainBenchmarkResourceSpecsMap) == 0 { t := ResourceSpecs{} json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &t) @@ -2148,6 +2159,22 @@ func InitCloudbrainOneGpuInfoMap() { CloudbrainTrainGpuInfosMap[GpuInfo.Queue] = GpuInfo } } + if CloudbrainInferenceGpuInfosMap == nil || len(CloudbrainInferenceGpuInfosMap) == 0 { + t := GpuInfos{} + json.Unmarshal([]byte(setting.InferenceGpuTypes), &t) + CloudbrainInferenceGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) + for _, GpuInfo := range t.GpuInfo { + CloudbrainInferenceGpuInfosMap[GpuInfo.Queue] = GpuInfo + } + } + if CloudbrainBenchmarkGpuInfosMap == nil || len(CloudbrainBenchmarkGpuInfosMap) == 0 { + t := GpuInfos{} + json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &t) + CloudbrainBenchmarkGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) + for _, GpuInfo := range t.GpuInfo { + CloudbrainBenchmarkGpuInfosMap[GpuInfo.Queue] = GpuInfo + } + } GpuInfosMapInitFlag = true } func GetNewestJobsByAiCenter() ([]int64, error) { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0d2998d90..53f465fae 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -373,7 +373,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } } - func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBrainInferencForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName @@ -494,6 +493,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/inference-job") } + /** 检查用户传输的参数是否符合专属资源池 */ @@ -2665,6 +2665,7 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { } return benchmarkTypesMap[lang] } + func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string { if task.Type == models.TypeCloudBrainOne { return ctx.Tr("repo.cloudbrain1") @@ -2792,11 +2793,14 @@ func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[GpuQueueDefault], nil } else if task.JobType == string(models.JobTypeDebug) { return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[GpuQueueDefault], nil - } else { - return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[GpuQueueDefault], nil + } else if task.JobType == string(models.JobTypeInference) { + return models.CloudbrainInferenceResourceSpecsMap[task.ResourceSpecId], models.CloudbrainInferenceGpuInfosMap[GpuQueueDefault], nil + } else if task.JobType == string(models.JobTypeBenchmark) || task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { + return models.CloudbrainBenchmarkResourceSpecsMap[task.ResourceSpecId], models.CloudbrainBenchmarkGpuInfosMap[GpuQueueDefault], nil } } else { err := errors.New("ResourceSpecId is null") return nil, nil, err } + return nil, nil, nil } From 2ac84599ca5f92d3fe4b5330ec2a7c4a0185b09b Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 26 Jul 2022 15:49:08 +0800 Subject: [PATCH 10/14] fix issue --- options/locale/locale_en-US.ini | 3 + options/locale/locale_zh-CN.ini | 3 + templates/admin/cloudbrain/list.tmpl | 159 ++++++++++++------ .../admin/cloudbrain/search_dashboard.tmpl | 2 +- templates/user/dashboard/cloudbrains.tmpl | 123 +++++++++----- 5 files changed, 201 insertions(+), 89 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 105abf006..e9f0ed8c0 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1098,6 +1098,9 @@ modelarts.createtime=CreateTime modelarts.version_nums = Version Nums modelarts.version = Version modelarts.computing_resources=compute Resources +modelarts.ai_center=Ai Center +modelarts.card_type=Card Type +modelarts.cluster=Cluster modelarts.notebook=Debug Task modelarts.train_job=Train Task modelarts.train_job.new_debug= New Debug Task diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 3d71bc3a7..cb37fa8b7 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1108,6 +1108,9 @@ modelarts.deletetime=删除时间 modelarts.version_nums=版本数 modelarts.version=版本 modelarts.computing_resources=计算资源 +modelarts.ai_center=智算中心 +modelarts.card_type=卡类型 +modelarts.cluster=集群 modelarts.notebook=调试任务 modelarts.train_job=训练任务 modelarts.train_job.new_debug=新建调试任务 diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index e66f40e84..1ac107484 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -18,7 +18,7 @@ data-all-compute="{{.i18n.Tr "admin.cloudbrain.all_computing_resources"}}" data-all-status="{{.i18n.Tr "admin.cloudbrain.all_status"}}"> {{template "admin/navbar" .}} -
+
{{template "base/alert" .}}
@@ -34,34 +34,46 @@
-
- {{$.i18n.Tr "repo.cloudbrain_task"}} +
+ {{$.i18n.Tr "repo.cloudbrain_task"}}
-
- {{$.i18n.Tr "repo.cloudbrain_task_type"}} + +
+ {{$.i18n.Tr "repo.modelarts.cluster"}}
-
+
+ {{$.i18n.Tr "repo.cloudbrain_task_type"}} +
+
{{$.i18n.Tr "repo.modelarts.status"}}
-
+
{{$.i18n.Tr "repo.modelarts.createtime"}}
-
+
{{$.i18n.Tr "repo.cloudbrain_status_runtime"}}
-
+
{{$.i18n.Tr "repo.modelarts.computing_resources"}}
-
+ +
+ {{$.i18n.Tr "repo.modelarts.ai_center"}} +
+ +
+ {{$.i18n.Tr "repo.modelarts.card_type"}} +
+
{{$.i18n.Tr "repo.cloudbrain_creator"}}
-
+
{{$.i18n.Tr "repository"}}
-
+
{{.i18n.Tr "admin.cloudbrain.cloudbrain_name"}}
-
+
{{$.i18n.Tr "repo.cloudbrain_operate"}}
@@ -78,51 +90,52 @@ {{$JobID = .JobID}} {{end}} -
+
{{if eq .JobType "DEBUG"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "INFERENCE"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "TRAIN"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "BENCHMARK"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{end}}
- -
- {{.JobType}} + +
+ {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
+ style="width: 6% !important;"> @@ -131,23 +144,39 @@ style="margin-left: 0.4em;font-size: 12px;">{{.Status}}
+ +
+ {{.JobType}} +
+ -
+
{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
-
+
{{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}}
-
+
{{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}}
+ +
+ {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} +
+ +
+ + {{if .CardType}}{{.CardType}}{{else}}--{{end}} + +
-
+
{{if .User.Name}} @@ -157,16 +186,24 @@ {{end}}
-
+
{{.Repo.OwnerName}}/{{.Repo.Alias}}
- {{.JobName}} + style="overflow: hidden;text-overflow:ellipsis;width:10% !important;"> + + {{.JobName}} +
-
+
{{if eq .JobType "DEBUG"}}
@@ -235,36 +272,37 @@
-
+
{{if eq .JobType "DEBUG"}} - + {{.DisplayJobName}} {{else if eq .JobType "INFERENCE"}} - + {{.DisplayJobName}} {{else if eq .JobType "TRAIN"}} - + {{.DisplayJobName}} {{else if eq .JobType "BENCHMARK"}} - + {{.DisplayJobName}} {{end}}
- -
- {{.JobType}} + +
+ {{if .Cluster}}{{.Cluster}}{{else}}--{{end}}
+ style="width: 6% !important;"> {{.Status}}
+ +
+ {{.JobType}} +
+ -
+
{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
-
+
{{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}}
-
+
{{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}}
+ +
+ {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} +
+ +
+ + {{if .CardType}}{{.CardType}}{{else}}--{{end}} + +
-
+
{{if .User.Name}} @@ -298,15 +352,24 @@ {{end}}
-
+
--
- {{.JobName}} + style="overflow: hidden;text-overflow:ellipsis;width:10% !important;"> + + {{.JobName}} + +
-
+
{{if eq .JobType "DEBUG"}}
diff --git a/templates/admin/cloudbrain/search_dashboard.tmpl b/templates/admin/cloudbrain/search_dashboard.tmpl index 61f02f182..e4d74836d 100644 --- a/templates/admin/cloudbrain/search_dashboard.tmpl +++ b/templates/admin/cloudbrain/search_dashboard.tmpl @@ -15,7 +15,7 @@
-
+
diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl index dc0337316..8019bc51d 100755 --- a/templates/user/dashboard/cloudbrains.tmpl +++ b/templates/user/dashboard/cloudbrains.tmpl @@ -20,7 +20,7 @@ data-all-compute="{{.i18n.Tr "admin.cloudbrain.all_computing_resources"}}" data-all-status="{{.i18n.Tr "admin.cloudbrain.all_status"}}">
{{template "admin/cloudbrain/search_dashboard" .}} -
+
{{template "base/alert" .}}
@@ -30,30 +30,42 @@
-
- {{$.i18n.Tr "repo.cloudbrain_task"}} +
+ {{$.i18n.Tr "repo.cloudbrain_task"}}
-
+ +
+ {{$.i18n.Tr "repo.modelarts.cluster"}} +
+
{{$.i18n.Tr "repo.modelarts.status"}}
-
- {{$.i18n.Tr "repo.cloudbrain_task_type"}} +
+ {{$.i18n.Tr "repo.cloudbrain_task_type"}}
-
+
{{$.i18n.Tr "repo.modelarts.createtime"}}
-
+
{{$.i18n.Tr "repo.cloudbrain_status_runtime"}}
-
+
{{$.i18n.Tr "repo.modelarts.computing_resources"}}
- -
+ + +
+ {{$.i18n.Tr "repo.modelarts.ai_center"}} +
+ +
+ {{$.i18n.Tr "repo.modelarts.card_type"}} +
+
{{$.i18n.Tr "repository"}}
-
+
{{$.i18n.Tr "repo.cloudbrain_operate"}}
@@ -70,48 +82,52 @@ {{$JobID = .JobID}} {{end}} -
+
{{if eq .JobType "DEBUG"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "INFERENCE"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "TRAIN"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{else if eq .JobType "BENCHMARK"}} + title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> {{.DisplayJobName}} {{end}}
- + +
+ {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} +
+ style="width: 8% !important;"> @@ -123,32 +139,44 @@ {{$JobType := $.i18n.Tr (printf "cloudbrain.%s" .JobType)}} -
+
{{$JobType}}
-
+
{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
-
+
{{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}}
-
+
{{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}}
+ + +
+ {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} +
+ +
+ + {{if .CardType}}{{.CardType}}{{else}}--{{end}} + +
-
+
{{.Repo.OwnerName}}/{{.Repo.Alias}}
-
+
{{if eq .JobType "DEBUG"}}
@@ -226,33 +254,37 @@
-
+
{{if eq .JobType "DEBUG"}} - + {{.DisplayJobName}} {{else if eq .JobType "INFERENCE"}} - + {{.DisplayJobName}} {{else if eq .JobType "TRAIN"}} - + {{.DisplayJobName}} {{else if eq .JobType "BENCHMARK"}} - + {{.DisplayJobName}} {{end}}
- + +
+ {{if .Cluster}}{{.Cluster}}{{else}}--{{end}} +
+ style="padding-left: 2.2rem !important; width: 8% !important;"> {{$JobType := $.i18n.Tr (printf "cloudbrain.%s" .JobType)}} -
+
{{$JobType}}
-
+
{{TimeSinceUnix1 .Cloudbrain.CreatedUnix}}
-
+
{{if .TrainJobDuration}}{{.TrainJobDuration}}{{else}}--{{end}}
-
+
{{if .ComputeResource}}{{.ComputeResource}}{{else}}--{{end}}
- + + +
+ {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} +
+ +
+ + {{if .CardType}}{{.CardType}}{{else}}--{{end}} + +
-
+
--
-
+
{{if eq .JobType "DEBUG"}}
From fdd622334741b2da9e44ad7795ee81862aa9b5cb Mon Sep 17 00:00:00 2001 From: chenshihai Date: Tue, 26 Jul 2022 16:03:41 +0800 Subject: [PATCH 11/14] =?UTF-8?q?fix-2508=20=E6=96=B0=E5=BB=BAgpu=E8=AE=AD?= =?UTF-8?q?=E7=BB=83=E4=BB=BB=E5=8A=A1=E9=A1=B5=E9=9D=A2=E7=9A=84=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=90=8D=E6=8F=90=E7=A4=BA=E6=96=87=E5=AD=97=E4=B8=8E?= =?UTF-8?q?=E5=AE=9E=E9=99=85=E5=91=BD=E5=90=8D=E8=A7=84=E5=88=99=E4=B8=8D?= =?UTF-8?q?=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/grampus.go | 2 +- routers/repo/modelarts.go | 8 ++++---- templates/repo/cloudbrain/benchmark/new.tmpl | 10 +++++----- templates/repo/cloudbrain/inference/new.tmpl | 6 +++--- templates/repo/cloudbrain/trainjob/new.tmpl | 6 +++--- templates/repo/grampus/trainjob/gpu/new.tmpl | 6 +++--- templates/repo/grampus/trainjob/npu/new.tmpl | 6 +++--- templates/repo/modelarts/inferencejob/new.tmpl | 6 +++--- templates/repo/modelarts/notebook/new.tmpl | 8 ++++---- templates/repo/modelarts/trainjob/new.tmpl | 6 +++--- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index a41f884d9..42d345530 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -66,7 +66,7 @@ func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) err ctx.Data["PageIsCloudBrain"] = true t := time.Now() - var displayJobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["display_job_name"] = displayJobName //get valid images diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index e5558af7d..d824cb05b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -658,7 +658,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { //} t := time.Now() - var displayJobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) @@ -726,7 +726,7 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts //} t := time.Now() - var displayJobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) @@ -906,7 +906,7 @@ func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai } t := time.Now() - var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = task.JobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) @@ -2171,7 +2171,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { ctx.Data["newInference"] = true t := time.Now() - var displayJobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index fb1296d27..e1b5c890e 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -69,8 +69,8 @@ - {{.i18n.Tr "cloudbrain.job_name_rule"}} + tabindex="3" autofocus required maxlength="36"> + {{.i18n.Tr "repo.cloudbrain_jobname_err"}}
- {{.i18n.Tr "cloudbrain.job_name_rule"}} + tabindex="3" autofocus required maxlength="36"> + {{.i18n.Tr "repo.cloudbrain_jobname_err"}}
@@ -134,6 +137,9 @@ function showNoteBook(){ var isNoteBook = {{.IsNoteBook}} if (isNoteBook) { var jsonStr = "{{.FileContent}}" + nb.markdown.setOptions({ + baseUrl: {{.FileParentURL}} + }); var notebook = nb.parse(JSON.parse(jsonStr)); var rendered = notebook.render(); $("#notebook").append(rendered); From 1865c868e7ef8e7726c6663d0649fccb2d706c58 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 26 Jul 2022 17:51:27 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- public/self/js/notebook/notebook.js | 365 ---------------------------- 1 file changed, 365 deletions(-) delete mode 100644 public/self/js/notebook/notebook.js diff --git a/public/self/js/notebook/notebook.js b/public/self/js/notebook/notebook.js deleted file mode 100644 index 20c8c9b66..000000000 --- a/public/self/js/notebook/notebook.js +++ /dev/null @@ -1,365 +0,0 @@ -// http://github.com/jsvine/notebookjs -// notebook.js may be freely distributed under the MIT license. -(function () { - var VERSION = "0.6.7"; - var root = this; - var isBrowser = root.window !== undefined; - var doc; - - // Get browser or JSDOM document - if (isBrowser) { - doc = root.document; - } else { - var jsdom = require("jsdom"); - var dom = new jsdom.JSDOM(); - doc = dom.window.document; - } - - // Helper functions - var ident = function (x) { return x; }; - - var makeElement = function (tag, classNames) { - var el = doc.createElement(tag); - el.className = (classNames || []).map(function (cn) { - return nb.prefix + cn; - }).join(" "); - return el; - }; - - var escapeHTML = function (raw) { - var replaced = raw - .replace(//g, ">"); - return replaced; - }; - - var joinText = function (text) { - if (text.join) { - return text.map(joinText).join(""); - } else { - return text; - } - }; - - // Get supporting libraries - var getMarkdown = function () { - return root.marked || (typeof require === "function" && require("marked")); - }; - - var getAnsi = function () { - var lib = root.ansi_up || (typeof require === "function" && require("ansi_up")); - return lib && lib.ansi_to_html; - }; - - var getSanitizer = function () { - var lib = root.DOMPurify || (typeof require === "function" && require("dompurify")); - if (isBrowser) { - return lib && lib.sanitize; - } else { - return lib(dom.window).sanitize; - } - }; - - // Set up `nb` namespace - var nb = { - prefix: "nb-", - markdown: getMarkdown() || ident, - ansi: getAnsi() || ident, - sanitizer: getSanitizer() || ident, - highlighter: ident, - VERSION: VERSION - }; - - // Inputs - nb.Input = function (raw, cell) { - this.raw = raw; - this.cell = cell; - }; - - nb.Input.prototype.render = function () { - if (!this.raw.length) { return makeElement("div"); } - var holder = makeElement("div", [ "input" ]); - var cell = this.cell; - if (typeof cell.number === "number") { - holder.setAttribute("data-prompt-number", this.cell.number); - } - var pre_el = makeElement("pre"); - var code_el = makeElement("code"); - var notebook = cell.worksheet.notebook; - var m = notebook.metadata; - var lang = this.cell.raw.language || m.language || (m.kernelspec && m.kernelspec.language) || (m.language_info && m.language_info.name); - code_el.setAttribute("data-language", lang); - code_el.className = "lang-" + lang; - code_el.innerHTML = nb.highlighter(escapeHTML(joinText(this.raw)), pre_el, code_el, lang); - pre_el.appendChild(code_el); - holder.appendChild(pre_el); - this.el = holder; - return holder; - }; - - // Outputs and output-renderers - var imageCreator = function (format) { - return function (data) { - var el = makeElement("img", [ "image-output" ]); - el.src = "data:image/" + format + ";base64," + joinText(data).replace(/\n/g, ""); - return el; - }; - }; - - nb.display = {}; - nb.display.text = function (text) { - var el = makeElement("pre", [ "text-output" ]); - el.innerHTML = nb.highlighter(nb.ansi(joinText(text)), el); - return el; - }; - nb.display["text/plain"] = nb.display.text; - - nb.display.html = function (html) { - var el = makeElement("div", [ "html-output" ]); - el.innerHTML = nb.sanitizer(joinText(html)); - return el; - }; - nb.display["text/html"] = nb.display.html; - - nb.display.marked = function(md) { - return nb.display.html(nb.markdown(joinText(md))); - }; - nb.display["text/markdown"] = nb.display.marked; - - nb.display.svg = function (svg) { - var el = makeElement("div", [ "svg-output" ]); - el.innerHTML = joinText(svg); - return el; - }; - nb.display["text/svg+xml"] = nb.display.svg; - nb.display["image/svg+xml"] = nb.display.svg; - - nb.display.latex = function (latex) { - var el = makeElement("div", [ "latex-output" ]); - el.innerHTML = joinText(latex); - return el; - }; - nb.display["text/latex"] = nb.display.latex; - - nb.display.javascript = function (js) { - var el = makeElement("script"); - el.innerHTML = joinText(js); - return el; - }; - nb.display["application/javascript"] = nb.display.javascript; - - nb.display.png = imageCreator("png"); - nb.display["image/png"] = nb.display.png; - nb.display.jpeg = imageCreator("jpeg"); - nb.display["image/jpeg"] = nb.display.jpeg; - - nb.display_priority = [ - "png", "image/png", "jpeg", "image/jpeg", - "svg", "image/svg+xml", "text/svg+xml", "html", "text/html", - "text/markdown", "latex", "text/latex", - "javascript", "application/javascript", - "text", "text/plain" - ]; - - var render_display_data = function () { - var o = this; - var formats = nb.display_priority.filter(function (d) { - return o.raw.data ? o.raw.data[d] : o.raw[d]; - }); - var format = formats[0]; - if (format) { - if (nb.display[format]) { - return nb.display[format](o.raw[format] || o.raw.data[format]); - } - } - return makeElement("div", [ "empty-output" ]); - }; - - var render_error = function () { - var el = makeElement("pre", [ "pyerr" ]); - var raw = this.raw.traceback.join("\n"); - el.innerHTML = nb.highlighter(nb.ansi(escapeHTML(raw)), el); - return el; - }; - - nb.Output = function (raw, cell) { - this.raw = raw; - this.cell = cell; - this.type = raw.output_type; - }; - - nb.Output.prototype.renderers = { - "display_data": render_display_data, - "execute_result": render_display_data, - "pyout": render_display_data, - "pyerr": render_error, - "error": render_error, - "stream": function () { - var el = makeElement("pre", [ (this.raw.stream || this.raw.name) ]); - var raw = joinText(this.raw.text); - el.innerHTML = nb.highlighter(nb.ansi(escapeHTML(raw)), el); - return el; - } - }; - - nb.Output.prototype.render = function () { - var outer = makeElement("div", [ "output" ]); - if (typeof this.cell.number === "number") { - outer.setAttribute("data-prompt-number", this.cell.number); - } - var inner = this.renderers[this.type].call(this); - outer.appendChild(inner); - this.el = outer; - return outer; - }; - - // Post-processing - nb.coalesceStreams = function (outputs) { - if (!outputs.length) { return outputs; } - var last = outputs[0]; - var new_outputs = [ last ]; - outputs.slice(1).forEach(function (o) { - if (o.raw.output_type === "stream" && - last.raw.output_type === "stream" && - o.raw.stream === last.raw.stream && - o.raw.name === last.raw.name) { - last.raw.text = last.raw.text.concat(o.raw.text); - } else { - new_outputs.push(o); - last = o; - } - }); - return new_outputs; - }; - - // Cells - nb.Cell = function (raw, worksheet) { - var cell = this; - cell.raw = raw; - cell.worksheet = worksheet; - cell.type = raw.cell_type; - if (cell.type === "code") { - cell.number = raw.prompt_number > -1 ? raw.prompt_number : raw.execution_count; - var source = raw.input || [ raw.source ]; - cell.input = new nb.Input(source, cell); - var raw_outputs = (cell.raw.outputs || []).map(function (o) { - return new nb.Output(o, cell); - }); - cell.outputs = nb.coalesceStreams(raw_outputs); - } - }; - - var math_delimiters = [ - {left: "$$", right: "$$", display: true}, - {left: "\\[", right: "\\]", display: true}, - {left: "\\(", right: "\\)", display: false}, - {left: "$", right: "$", display: false} - ]; - - nb.Cell.prototype.renderers = { - markdown: function () { - var el = makeElement("div", [ "cell", "markdown-cell" ]); - - var joined = joinText(this.raw.source); - - // Pre-render math via KaTeX's auto-render extension, if available - if (root.renderMathInElement != null) { - el.innerHTML = nb.sanitizer(joined); - root.renderMathInElement(el, { delimiters: math_delimiters }); - el.innerHTML = nb.sanitizer(nb.markdown( - el.innerHTML - .replace(/>/g, ">") // Necessary to enable blockquote syntax - )); - } else { - el.innerHTML = nb.sanitizer(nb.markdown(joined)); - } - - return el; - }, - heading: function () { - var el = makeElement("h" + this.raw.level, [ "cell", "heading-cell" ]); - el.innerHTML = nb.sanitizer(joinText(this.raw.source)); - return el; - }, - raw: function () { - var el = makeElement("div", [ "cell", "raw-cell" ]); - el.innerHTML = escapeHTML(joinText(this.raw.source)); - return el; - }, - code: function () { - var cell_el = makeElement("div", [ "cell", "code-cell" ]); - cell_el.appendChild(this.input.render()); - var output_els = this.outputs.forEach(function (o) { - cell_el.appendChild(o.render()); - }); - return cell_el; - } - }; - - nb.Cell.prototype.render = function () { - var el = this.renderers[this.type].call(this); - this.el = el; - return el; - }; - - // Worksheets - nb.Worksheet = function (raw, notebook) { - var worksheet = this; - this.raw = raw; - this.notebook = notebook; - this.cells = raw.cells.map(function (c) { - return new nb.Cell(c, worksheet); - }); - this.render = function () { - var worksheet_el = makeElement("div", [ "worksheet" ]); - worksheet.cells.forEach(function (c) { - worksheet_el.appendChild(c.render()); - }); - this.el = worksheet_el; - return worksheet_el; - }; - }; - - // Notebooks - nb.Notebook = function (raw, config) { - var notebook = this; - this.raw = raw; - this.config = config; - var meta = this.metadata = raw.metadata || {}; - this.title = meta.title || meta.name; - var _worksheets = raw.worksheets || [ { cells: raw.cells } ]; - this.worksheets = _worksheets.map(function (ws) { - return new nb.Worksheet(ws, notebook); - }); - this.sheet = this.worksheets[0]; - }; - - nb.Notebook.prototype.render = function () { - var notebook_el = makeElement("div", [ "notebook" ]); - this.worksheets.forEach(function (w) { - notebook_el.appendChild(w.render()); - }); - this.el = notebook_el; - return notebook_el; - }; - - nb.parse = function (nbjson, config) { - return new nb.Notebook(nbjson, config); - }; - - // Exports - if (typeof define === 'function' && define.amd) { - define(function() { - return nb; - }); - } - if (typeof exports !== 'undefined') { - if (typeof module !== 'undefined' && module.exports) { - exports = module.exports = nb; - } - exports.nb = nb; - } else { - root.nb = nb; - } - -}).call(this);