wangwei
/
aiforge
Not watched
1
0
Fork 0
Code
Releases 128 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

update

tags/v1.22.6.1^2
liuzx 3 years ago
parent
14753d2a34
commit
cc8136a372
4 changed files with 367 additions and 307 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +2
    -0
      models/cloudbrain.go
  2. +59
    -75
      models/cloudbrain_static.go
  3. +1
    -1
      modules/setting/setting.go
  4. +305
    -231
      routers/api/v1/repo/cloudbrain_dashboard.go

+ 2
- 0
models/cloudbrain.go View File

@@ -327,6 +327,8 @@ type CloudbrainsOptions struct {
JobTypeNot bool
NeedRepoInfo bool
RepoIDList []int64
BeginTime time.Time
EndTime time.Time
}

type TaskPod struct {


+ 59
- 75
models/cloudbrain_static.go View File

@@ -5,48 +5,69 @@ import (
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
)

func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and status ='" + string(JobWaiting) + "'"

return x.SQL(countSql).Count()
type AllTask struct {
List []TaskDetail `json:"list"`
}
type TaskDetail struct {
ID int64 `json:"ID"`
JobName string `json:"JobName"`
DisplayJobName string `json:"DisplayJobName"`
Status string `json:"Status"`
JobType string `json:"JobType"`
CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"`
WaitTime timeutil.TimeStamp `json:"WaitTime"`
RunTime timeutil.TimeStamp `json:"RunTime"`
StartTime timeutil.TimeStamp `json:"StartTime"`
EndTime timeutil.TimeStamp `json:"EndTime"`
ComputeResource string `json:"ComputeResource"`
Type int `json:"Type"`
UserID int64 `json:"UserID"`
RepoID int64 `json:"RepoID"`
}
func GetJobRunningPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and status ='" + string(JobRunning) + "'"

return x.SQL(countSql).Count()
}
func GetJobStoppedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and status ='" + string(JobStopped) + "'"
// func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
// countSql := "SELECT count(*) FROM " +
// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
// " and status ='" + string(JobWaiting) + "'"

return x.SQL(countSql).Count()
}
func GetJobSucceededPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and status ='" + string(JobSucceeded) + "'"
// return x.SQL(countSql).Count()
// }
// func GetJobRunningPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
// countSql := "SELECT count(*) FROM " +
// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
// " and status ='" + string(JobRunning) + "'"

return x.SQL(countSql).Count()
}
func GetJobFailedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and status ='" + string(JobFailed) + "'"
// return x.SQL(countSql).Count()
// }
// func GetJobStoppedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
// countSql := "SELECT count(*) FROM " +
// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
// " and status ='" + string(JobStopped) + "'"

return x.SQL(countSql).Count()
}
// return x.SQL(countSql).Count()
// }
// func GetJobSucceededPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
// countSql := "SELECT count(*) FROM " +
// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
// " and status ='" + string(JobSucceeded) + "'"

// return x.SQL(countSql).Count()
// }
// func GetJobFailedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
// countSql := "SELECT count(*) FROM " +
// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
// " and status ='" + string(JobFailed) + "'"

// return x.SQL(countSql).Count()
// }

func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
@@ -175,36 +196,7 @@ func GetCloudBrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int
return total, nil
}

func GetCloudBrainOneCount() (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where type='" + strconv.Itoa(TypeCloudBrainOne) + "'"

return x.SQL(countSql).Count()
}
func GetCloudBrainOneDuration() (int64, error) {
total, err := x.Where("type = ? ", TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}

return total, nil
}
func GetCloudBrainTwoCount() (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"

return x.SQL(countSql).Count()
}
func GetCloudBrainTwoDuration() (int64, error) {
total, err := x.Where("type = ? ", TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}

return total, nil
}

func GetCreatorPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)
@@ -215,7 +207,7 @@ func GetCreatorCount() (int64, error) {
return x.SQL(countSql).Count()
}

func GetAllStatusCloudBrain() map[string]int {
func GetAllCloudBrain() ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
cloudbrains := make([]*CloudbrainInfo, 0)
@@ -223,15 +215,7 @@ func GetAllStatusCloudBrain() map[string]int {
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
cloudBrainStatusResult := make(map[string]int)
for _, cloudbrain := range cloudbrains {
if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok {
cloudBrainStatusResult[cloudbrain.Status] = 1
} else {
cloudBrainStatusResult[cloudbrain.Status] += 1
}
}
return cloudBrainStatusResult
return cloudbrains, nil
}

func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) {


+ 1
- 1
modules/setting/setting.go View File

@@ -1293,7 +1293,7 @@ func NewContext() {
MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400)
TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("")
TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("")
BrainRecordBeginTime = sec.Key("brain_record_beigin_time").MustString("2021-01-01")
BrainRecordBeginTime = sec.Key("brain_record_begin_time").MustString("2021-01-01")

sec = Cfg.Section("benchmark")
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)


+ 305
- 231
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -1,7 +1,6 @@
package repo

import (
"fmt"
"net/http"
"net/url"
"strings"
@@ -16,126 +15,99 @@ import (
"code.gitea.io/gitea/modules/setting"
)

type CloudbrainsPeriodData struct {
DebugOnePeriodCount int64 `json:"debugOnePeriodCount"`
BenchmarkOnePeriodCount int64 `json:"benchmarkOnePeriodCount"`
TrainOnePeriodCount int64 `json:"trainOnePeriodCount"`
DebugTwoPeriodCount int64 `json:"debugTwoPeriodCount"`
TrainTwoPeriodCount int64 `json:"trainTwoPeriodCount"`
InferenceTwoPeriodCount int64 `json:"inferenceTwoPeriodCount"`
JobWaitingPeriodCount int64 `json:"jobWaitingPeriodCount"`
JobRunningPeriodCount int64 `json:"jobRunningPeriodCount"`
JobSucceededPeriodCount int64 `json:"jobRunningPeriodCount"`
CloudBrainOnePeriodCount int64 `json:"cloudBrainOnePeriodCount"`
CloudBrainTwoPeriodCount int64 `json:"cloudBrainTwoPeriodCount"`
}
type CloudbrainsOverviewData struct {
JobWaitingPeriodCount int64 `json:"jobWaitingPeriodCount"`
JobRunningPeriodCount int64 `json:"jobRunningPeriodCount"`
JobStoppedPeriodCount int64 `json:"jobStoppedPeriodCount"`
JobSucceededPeriodCount int64 `json:"jobSucceededPeriodCount"`
JobFailedPeriodCount int64 `json:"jobFailedPeriodCount"`
CloudBrainOneCount int64 `json:"cloudBrainOneCount"`
CloudBrainTwoCount int64 `json:"cloudBrainTwoCount"`
CloudBrainOneDuration int64 `json:"cloudBrainOneDuration"`
CloudBrainTwoDuration int64 `json:"cloudBrainTwoDuration"`
CreatorPeriodCount int64 `json:"creatorPeriodCount"`
CreatorCount int64 `json:"creatorCount"`
}

type TimeCloudbrainsNum struct {
TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"`
}
type DateCloudbrainNum struct {
Date string `json:"date"`
DebugOneCount int64 `json:"debugOneCount"`
BenchmarkOneCount int64 `json:"benchmarkOneCount"`
TrainOneCount int64 `json:"trainOneCount"`
DebugTwoCount int64 `json:"debugTwoCount"`
TrainTwoCount int64 `json:"trainTwoCount"`
InferenceTwoCount int64 `json:"inferenceTwoCount"`
CloudbrainOneCount int64 `json:"cloudbrainOneCount"`
CloudbrainTwoCount int64 `json:"cloudbrainTwoCount"`
CloudbrainCount int64 `json:"cloudbrainCount"`
Date string `json:"date"`
CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"`
CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"`
IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"`
CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"`
CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"`
}

func GetAllCloudbrainsOverview(ctx *context.Context) {
recordBeginTime, err := getBrainRecordBeginTime()
if err != nil {
log.Error("Can not get record begin time", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
return
}
now := time.Now()
//today overview
beginTime := now.AddDate(0, 0, 0)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime := now
jobWaitingPeriodCount, err := models.GetJobWaitingPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query jobWaitingCount.", err)
return
}
jobRunningPeriodCount, err := models.GetJobRunningPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query jobRunningCount.", err)
return
}
jobStoppedPeriodCount, err := models.GetJobStoppedPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query jobStoppedCount.", err)
return
}
jobSucceededPeriodCount, err := models.GetJobSucceededPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query jobSucceededCount.", err)
return
}
jobFailedPeriodCount, err := models.GetJobFailedPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query jobFailedCount.", err)
return
}
cloudBrainOneCount, err := models.GetCloudBrainOneCount()
todayCreatorCount, err := models.GetTodayCreatorCount(beginTime, endTime)
if err != nil {
log.Error("Can not query cloudBrainOneCount.", err)
log.Error("Can not query todayCreatorCount.", err)
return
}
cloudBrainOneDuration, err := models.GetCloudBrainOneDuration()
if err != nil {
log.Error("Can not query cloudBrainOneDuration.", err)
return
}
cloudBrainTwoCount, err := models.GetCloudBrainTwoCount()
if err != nil {
log.Error("Can not query cloudBrainTwoCount.", err)
return
}
cloudBrainTwoDuration, err := models.GetCloudBrainTwoDuration()
creatorCount, err := models.GetCreatorCount()
if err != nil {
log.Error("Can not query cloudBrainTwoDuration.", err)
log.Error("Can not query creatorCount.", err)
return
}
creatorPeriodCount, err := models.GetCreatorPeriodCount(beginTime, endTime)

cloudbrains, err := models.GetAllCloudBrain()
if err != nil {
log.Error("Can not query creatorPeriodCount.", err)
log.Error("Getcloudbrains failed:%v", err)
return
}
creatorCount, err := models.GetCreatorCount()
if err != nil {
log.Error("Can not query creatorCount.", err)
return

todayStatusResult := make(map[string]int)
cloudBrainNum := make(map[int]int)
cloudBrainOneDuration := int64(0)
cloudBrainTwoDuration := int64(0)
for _, cloudbrain := range cloudbrains {
if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() {
if _, ok := todayStatusResult[cloudbrain.Status]; !ok {
todayStatusResult[cloudbrain.Status] = 1
} else {
todayStatusResult[cloudbrain.Status] += 1
}
}

if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok {
cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1
} else {
cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1
}

if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration
}
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration
}

}

CloudbrainsOverviewData := CloudbrainsOverviewData{
JobWaitingPeriodCount: jobWaitingPeriodCount,
JobRunningPeriodCount: jobRunningPeriodCount,
JobStoppedPeriodCount: jobStoppedPeriodCount,
JobSucceededPeriodCount: jobSucceededPeriodCount,
JobFailedPeriodCount: jobFailedPeriodCount,
CloudBrainOneCount: cloudBrainOneCount,
CloudBrainTwoCount: cloudBrainTwoCount,
CloudBrainOneDuration: cloudBrainOneDuration,
CloudBrainTwoDuration: cloudBrainTwoDuration,
CreatorPeriodCount: creatorPeriodCount,
CreatorCount: creatorCount,
statusNameList := []string{"COMPLETED", "FAILED", "INIT", "RUNNING", "START_FAILED", "STOPPED", "SUCCEEDED", "WAITING", "KILLED"}
for _, v := range statusNameList {
if _, ok := todayStatusResult[v]; !ok {
todayStatusResult[v] = 0
}
}

ctx.JSON(http.StatusOK, CloudbrainsOverviewData)
todayRunningCount := todayStatusResult["RUNNING"]
todayCompletedCount := todayStatusResult["COMPLETED"] + todayStatusResult["FAILED"] + todayStatusResult["START_FAILED"] + todayStatusResult["STOPPED"] + todayStatusResult["SUCCEEDED"] + todayStatusResult["KILLED"]
todayWaitingCount := todayStatusResult["INIT"] + todayStatusResult["WAITING"]

ctx.JSON(http.StatusOK, map[string]interface{}{
// "todayStatusResult": todayStatusResult,
"recordBeginTime": recordBeginTime,
"cloudBrainNum": cloudBrainNum,
"cloudBrainOneDuration": cloudBrainOneDuration,
"cloudBrainTwoDuration": cloudBrainTwoDuration,
"intelligentNetDuration": 0,
"todayCreatorCount": todayCreatorCount,
"creatorCount": creatorCount,
"todayRunningCount": todayRunningCount,
"todayCompletedCount": todayCompletedCount,
"todayWaitingCount": todayWaitingCount,
})
}

func GetAllCloudbrainsTrend(ctx *context.Context) {
@@ -308,71 +280,128 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
return
}
debugOnePeriodCount, err := models.GetDebugOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query debugOneCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error"))
return
}
benchmarkOnePeriodCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query benchmarkCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error"))
return
}
trainOnePeriodCount, err := models.GetTrainOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query trainOneCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error"))
return
}
debugTwoPeriodCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query debugTwoCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error"))
// debugOnePeriodCount, err := models.GetDebugOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query debugOneCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error"))
// return
// }
// benchmarkOnePeriodCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query benchmarkCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error"))
// return
// }
// trainOnePeriodCount, err := models.GetTrainOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query trainOneCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error"))
// return
// }
// debugTwoPeriodCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query debugTwoCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error"))
// return
// }
// trainTwoPeriodCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query DebugOneTotal count.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error"))
// return
// }
// inferenceTwoPeriodCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query inferenceTwoCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error"))
// return
// }
// cloudBrainOnePeriodCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query cloudBrainOnePeriodCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainOnePeriodCount_get_error"))
// return
// }
// cloudBrainTwoPeriodCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query cloudBrainTwoPeriodCount.", err)
// ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainTwoPeriodCount_get_error"))
// return
// }

cloudbrains, err := models.GetAllCloudBrain()
if err != nil {
log.Error("Getcloudbrains failed:%v", err)
return
}
trainTwoPeriodCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query DebugOneTotal count.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error"))
return
}
inferenceTwoPeriodCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query inferenceTwoCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error"))
return
}
cloudBrainOnePeriodCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query cloudBrainOnePeriodCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainOnePeriodCount_get_error"))
return
}
cloudBrainTwoPeriodCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query cloudBrainTwoPeriodCount.", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainTwoPeriodCount_get_error"))
return
cloudOneJobTypeRes := make(map[string]int)
cloudTwoJobTypeRes := make(map[string]int)
intelligentNetJobTypeRes := make(map[string]int)
cloudBrainPeriodNum := make(map[int]int)
for _, cloudbrain := range cloudbrains {
if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() {
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
if _, ok := cloudOneJobTypeRes[cloudbrain.JobType]; !ok {
cloudOneJobTypeRes[cloudbrain.JobType] = 1
} else {
cloudOneJobTypeRes[cloudbrain.JobType] += 1
}
}
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok {
cloudTwoJobTypeRes[cloudbrain.JobType] = 1
} else {
cloudTwoJobTypeRes[cloudbrain.JobType] += 1
}
}

if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok {
cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1
} else {
cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1
}

}
}

cloudbrainsPeriodData := CloudbrainsPeriodData{
DebugOnePeriodCount: debugOnePeriodCount,
BenchmarkOnePeriodCount: benchmarkOnePeriodCount,
TrainOnePeriodCount: trainOnePeriodCount,
DebugTwoPeriodCount: debugTwoPeriodCount,
TrainTwoPeriodCount: trainTwoPeriodCount,
InferenceTwoPeriodCount: inferenceTwoPeriodCount,
CloudBrainOnePeriodCount: cloudBrainOnePeriodCount,
CloudBrainTwoPeriodCount: cloudBrainTwoPeriodCount,
jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"}
for _, v := range jobTypeList {
if _, ok := cloudOneJobTypeRes[v]; !ok {
cloudOneJobTypeRes[v] = 0
}
if _, ok := cloudTwoJobTypeRes[v]; !ok {
cloudTwoJobTypeRes[v] = 0
}
if _, ok := intelligentNetJobTypeRes[v]; !ok {
intelligentNetJobTypeRes[v] = 0
}
}

ctx.JSON(http.StatusOK, cloudbrainsPeriodData)
cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes["BENCHMARK"] + cloudOneJobTypeRes["SNN4IMAGENET"] + cloudOneJobTypeRes["BRAINSCORE"]
cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes["BENCHMARK"] + cloudTwoJobTypeRes["SNN4IMAGENET"] + cloudTwoJobTypeRes["BRAINSCORE"]
intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes["BENCHMARK"] + intelligentNetJobTypeRes["SNN4IMAGENET"] + intelligentNetJobTypeRes["BRAINSCORE"]
ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudOneJobTypeRes": cloudOneJobTypeRes,
"cloudTwoJobTypeRes": cloudTwoJobTypeRes,
"intelligentNetJobTypeRes": intelligentNetJobTypeRes,
"cloudBrainPeriodNum": cloudBrainPeriodNum,
})
}

func GetCloudbrainsStatusAnalysis(ctx *context.Context) {
cloudBrainStatusResult := models.GetAllStatusCloudBrain()
cloudbrains, err := models.GetAllCloudBrain()
if err != nil {
log.Error("Getcloudbrains failed:%v", err)
return
}
cloudBrainStatusResult := make(map[string]int)
for _, cloudbrain := range cloudbrains {
if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok {
cloudBrainStatusResult[cloudbrain.Status] = 1
} else {
cloudBrainStatusResult[cloudbrain.Status] += 1
}
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudBrainStatusResult": cloudBrainStatusResult,
})
@@ -434,11 +463,13 @@ func GetCloudbrainsDetailData(ctx *context.Context) {
ctx.ServerError("Get job failed:", err)
return
}
// var taskList := []*models.TaskDetail

for i, task := range ciTasks {
ciTasks[i].CanDebug = true
ciTasks[i].CanDel = true
ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource

}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum))
@@ -598,70 +629,127 @@ func GetCloudbrainsRunHoursData(ctx *context.Context) {

}

func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) {
debugOneCount, err := models.GetDebugOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query debugOneCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
}
benchmarkOneCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query benchmarkCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
}
trainOneCount, err := models.GetTrainOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query trainOneCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
}
debugTwoCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query debugTwoCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
}
trainTwoCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query DebugOneTotal count.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
}
inferenceTwoCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query inferenceTwoCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
func getCloudbrainCount(beginTime time.Time, endTime time.Time) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int, error) {
// debugOneCount, err := models.GetDebugOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query debugOneCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// benchmarkOneCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query benchmarkCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// trainOneCount, err := models.GetTrainOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query trainOneCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// debugTwoCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query debugTwoCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// trainTwoCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query DebugOneTotal count.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// inferenceTwoCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query inferenceTwoCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// cloudbrainOneCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query cloudbrainOneCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// cloudbrainTwoCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime)
// if err != nil {
// log.Error("Can not query cloudbrainOneCount.", err)
// return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
// }
// cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount

cloudbrains, err := models.GetAllCloudBrain()
if err != nil {
log.Error("Getcloudbrains failed:%v", err)
return nil, nil, nil, nil, nil, err
}
cloudOneJobTypeRes := make(map[string]int)
cloudTwoJobTypeRes := make(map[string]int)
intelligentNetJobTypeRes := make(map[string]int)
cloudBrainPeriodNum := make(map[int]int)
cloudBrainComputeResource := make(map[string]int)
for _, cloudbrain := range cloudbrains {
if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() {
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
if _, ok := cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType]; !ok {
cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1
} else {
cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1
}
}
if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok {
cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1
} else {
cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1
}
}

if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok {
cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1
} else {
cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1
}

if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok {
cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1
} else {
cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1
}

}
}
cloudbrainOneCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query cloudbrainOneCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err

jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"}
cloudBrainTypeList := []int{0, 1, 2}
for _, v := range jobTypeList {
if _, ok := cloudOneJobTypeRes[v]; !ok {
cloudOneJobTypeRes[v] = 0
}
if _, ok := cloudTwoJobTypeRes[v]; !ok {
cloudTwoJobTypeRes[v] = 0
}
if _, ok := intelligentNetJobTypeRes[v]; !ok {
intelligentNetJobTypeRes[v] = 0
}
}
cloudbrainTwoCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime)
if err != nil {
log.Error("Can not query cloudbrainOneCount.", err)
return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
for _, v := range cloudBrainTypeList {
if _, ok := cloudBrainPeriodNum[v]; !ok {
cloudBrainPeriodNum[v] = 0
}
}
cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount
return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err
return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err
}

func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
dayCloudbrainNum := make([]DateCloudbrainNum, 0)
for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) {
debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp)
if err != nil {
log.Error("Can not query getCloudbrainCount.", err)
return nil, err
}
dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{
Date: beginTime.Format(time.RFC3339),
DebugOneCount: debugOneCount,
BenchmarkOneCount: benchmarkOneCount,
TrainOneCount: trainOneCount,
DebugTwoCount: debugTwoCount,
TrainTwoCount: trainTwoCount,
InferenceTwoCount: inferenceTwoCount,
CloudbrainOneCount: cloudbrainOneCount,
CloudbrainTwoCount: cloudbrainTwoCount,
CloudbrainCount: cloudbrainCount,
Date: beginTime.Format(time.RFC3339),
CloudOneJobTypeRes: cloudOneJobTypeRes,
CloudTwoJobTypeRes: cloudTwoJobTypeRes,
IntelligentNetJobTypeRes: intelligentNetJobTypeRes,
CloudBrainPeriodNum: cloudBrainPeriodNum,
CloudBrainComputeResource: cloudBrainComputeResource,
})
beginTime = endTimeTemp
endTimeTemp = beginTime.Add(time.Hour)
@@ -671,23 +759,18 @@ func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime ti
func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
dayCloudbrainNum := make([]DateCloudbrainNum, 0)
for endTimeTemp.Before(endTime) {
debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount,
cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp)
if err != nil {
log.Error("Can not query getCloudbrainCount.", err)
return nil, err
}
dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{
Date: beginTime.Format("2006-01-02"),
DebugOneCount: debugOneCount,
BenchmarkOneCount: benchmarkOneCount,
TrainOneCount: trainOneCount,
DebugTwoCount: debugTwoCount,
TrainTwoCount: trainTwoCount,
InferenceTwoCount: inferenceTwoCount,
CloudbrainOneCount: cloudbrainOneCount,
CloudbrainTwoCount: cloudbrainTwoCount,
CloudbrainCount: cloudbrainCount,
Date: beginTime.Format("2006-01-02"),
CloudOneJobTypeRes: cloudOneJobTypeRes,
CloudTwoJobTypeRes: cloudTwoJobTypeRes,
IntelligentNetJobTypeRes: intelligentNetJobTypeRes,
CloudBrainPeriodNum: cloudBrainPeriodNum,
CloudBrainComputeResource: cloudBrainComputeResource,
})
beginTime = endTimeTemp
endTimeTemp = beginTime.AddDate(0, 0, 1)
@@ -697,23 +780,18 @@ func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime tim
func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
yearCloudbrainNum := make([]DateCloudbrainNum, 0)
for endTimeTemp.Before(endTime) {
debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount,
cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp)
if err != nil {
log.Error("Can not query getCloudbrainCount.", err)
return nil, err
}
yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{
Date: beginTime.Format("2006-01"),
DebugOneCount: debugOneCount,
BenchmarkOneCount: benchmarkOneCount,
TrainOneCount: trainOneCount,
DebugTwoCount: debugTwoCount,
TrainTwoCount: trainTwoCount,
InferenceTwoCount: inferenceTwoCount,
CloudbrainOneCount: cloudbrainOneCount,
CloudbrainTwoCount: cloudbrainTwoCount,
CloudbrainCount: cloudbrainCount,
Date: beginTime.Format("2006-01"),
CloudOneJobTypeRes: cloudOneJobTypeRes,
CloudTwoJobTypeRes: cloudTwoJobTypeRes,
IntelligentNetJobTypeRes: intelligentNetJobTypeRes,
CloudBrainPeriodNum: cloudBrainPeriodNum,
CloudBrainComputeResource: cloudBrainComputeResource,
})
beginTime = endTimeTemp
endTimeTemp = beginTime.AddDate(0, 1, 0)
@@ -747,11 +825,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) {
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail"))
return
}

totalPage := getTotalPage(total, pageSize)
fmt.Printf("total:%v", total)
fmt.Printf("totalPage:%v", totalPage)

f := excelize.NewFile()

index := f.NewSheet(cloudBrain)


Write Preview
Loading…
Cancel
Save