package repo import ( "net/http" "net/url" "strings" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "github.com/360EntSecGroup-Skylar/excelize/v2" "code.gitea.io/gitea/modules/setting" ) type TimeCloudbrainsNum struct { TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"` TotalCount int `json:"totalCount"` } type DateCloudbrainNum struct { Date string `json:"date"` CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"` CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"` IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"` CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` } func GetAllCloudbrainsOverview(ctx *context.Context) { recordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } now := time.Now() beginTime := now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime := now todayCreatorCount, err := models.GetTodayCreatorCount(beginTime, endTime) if err != nil { log.Error("Can not query todayCreatorCount.", err) return } creatorCount, err := models.GetCreatorCount() if err != nil { log.Error("Can not query creatorCount.", err) return } cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return } todayStatusResult := make(map[string]int) cloudBrainNum := make(map[int]int) cloudBrainOneDuration := int64(0) cloudBrainTwoDuration := int64(0) for _, cloudbrain := range cloudbrains { if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { if _, ok := todayStatusResult[cloudbrain.Status]; !ok { todayStatusResult[cloudbrain.Status] = 1 } else { todayStatusResult[cloudbrain.Status] += 1 } } if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 } else { cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 } if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration } if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration } } statusNameList := []string{"COMPLETED", "FAILED", "INIT", "RUNNING", "START_FAILED", "STOPPED", "SUCCEEDED", "WAITING", "KILLED"} for _, v := range statusNameList { if _, ok := todayStatusResult[v]; !ok { todayStatusResult[v] = 0 } } todayRunningCount := todayStatusResult["RUNNING"] todayCompletedCount := todayStatusResult["COMPLETED"] + todayStatusResult["FAILED"] + todayStatusResult["START_FAILED"] + todayStatusResult["STOPPED"] + todayStatusResult["SUCCEEDED"] + todayStatusResult["KILLED"] todayWaitingCount := todayStatusResult["INIT"] + todayStatusResult["WAITING"] ctx.JSON(http.StatusOK, map[string]interface{}{ "recordBeginTime": recordBeginTime, "updateTime": now, "cloudBrainNum": cloudBrainNum, "cloudBrainOneDuration": cloudBrainOneDuration, "cloudBrainTwoDuration": cloudBrainTwoDuration, "intelligentNetDuration": 0, "todayCreatorCount": todayCreatorCount, "creatorCount": creatorCount, "todayRunningCount": todayRunningCount, "todayCompletedCount": todayCompletedCount, "todayWaitingCount": todayWaitingCount, }) } func GetAllCloudbrainsTrend(ctx *context.Context) { brainRecordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get brain record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err")) return } queryType := ctx.QueryTrim("type") now := time.Now() beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") var beginTime time.Time var endTime time.Time var endTimeTemp time.Time dateCloudbrainNum := make([]DateCloudbrainNum, 0) if queryType != "" { if queryType == "all" { beginTime = brainRecordBeginTime endTime = now endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getYearCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "yesterday" { beginTime = now.AddDate(0, 0, -1) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.Add(time.Hour) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getHourCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "last_7day" { beginTime = now.AddDate(0, 0, -7) //begin from monday beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "last_30day" { beginTime = now.AddDate(0, 0, -30) //begin from monday beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "monthly" { endTime = now beginTime = now.AddDate(0, -1, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else if queryType == "last_month" { lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) endTimeTemp = beginTime.AddDate(0, 0, 1) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } } else { if beginTimeStr == "" || endTimeStr == "" { //如果查询类型和开始时间结束时间都未设置,按queryType=all处理 beginTime = brainRecordBeginTime endTime = now endTimeTemp = beginTime.AddDate(0, 1, 0) dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) return } } else { beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } days := (endTime.Unix() - beginTime.Unix()) / 3600 / 24 if 1 < days { endTimeTemp = beginTime.AddDate(0, 0, 1) endTime = endTime.AddDate(0, 0, 2) dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) return } } else if 0 < days || days <= 1 { endTimeTemp = beginTime.Add(time.Hour) dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getHourCloudbrainNum.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getHourCloudbrainNum_get_error")) return } } else { return } } } cloudbrainsPeriodData := TimeCloudbrainsNum{ TimeCloudbrainNum: dateCloudbrainNum, } ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } func GetAllCloudbrainsTrendDetail(ctx *context.Context) { now := time.Now() var beginTime time.Time var endTime time.Time var endTimeTemp time.Time beginTime = now.AddDate(0, 0, -30) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now endTimeTemp = endTime.AddDate(0, 0, -1) dayCloudbrainInfo, err := getDayCloudbrainInfo(beginTime, endTimeTemp, endTime) if err != nil { log.Error("Can not query getDayCloudbrainInfo.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) return } page := ctx.QueryInt("page") if page <= 0 { page = 1 } pagesize := ctx.QueryInt("pagesize") if pagesize <= 0 { pagesize = 5 } pageDateCloudbrainNum := getPageDateCloudbrainNum(dayCloudbrainInfo, page, pagesize) cloudbrainsPeriodData := TimeCloudbrainsNum{ TotalCount: 30, TimeCloudbrainNum: pageDateCloudbrainNum, } ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } func getPageDateCloudbrainNum(dateCloudbrainNums []DateCloudbrainNum, page int, pagesize int) []DateCloudbrainNum { begin := (page - 1) * pagesize end := (page) * pagesize if begin > len(dateCloudbrainNums)-1 { return nil } if end > len(dateCloudbrainNums)-1 { return dateCloudbrainNums[begin:] } else { return dateCloudbrainNums[begin:end] } } func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return } cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) for _, cloudbrain := range cloudbrains { if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { if _, ok := cloudOneJobTypeRes[cloudbrain.JobType]; !ok { cloudOneJobTypeRes[cloudbrain.JobType] = 1 } else { cloudOneJobTypeRes[cloudbrain.JobType] += 1 } } if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { cloudTwoJobTypeRes[cloudbrain.JobType] = 1 } else { cloudTwoJobTypeRes[cloudbrain.JobType] += 1 } } if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 } else { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 } if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok { cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1 } else { cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1 } } } jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 } if _, ok := cloudTwoJobTypeRes[v]; !ok { cloudTwoJobTypeRes[v] = 0 } if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } } cloudBrainTypeList := []int{0, 1, 2} for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { cloudBrainPeriodNum[v] = 0 } } ComputeResourceList := []string{"CPU/GPU", "NPU"} for _, v := range ComputeResourceList { if _, ok := cloudBrainComputeResource[v]; !ok { cloudBrainComputeResource[v] = 0 } } cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes["BENCHMARK"] + cloudOneJobTypeRes["SNN4IMAGENET"] + cloudOneJobTypeRes["BRAINSCORE"] cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes["BENCHMARK"] + cloudTwoJobTypeRes["SNN4IMAGENET"] + cloudTwoJobTypeRes["BRAINSCORE"] intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes["BENCHMARK"] + intelligentNetJobTypeRes["SNN4IMAGENET"] + intelligentNetJobTypeRes["BRAINSCORE"] ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudOneJobTypeRes": cloudOneJobTypeRes, "cloudTwoJobTypeRes": cloudTwoJobTypeRes, "intelligentNetJobTypeRes": intelligentNetJobTypeRes, "cloudBrainPeriodNum": cloudBrainPeriodNum, "cloudBrainComputeResource": cloudBrainComputeResource, }) } func GetCloudbrainsStatusAnalysis(ctx *context.Context) { cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return } cloudBrainStatusResult := make(map[string]int) for _, cloudbrain := range cloudbrains { if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { cloudBrainStatusResult[cloudbrain.Status] = 1 } else { cloudBrainStatusResult[cloudbrain.Status] += 1 } } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainStatusResult": cloudBrainStatusResult, }) } func GetCloudbrainsDetailData(ctx *context.Context) { listType := ctx.Query("listType") jobType := ctx.Query("jobType") jobStatus := ctx.Query("jobStatus") cloudBrainType := ctx.QueryInt("Type") page := ctx.QueryInt("page") pageSize := ctx.QueryInt("pageSize") if page <= 0 { page = 1 } if pageSize <= 0 { pageSize = 10 } var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeDebug) { jobTypes = append(jobTypes, string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } var jobStatuses []string jobStatusNot := false if jobStatus == "other" { jobStatusNot = true jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) } else if jobStatus != "all" && jobStatus != "" { jobStatuses = append(jobStatuses, jobStatus) } keyword := strings.Trim(ctx.Query("q"), " ") ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, }, Keyword: keyword, Type: cloudBrainType, ComputeResource: listType, JobTypeNot: jobTypeNot, JobStatusNot: jobStatusNot, JobStatus: jobStatuses, JobTypes: jobTypes, NeedRepoInfo: true, IsLatestVersion: modelarts.IsLatestVersion, }) if err != nil { ctx.ServerError("Get job failed:", err) return } tasks := []models.TaskDetail{} for i, task := range ciTasks { ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource var taskDetail models.TaskDetail taskDetail.ID = ciTasks[i].Cloudbrain.ID taskDetail.JobName = ciTasks[i].JobName taskDetail.DisplayJobName = ciTasks[i].DisplayJobName taskDetail.Status = ciTasks[i].Status taskDetail.JobType = ciTasks[i].JobType taskDetail.CreatedUnix = ciTasks[i].Cloudbrain.CreatedUnix taskDetail.RunTime = ciTasks[i].EndTime - ciTasks[i].StartTime taskDetail.StartTime = ciTasks[i].StartTime taskDetail.EndTime = ciTasks[i].EndTime taskDetail.ComputeResource = ciTasks[i].ComputeResource taskDetail.Type = ciTasks[i].Cloudbrain.Type taskDetail.UserName = ciTasks[i].User.Name taskDetail.RepoID = ciTasks[i].RepoID if ciTasks[i].Repo != nil { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name } taskDetail.WaitTime = ciTasks[i].StartTime - ciTasks[i].Cloudbrain.CreatedUnix if taskDetail.WaitTime < 0 { taskDetail.WaitTime = 0 } tasks = append(tasks, taskDetail) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) pager.SetDefaultParams(ctx) pager.AddParam(ctx, "listType", "ListType") ctx.JSON(http.StatusOK, map[string]interface{}{ "Title": ctx.Tr("kanban.cloudBrains"), "Tasks": tasks, "Keyword": keyword, "pager": pager, "count": count, }) } func GetCloudbrainsCreateHoursData(ctx *context.Context) { createHourPeriodCount := make(map[string]interface{}) recordBeginTime, err := getBrainRecordBeginTime() if err != nil { log.Error("Can not get brain record begin time", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err")) return } now := time.Now() queryType := ctx.QueryTrim("type") beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") var beginTime time.Time var endTime time.Time if queryType != "" { if queryType == "all" { beginTime = recordBeginTime endTime = now } else if queryType == "today" { beginTime = now.AddDate(0, 0, 0) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "yesterday" { beginTime = now.AddDate(0, 0, -1) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "current_week" { beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday endTime = now } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) } else if queryType == "monthly" { endTime = now beginTime = now.AddDate(0, -1, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) } else if queryType == "last_month" { lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) } else if queryType == "last_7day" { lastMonthTime := now.AddDate(0, 0, -7) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) } else if queryType == "last_30day" { lastMonthTime := now.AddDate(0, 0, -30) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) } } else { if beginTimeStr == "" || endTimeStr == "" { //如果查询类型和开始时间结束时间都未设置,按queryType=all处理 beginTime = recordBeginTime endTime = now } else { beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) return } } } dateBeginTime := beginTime.Format("2006-01-02") dateEndTime := endTime.Format("2006-01-02") createHourPeriodCount, err = models.GetCreateHourPeriodCount(dateBeginTime, dateEndTime) if err != nil { log.Error("Can not query hourPeriodCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("hourPeriodCount_get_error")) return } runHourPeriodCount, err := models.GetRunHourPeriodCount(dateBeginTime, dateEndTime) if err != nil { log.Error("Can not query runHourPeriodCount.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("runHourPeriodCount_get_error")) return } ctx.JSON(http.StatusOK, map[string]interface{}{ "recordBeginTime": recordBeginTime, "updateTime": now, "createHourPeriodCount": createHourPeriodCount, "runHourPeriodCount": runHourPeriodCount, }) } func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { cloudOneJobTypeRes := make(map[string]int) cloudTwoJobTypeRes := make(map[string]int) intelligentNetJobTypeRes := make(map[string]int) cloudBrainPeriodNum := make(map[int]int) cloudBrainComputeResource := make(map[string]int) for _, cloudbrain := range cloudbrains { if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { if _, ok := cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType]; !ok { cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 } else { cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 } } if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 } else { cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 } } if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 } else { cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 } if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok { cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1 } else { cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1 } } } ComputeResourceList := []string{"CPU/GPU", "NPU"} for _, v := range ComputeResourceList { if _, ok := cloudBrainComputeResource[v]; !ok { cloudBrainComputeResource[v] = 0 } } jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} cloudBrainTypeList := []int{0, 1, 2} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 } if _, ok := cloudTwoJobTypeRes[v]; !ok { cloudTwoJobTypeRes[v] = 0 } if _, ok := intelligentNetJobTypeRes[v]; !ok { intelligentNetJobTypeRes[v] = 0 } } for _, v := range cloudBrainTypeList { if _, ok := cloudBrainPeriodNum[v]; !ok { cloudBrainPeriodNum[v] = 0 } } cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource } func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return nil, err } dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format(time.RFC3339), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.Add(time.Hour) } return dayCloudbrainNum, nil } func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return nil, err } dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) { cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01/02"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 0, 1) } return dayCloudbrainNum, nil } func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { yearCloudbrainNum := make([]DateCloudbrainNum, 0) cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return nil, err } for endTimeTemp.Before(endTime) { cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{ Date: beginTime.Format("2006/01"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 1, 0) } return yearCloudbrainNum, nil } func getDayCloudbrainInfo(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { cloudbrains, err := models.GetAllCloudBrain() if err != nil { log.Error("Getcloudbrains failed:%v", err) return nil, err } dayCloudbrainNum := make([]DateCloudbrainNum, 0) for beginTime.Before(endTimeTemp) { cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ Date: endTime.Format("2006/01/02"), CloudOneJobTypeRes: cloudOneJobTypeRes, CloudTwoJobTypeRes: cloudTwoJobTypeRes, IntelligentNetJobTypeRes: intelligentNetJobTypeRes, CloudBrainPeriodNum: cloudBrainPeriodNum, CloudBrainComputeResource: cloudBrainComputeResource, }) if beginTime.Before(endTimeTemp) { endTime = endTimeTemp endTimeTemp = endTimeTemp.AddDate(0, 0, -1) } } return dayCloudbrainNum, nil } func getBrainRecordBeginTime() (time.Time, error) { return time.ParseInLocation(DATE_FORMAT, setting.BrainRecordBeginTime, time.Local) } func DownloadCloudBrainBoard(ctx *context.Context) { page := 1 pageSize := 300 var cloudBrain = ctx.Tr("repo.cloudbrain") fileName := getCloudbrainFileName(cloudBrain) _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, }, Type: models.TypeCloudBrainAll, NeedRepoInfo: false, }) if err != nil { log.Warn("Can not get cloud brain info", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) return } totalPage := getTotalPage(total, pageSize) f := excelize.NewFile() index := f.NewSheet(cloudBrain) f.DeleteSheet("Sheet1") for k, v := range allCloudbrainHeader(ctx) { f.SetCellValue(cloudBrain, k, v) } var row = 2 for i := 0; i < totalPage; i++ { pageRecords, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, }, Type: models.TypeCloudBrainAll, NeedRepoInfo: true, }) if err != nil { log.Warn("Can not get cloud brain info", err) continue } for _, record := range pageRecords { for k, v := range allCloudbrainValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } row++ } page++ } f.SetActiveSheet(index) ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) ctx.Resp.Header().Set("Content-Type", "application/octet-stream") f.WriteTo(ctx.Resp) } func getCloudbrainFileName(baseName string) string { return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" } func allCloudbrainHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.wait_time"), "F1": ctx.Tr("repo.modelarts.train_job.dura_time"), "G1": ctx.Tr("repo.modelarts.train_job.start_time"), "H1": ctx.Tr("repo.modelarts.train_job.end_time"), "I1": ctx.Tr("repo.modelarts.computing_resources"), "J1": ctx.Tr("repo.cloudbrain_creator"), "K1": ctx.Tr("repo.repo_name"), "L1": ctx.Tr("repo.cloudbrain_task_name")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): getBrainWaitTime(rs), getCellName("F", row): rs.TrainJobDuration, getCellName("G", row): getBrainStartTime(rs), getCellName("H", row): getBrainEndTime(rs), getCellName("I", row): rs.ComputeResource, getCellName("J", row): rs.Name, getCellName("K", row): getBrainRepo(rs), getCellName("L", row): rs.JobName, } } func getBrainRepo(rs *models.CloudbrainInfo) string { if rs.Repo != nil { return rs.Repo.OwnerName + "/" + rs.Repo.Alias } return "" } func getBrainStartTime(rs *models.CloudbrainInfo) string { timeString := time.Unix(int64(rs.Cloudbrain.StartTime), 0).Format(CREATE_TIME_FORMAT) if timeString != "1970/01/01 08:00:00" { return timeString } else { return "0" } } func getBrainEndTime(rs *models.CloudbrainInfo) string { timeString := time.Unix(int64(rs.Cloudbrain.EndTime), 0).Format(CREATE_TIME_FORMAT) if timeString != "1970/01/01 08:00:00" { return timeString } else { return "0" } } func getBrainWaitTime(rs *models.CloudbrainInfo) string { waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix if waitTime <= 0 { return "0" } else { return models.ConvertDurationToStr(int64(waitTime)) } }