From 17718c507fc3234609040ccf921fb17486727c40 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 22 Mar 2022 17:50:50 +0800 Subject: [PATCH 01/87] update --- models/cloudbrain_static.go | 5 + routers/api/v1/api.go | 15 +++ routers/api/v1/repo/cloudbrain_dashboard.go | 119 ++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 models/cloudbrain_static.go create mode 100644 routers/api/v1/repo/cloudbrain_dashboard.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go new file mode 100644 index 000000000..9898ebf92 --- /dev/null +++ b/models/cloudbrain_static.go @@ -0,0 +1,5 @@ +package models + +func CountBrainStatByRawSql(sql string) (int64, error) { + return xStatistic.SQL(sql).Count() +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 306854af3..6118db265 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -554,6 +554,21 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) + //cloudbrain board + m.Group("/cloudbrainboard", func() { + m.Get("/restoreFork", repo.RestoreForkNumber) + m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) + m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) + m.Group("/cloudbrain", func() { + m.Get("", repo.GetAllCloudbrainsPeriodStatistics) + m.Group("/:id", func() { + m.Get("", repo.GetProjectLatestStatistics) + m.Get("/period", repo.GetProjectPeriodStatistics) + + }) + }) + }, operationReq) + // Users m.Group("/users", func() { m.Get("/search", user.Search) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go new file mode 100644 index 000000000..d39f5c373 --- /dev/null +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -0,0 +1,119 @@ +package repo + +import ( + "net/http" + "strconv" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + orderBy := getOrderBy(ctx) + + latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + DebugOneCountSql := generateDebugOneCountSql(beginTime, endTime) + DebugOneTotal, err := models.CountBrainStatByRawSql(beginTime, endTime) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + DebugTwoCountSql := generateDebugTwoCountSql(beginTime, endTime) + DebugOneTotal, err := models.CountBrainStatByRawSql(DebugTwoCountSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + + sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) + + projectsPeriodData := ProjectsPeriodData{ + RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), + PageSize: pageSize, + TotalPage: getTotalPage(DebugOneTotal, pageSize), + TotalCount: DebugOneTotal, + LastUpdatedTime: latestUpdatedTime, + PageRecords: models.GetRepoStatisticByRawSql(sql), + } + + ctx.JSON(http.StatusOK, projectsPeriodData) + +} + +func generateDebugOneCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "0" + + " group by job_id) " + return countSql +} +func generateBenchmarkOneCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "BENCHMARK" + + " and type=" + "0" + + " group by job_id) " + return countSql +} +func generateDebugTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "2" + + " group by job_id) " + return countSql +} +func generateTrainTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "TRAIN" + + " and type=" + "1" + + " group by job_id) " + return countSql +} +func generateInferenceTwoCountSql(beginTime time.Time, endTime time.Time) string { + countSql := "SELECT count(*) FROM " + + "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "INFERENCE" + + " and type=" + "1" + + " group by job_id) " + return countSql +} From 851262f18a4a516cfdc54a5ed65395203934badd Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 23 Mar 2022 11:12:06 +0800 Subject: [PATCH 02/87] update --- models/cloudbrain_static.go | 47 ++++++++- routers/api/v1/repo/cloudbrain_dashboard.go | 108 +++++++------------- 2 files changed, 81 insertions(+), 74 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 9898ebf92..f388caae7 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,5 +1,48 @@ package models -func CountBrainStatByRawSql(sql string) (int64, error) { - return xStatistic.SQL(sql).Count() +import ( + "strconv" + "time" +) + +func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + return xStatistic.SQL(countSql).Count() +} + +func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "BENCHMARK" + + " and type=" + "0" + return xStatistic.SQL(countSql).Count() +} +func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "DEBUG" + + " and type=" + "2" + return xStatistic.SQL(countSql).Count() +} +func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "TRAIN" + + " and type=" + "1" + return xStatistic.SQL(countSql).Count() +} +func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type=" + "INFERENCE" + + " and type=" + "1" + return xStatistic.SQL(countSql).Count() } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index d39f5c373..66df42d88 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -2,14 +2,20 @@ package repo import ( "net/http" - "strconv" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" ) +type CloudbrainsPeriodData struct { + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` +} + func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -24,7 +30,6 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } - q := ctx.QueryTrim("q") page := ctx.QueryInt("page") if page <= 0 { page = 1 @@ -33,87 +38,46 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { if pageSize <= 0 { pageSize = DEFAULT_PAGE_SIZE } - orderBy := getOrderBy(ctx) - latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime() + debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) if err != nil { - log.Error("Can not query the last updated time.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + log.Error("Can not query debugOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) return } - - DebugOneCountSql := generateDebugOneCountSql(beginTime, endTime) - DebugOneTotal, err := models.CountBrainStatByRawSql(beginTime, endTime) + benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + log.Error("Can not query benchmarkCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) return } - DebugTwoCountSql := generateDebugTwoCountSql(beginTime, endTime) - DebugOneTotal, err := models.CountBrainStatByRawSql(DebugTwoCountSql) + debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + log.Error("Can not query debugTwoCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error")) return } - - sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize) - - projectsPeriodData := ProjectsPeriodData{ - RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), - PageSize: pageSize, - TotalPage: getTotalPage(DebugOneTotal, pageSize), - TotalCount: DebugOneTotal, - LastUpdatedTime: latestUpdatedTime, - PageRecords: models.GetRepoStatisticByRawSql(sql), + trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query DebugOneTotal count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error")) + return + } + inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query inferenceTwoCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error")) + return } - ctx.JSON(http.StatusOK, projectsPeriodData) + cloudbrainsPeriodData := CloudbrainsPeriodData{ + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + } -} + ctx.JSON(http.StatusOK, cloudbrainsPeriodData) -func generateDebugOneCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "0" + - " group by job_id) " - return countSql -} -func generateBenchmarkOneCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "BENCHMARK" + - " and type=" + "0" + - " group by job_id) " - return countSql -} -func generateDebugTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "2" + - " group by job_id) " - return countSql -} -func generateTrainTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "TRAIN" + - " and type=" + "1" + - " group by job_id) " - return countSql -} -func generateInferenceTwoCountSql(beginTime time.Time, endTime time.Time) string { - countSql := "SELECT count(*) FROM " + - "(SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "INFERENCE" + - " and type=" + "1" + - " group by job_id) " - return countSql } From 0034d1f23068e87061bdaad427f5942d553ee37d Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 23 Mar 2022 16:55:24 +0800 Subject: [PATCH 03/87] update --- models/cloudbrain_static.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index f388caae7..e6bdd7848 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -9,40 +9,41 @@ func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" - return xStatistic.SQL(countSql).Count() + return x.SQL(countSql).Count() } func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "BENCHMARK" + - " and type=" + "0" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeBenchmark) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + return x.SQL(countSql).Count() } func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "DEBUG" + - " and type=" + "2" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "TRAIN" + - " and type=" + "1" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + - "SELECT job_id FROM cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and job_type=" + "INFERENCE" + - " and type=" + "1" - return xStatistic.SQL(countSql).Count() + " and job_type ='" + string(JobTypeInference) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() } From 43358dee5eca837cabfd59a41496e498ec63ef7e Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 24 Mar 2022 10:42:28 +0800 Subject: [PATCH 04/87] update --- routers/repo/repo_statistic.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 468e6fa85..8c7ea0fe5 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -17,6 +17,7 @@ import ( func StatisticAuto() { RepoStatisticAuto() TimingCountData() + CloudbrainStatisticAuto() } //auto daily @@ -334,3 +335,12 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string } return nil } + +func CloudbrainStatisticAuto() { + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + CloudbrainStatisticDaily(yesterday) +} + +func CloudbrainStatisticDaily(date string) { + return +} From 3ea149bada66466eadf721bed4ebf5dba6f180f3 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 25 Mar 2022 09:46:55 +0800 Subject: [PATCH 05/87] update --- models/cloudbrain_static.go | 17 +++++++++++++++++ modules/cron/tasks_basic.go | 11 +++++++++++ routers/repo/repo_statistic.go | 1 - 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index e6bdd7848..f646daf0c 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -3,8 +3,25 @@ package models import ( "strconv" "time" + + "code.gitea.io/gitea/modules/timeutil" ) +// Cloudbrain statistic info of all CloudbrainTasks +type CloudbrainStatistic struct { + ID int64 `xorm:"pk autoincr" json:"-"` + Date string `xorm:"unique(s) NOT NULL" json:"date"` + NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` + NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` + NumTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"` + NumDubugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"` + NumTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"` + NumInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"` + + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` +} + func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b9838e66f..011cc8dba 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -173,6 +173,16 @@ func registerHandleRepoAndUserStatistic() { return nil }) } +func registerHandleCloudbrainStatistic() { + RegisterTaskFatal("handle_cloudbrain_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainStatisticAuto() + return nil + }) +} func registerHandleSummaryStatistic() { RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ @@ -212,6 +222,7 @@ func initBasicTasks() { registerHandleBlockChainUnSuccessCommits() registerHandleRepoAndUserStatistic() + registerHandleCloudbrainStatistic() registerHandleSummaryStatistic() registerSyncCloudbrainStatus() diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 8c7ea0fe5..b889eb951 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -17,7 +17,6 @@ import ( func StatisticAuto() { RepoStatisticAuto() TimingCountData() - CloudbrainStatisticAuto() } //auto daily From 6871c8c15e8b1add99775b8fc839ff22876b4ce8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 28 Mar 2022 18:03:42 +0800 Subject: [PATCH 06/87] update --- models/cloudbrain_static.go | 119 +++++++++++++++++++++++++++ routers/repo/cloudbrain_statistic.go | 115 ++++++++++++++++++++++++++ routers/repo/repo_statistic.go | 9 -- 3 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 routers/repo/cloudbrain_statistic.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index f646daf0c..1b7bef01a 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,9 +1,11 @@ package models import ( + "fmt" "strconv" "time" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) @@ -11,12 +13,24 @@ import ( type CloudbrainStatistic struct { ID int64 `xorm:"pk autoincr" json:"-"` Date string `xorm:"unique(s) NOT NULL" json:"date"` + WhichHour int64 `xorm:"NOT NULL DEFAULT -1" json:"whichHour"` NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` NumTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"` NumDubugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"` NumTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"` NumInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"` + NumCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudOneAll"` + NumCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"numCloudTwoAll"` + + DurationDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDubugOne"` + DurationBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationBenchmarkOne"` + DurationTrainOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainOne"` + DurationDebugTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDebugTwo"` + DurationTrainTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainTwo"` + DurationInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationInferenceTwo"` + DurationCloudOneAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudOneAll"` + DurationCloudTwoAll int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudTwoAll"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` @@ -31,6 +45,32 @@ func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error return x.SQL(countSql).Count() } +func GetDebugOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} + +func GenerateTrainOneCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + + return x.SQL(countSql).Count() +} +func GetTrainOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + @@ -40,6 +80,14 @@ func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, e " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" return x.SQL(countSql).Count() } +func GetBenchmarkOneDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -48,6 +96,13 @@ func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetDebugTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -56,6 +111,13 @@ func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetTrainTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -64,3 +126,60 @@ func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, e " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" return x.SQL(countSql).Count() } +func GetInferenceTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} + +func DeleteCloudbrainStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&CloudbrainStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetHourStatTime(timeStr string, whichHour int64) (time.Time, time.Time) { + t, _ := time.Parse("2006-01-02", timeStr) + timeNumber := t.Unix() + beginTimeNumber := timeNumber - 8*60*60 + whichHour*60*60 + endTimeNumber := beginTimeNumber + 1*60*60 + beginTime := time.Unix(beginTimeNumber, 0) + endTime := time.Unix(endTimeNumber, 0) + log.Info("%s, %s", beginTime, endTime) + + return beginTime, endTime +} +func GetDayStatTime(timeStr string) (time.Time, time.Time) { + t, _ := time.Parse("2006-01-02", timeStr) + timeNumber := t.Unix() + beginTimeNumber := timeNumber - 8*60*60 + endTimeNumber := beginTimeNumber + 16*60*60 + beginTime := time.Unix(beginTimeNumber, 0) + endTime := time.Unix(endTimeNumber, 0) + log.Info("%s, %s", beginTime, endTime) + + return beginTime, endTime +} + +func CreateCloudbrainStatistic(cloudbrainStat *CloudbrainStatistic) (err error) { + if _, err = xStatistic.Insert(cloudbrainStat); err != nil { + return err + } + return nil +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..926fa68c7 --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,115 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/mailer" +) + +func CloudbrainStatisticAuto() { + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + CloudbrainStatisticDaily(yesterday) +} + +func CloudbrainStatisticDaily(date string) { + log.Info("%s", date) + log.Info("begin Repo Statistic") + warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。" + if err := models.DeleteCloudbrainStatisticDaily(date); err != nil { + log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error()) + mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) + return + } + + //0 to 23 for each hour, 25 for all 24 hours + var WhichHour [25]int64 = [25]int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25} + for _, whichHour := range WhichHour { + log.Info("start statistic: %s", date) + var beginTime, endTime time.Time + beginTime, endTime = models.GetHourStatTime(date, whichHour) + if whichHour == 25 { + beginTime, endTime = models.GetDayStatTime(date) + } + numDubugOne, err := models.GenerateDebugOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateDebugOneCount failed(%s): %v", numDubugOne, err) + } + numBenchmarkOne, err := models.GenerateBenchmarkOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateBenchmarkOneCount failed(%s): %v", numBenchmarkOne, err) + } + numTrainOne, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("GenerateTrainOneCount failed(%s): %v", numTrainOne, err) + } + numDebugTwo, err := models.GenerateDebugTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateDebugTwoCount failed(%s): %v", numDebugTwo, err) + } + numTrainTwo, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateTrainTwoCount failed(%s): %v", numTrainTwo, err) + } + numInferenceTwo, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("GenerateInferenceTwoCount failed(%s): %v", numInferenceTwo, err) + } + numCloudOneAll := numDubugOne + numBenchmarkOne + numTrainOne + numCloudTwoAll := numDebugTwo + numTrainTwo + numInferenceTwo + + durationDubugOne, err := models.GetDebugOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetDebugOneDuration failed(%s): %v", durationDubugOne, err) + } + durationBenchmarkOne, err := models.GetBenchmarkOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetBenchmarkOneDuration failed(%s): %v", durationBenchmarkOne, err) + } + durationTrainOne, err := models.GetTrainOneDuration(beginTime, endTime) + if err != nil { + log.Error("GetTrainOneDuration failed(%s): %v", durationTrainOne, err) + } + durationDebugTwo, err := models.GetDebugTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetDebugTwoDuration failed(%s): %v", durationDebugTwo, err) + } + durationTrainTwo, err := models.GetTrainTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetTrainTwoDuration failed(%s): %v", durationTrainTwo, err) + } + durationInferenceTwo, err := models.GetInferenceTwoDuration(beginTime, endTime) + if err != nil { + log.Error("GetInferenceTwoDuration failed(%s): %v", durationInferenceTwo, err) + } + durationCloudOneAll := durationDubugOne + durationBenchmarkOne + durationTrainOne + durationCloudTwoAll := durationDebugTwo + durationTrainTwo + durationInferenceTwo + + err = models.CreateCloudbrainStatistic(&models.CloudbrainStatistic{ + Date: date, + WhichHour: whichHour, + NumDubugOne: numDubugOne, + NumBenchmarkOne: numBenchmarkOne, + NumTrainOne: numTrainOne, + NumDubugTwo: numDebugTwo, + NumTrainTwo: numTrainTwo, + NumInferenceTwo: numInferenceTwo, + NumCloudOneAll: numCloudOneAll, + NumCloudTwoAll: numCloudTwoAll, + DurationDubugOne: durationDubugOne, + DurationBenchmarkOne: durationBenchmarkOne, + DurationTrainOne: durationTrainOne, + DurationDebugTwo: durationDebugTwo, + DurationTrainTwo: durationTrainTwo, + DurationInferenceTwo: durationInferenceTwo, + DurationCloudOneAll: durationCloudOneAll, + DurationCloudTwoAll: durationCloudTwoAll, + }) + if err != nil { + log.Error("CreateCloudbrainStatistic(%s) failed:%v", date, err.Error()) + return + } + } +} diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index b889eb951..468e6fa85 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -334,12 +334,3 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string } return nil } - -func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") - CloudbrainStatisticDaily(yesterday) -} - -func CloudbrainStatisticDaily(date string) { - return -} From 34cf97e9bb7d7e97407a12585d3928ec69f5f8b5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 29 Mar 2022 09:42:48 +0800 Subject: [PATCH 07/87] update --- routers/api/v1/api.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 6118db265..89914f688 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -556,16 +556,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { - m.Get("/restoreFork", repo.RestoreForkNumber) - m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) - m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) m.Group("/cloudbrain", func() { m.Get("", repo.GetAllCloudbrainsPeriodStatistics) - m.Group("/:id", func() { - m.Get("", repo.GetProjectLatestStatistics) - m.Get("/period", repo.GetProjectPeriodStatistics) - - }) }) }, operationReq) From 3696ce23c414dd9bfd56d82c82f74a9ee8badaa8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 31 Mar 2022 10:33:25 +0800 Subject: [PATCH 08/87] update --- models/cloudbrain_static.go | 2 +- models/models.go | 1 + routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 74 +++++++++++++++++++++ routers/repo/cloudbrain_statistic.go | 4 +- routers/routes/routes.go | 1 + 6 files changed, 80 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 1b7bef01a..4d0e7cfaa 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -12,7 +12,7 @@ import ( // Cloudbrain statistic info of all CloudbrainTasks type CloudbrainStatistic struct { ID int64 `xorm:"pk autoincr" json:"-"` - Date string `xorm:"unique(s) NOT NULL" json:"date"` + Date string `xorm:"NOT NULL DEFAULT 0" json:"date"` WhichHour int64 `xorm:"NOT NULL DEFAULT -1" json:"whichHour"` NumDubugOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"` NumBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"` diff --git a/models/models.go b/models/models.go index 0f4679b4f..02cfb3144 100755 --- a/models/models.go +++ b/models/models.go @@ -151,6 +151,7 @@ func init() { new(UserBusinessAnalysisCurrentWeek), new(UserBusinessAnalysisYesterday), new(UserLoginLog), + new(CloudbrainStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 89914f688..87af1d19b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -556,6 +556,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { + m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) m.Group("/cloudbrain", func() { m.Get("", repo.GetAllCloudbrainsPeriodStatistics) }) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 66df42d88..358aa6c37 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -2,10 +2,12 @@ package repo import ( "net/http" + "net/url" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "github.com/360EntSecGroup-Skylar/excelize/v2" ) type CloudbrainsPeriodData struct { @@ -81,3 +83,75 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } +func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 1000 + orderBy := getOrderBy(ctx) + + _, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + countSql := generateCountSql(beginTime, endTime, latestDate, q) + total, err := models.CountRepoStatByRawSql(countSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") + fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) + + totalPage := getTotalPage(total, pageSize) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeroidHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var row = 2 + for i := 0; i <= totalPage; i++ { + + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + for _, record := range pageRecords { + + for k, v := range allProjectsPeroidValues(row, record, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + + } + + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 926fa68c7..16137ed2e 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -10,13 +10,13 @@ import ( ) func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + yesterday := time.Now().AddDate(0, 0, 0).Format("2006-01-02") CloudbrainStatisticDaily(yesterday) } func CloudbrainStatisticDaily(date string) { log.Info("%s", date) - log.Info("begin Repo Statistic") + log.Info("begin Cloudbrain Statistic") warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。" if err := models.DeleteCloudbrainStatisticDaily(date); err != nil { log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error()) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 2d146c2c6..a2e969a60 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -990,6 +990,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/cloudbrain", func() { + m.Get("/test", repo.CloudbrainStatisticAuto) m.Group("/:id", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDebug) From 29fdbd5d580f6cd341255332616f0a2a3bea1070 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 6 Apr 2022 18:15:35 +0800 Subject: [PATCH 09/87] update --- models/cloudbrain_static.go | 10 + routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 272 +++++++++++++++++++- routers/repo/cloudbrain_statistic.go | 2 +- 4 files changed, 283 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 4d0e7cfaa..8e9329ce2 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -36,6 +36,16 @@ type CloudbrainStatistic struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` } +type TimeCloudbrainNum struct { + Time time.Time `json:"time"` + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` +} + func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index fded809f4..c03ffbe02 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -559,6 +559,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrainboard", func() { m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) m.Group("/cloudbrain", func() { + m.Get("/trend", repo.GetAllCloudbrainsTrend) m.Get("", repo.GetAllCloudbrainsPeriodStatistics) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 358aa6c37..609783bfb 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1,8 +1,10 @@ package repo import ( + "fmt" "net/http" "net/url" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -13,10 +15,52 @@ import ( type CloudbrainsPeriodData struct { DebugOneCount int64 `json:"debugOneCount"` BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` DebugTwoCount int64 `json:"debugTwoCount"` TrainTwoCount int64 `json:"trainTwoCount"` InferenceTwoCount int64 `json:"inferenceTwoCount"` } +type TimeCloudbrainsNum struct { + TimeCloudbrainNum []models.TimeCloudbrainNum `json:"imeCloudbrainNum"` +} + +func GetAllCloudbrainsTrend(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + timeCloudbrainNum, beginTime, endTime, err := getCloudbrainTrendTime(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + + cloudbrainsPeriodData := TimeCloudbrainsNum{ + TimeCloudbrainNum: timeCloudbrainNum, + } + now := time.Now() + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + testtime := now.AddDate(0, -1, 1) + fmt.Printf("beginTime is: %s", beginTime) + fmt.Printf("endTime is: %s", endTime) + fmt.Printf("testtime is:%s", testtime) + + ctx.JSON(http.StatusOK, cloudbrainsPeriodData) + +} func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { @@ -53,6 +97,12 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) return } + trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query trainOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error")) + return + } debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) @@ -75,10 +125,18 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { cloudbrainsPeriodData := CloudbrainsPeriodData{ DebugOneCount: debugOneCount, BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, DebugTwoCount: debugTwoCount, TrainTwoCount: trainTwoCount, InferenceTwoCount: inferenceTwoCount, } + now := time.Now() + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + testtime := now.AddDate(0, -1, 1) + fmt.Printf("beginTime is: %s", beginTime) + fmt.Printf("endTime is: %s", endTime) + fmt.Printf("testtime is:%s", testtime) ctx.JSON(http.StatusOK, cloudbrainsPeriodData) @@ -91,7 +149,7 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return } - beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { log.Error("Parameter is wrong", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) @@ -155,3 +213,215 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { f.WriteTo(ctx.Resp) } + +func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { + queryType := ctx.QueryTrim("type") + now := time.Now() + recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var err error + if queryType != "" { + + if queryType == "all" { + beginTime = recordBeginTimeTemp + endTime = now + } else if queryType == "yesterday" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + + } else { + return now, now, fmt.Errorf("The value of type parameter is wrong.") + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTimeTemp + endTime = now + + } else { + + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + return now, now, err + } + + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + return now, now, err + } + + beginTime = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 1) + } + + } + + if beginTime.Before(recordBeginTimeTemp) { + beginTime = recordBeginTimeTemp + } + + return beginTime, endTime, nil + +} + +func getCloudbrainTrendTime(ctx *context.Context, recordBeginTime time.Time) ([]models.TimeCloudbrainNum, time.Time, time.Time, error) { + queryType := ctx.QueryTrim("type") + now := time.Now() + recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var err error + timeCloudbrainNum := make([]models.TimeCloudbrainNum, 0) + if queryType != "" { + + if queryType == "all" { + beginTime = recordBeginTimeTemp + endTime = now + } else if queryType == "yesterday" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime1 := beginTime.AddDate(0, 0, 1) + for endTime1.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err := getCloudbrainCount(beginTime, endTime1) + if err != nil { + log.Error("Can not query debugOneCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) + } + timeCloudbrainNum = append(timeCloudbrainNum, models.TimeCloudbrainNum{ + Time: beginTime, + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + }) + beginTime = endTime1 + endTime1 = beginTime.AddDate(0, 0, 1) + } + return timeCloudbrainNum, beginTime, endTime, nil + + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + + } else { + return timeCloudbrainNum, now, now, fmt.Errorf("The value of type parameter is wrong.") + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTimeTemp + endTime = now + + } else { + + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + return timeCloudbrainNum, now, now, err + } + + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + return timeCloudbrainNum, now, now, err + } + + beginTime = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 1) + } + + } + + if beginTime.Before(recordBeginTimeTemp) { + beginTime = recordBeginTimeTemp + } + + return timeCloudbrainNum, beginTime, endTime, nil + +} + +func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, error) { + debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query debugOneCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query benchmarkCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) + if err != nil { + log.Error("Can not query trainOneCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query debugTwoCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query DebugOneTotal count.", err) + return 0, 0, 0, 0, 0, 0, err + } + inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) + if err != nil { + log.Error("Can not query inferenceTwoCount.", err) + return 0, 0, 0, 0, 0, 0, err + } + return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 16137ed2e..142b0c14e 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -10,7 +10,7 @@ import ( ) func CloudbrainStatisticAuto() { - yesterday := time.Now().AddDate(0, 0, 0).Format("2006-01-02") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") CloudbrainStatisticDaily(yesterday) } From f7bff308e2b7ba9f43890da10f5da948faf3bc20 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 7 Apr 2022 17:01:30 +0800 Subject: [PATCH 10/87] update --- models/cloudbrain_static.go | 10 - modules/setting/setting.go | 26 +- routers/api/v1/repo/cloudbrain_dashboard.go | 528 ++++++++++++-------- 3 files changed, 334 insertions(+), 230 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 8e9329ce2..4d0e7cfaa 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -36,16 +36,6 @@ type CloudbrainStatistic struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"` } -type TimeCloudbrainNum struct { - Time time.Time `json:"time"` - DebugOneCount int64 `json:"debugOneCount"` - BenchmarkOneCount int64 `json:"benchmarkOneCount"` - TrainOneCount int64 `json:"trainOneCount"` - DebugTwoCount int64 `json:"debugTwoCount"` - TrainTwoCount int64 `json:"trainTwoCount"` - InferenceTwoCount int64 `json:"inferenceTwoCount"` -} - func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 26f068193..19620f5db 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -452,18 +452,19 @@ var ( DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - CBCodePathPrefix string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string - MaxDuration int64 - TrainGpuTypes string - TrainResourceSpecs string + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string + MaxDuration int64 + TrainGpuTypes string + TrainResourceSpecs string + BrainRecordBeginTime string //benchmark config IsBenchmarkEnabled bool @@ -1290,6 +1291,7 @@ func NewContext() { MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") + BrainRecordBeginTime = sec.Key("brain_record_beigin_time").MustString("2021-01-01") sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 609783bfb..85bea2189 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -9,6 +9,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "github.com/360EntSecGroup-Skylar/excelize/v2" ) @@ -21,42 +22,172 @@ type CloudbrainsPeriodData struct { InferenceTwoCount int64 `json:"inferenceTwoCount"` } type TimeCloudbrainsNum struct { - TimeCloudbrainNum []models.TimeCloudbrainNum `json:"imeCloudbrainNum"` + TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"` +} +type DateCloudbrainNum struct { + Date string `json:"date"` + DebugOneCount int64 `json:"debugOneCount"` + BenchmarkOneCount int64 `json:"benchmarkOneCount"` + TrainOneCount int64 `json:"trainOneCount"` + DebugTwoCount int64 `json:"debugTwoCount"` + TrainTwoCount int64 `json:"trainTwoCount"` + InferenceTwoCount int64 `json:"inferenceTwoCount"` + CloudbrainOneCount int64 `json:"cloudbrainOneCount"` + CloudbrainTwoCount int64 `json:"cloudbrainTwoCount"` + CloudbrainCount int64 `json:"cloudbrainCount"` } func GetAllCloudbrainsTrend(ctx *context.Context) { - - recordBeginTime, err := getRecordBeginTime() + brainRecordBeginTime, err := getBrainRecordBeginTime() if err != nil { - log.Error("Can not get record begin time", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + log.Error("Can not get brain record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err")) return } - timeCloudbrainNum, beginTime, endTime, err := getCloudbrainTrendTime(ctx, recordBeginTime) - if err != nil { - log.Error("Parameter is wrong", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) - return - } - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pageSize := ctx.QueryInt("pagesize") - if pageSize <= 0 { - pageSize = DEFAULT_PAGE_SIZE - } + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var endTimeTemp time.Time + dateCloudbrainNum := make([]DateCloudbrainNum, 0) + if queryType != "" { + if queryType == "all" { + beginTime = brainRecordBeginTime + endTime = now + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getYearCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.Add(time.Hour) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getHourCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) + endTimeTemp = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = brainRecordBeginTime + endTime = now + endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + days := (endTime.Unix() - beginTime.Unix()) / 3600 / 24 + if 1 < days { + endTimeTemp = beginTime.AddDate(0, 0, 1) + endTime = endTime.AddDate(0, 0, 2) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if 0 < days || days <= 1 { + endTimeTemp = beginTime.Add(time.Hour) + dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime) + if err != nil { + log.Error("Can not query getHourCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getHourCloudbrainNum_get_error")) + return + } + } else { + return + } + } + } cloudbrainsPeriodData := TimeCloudbrainsNum{ - TimeCloudbrainNum: timeCloudbrainNum, + TimeCloudbrainNum: dateCloudbrainNum, } - now := time.Now() - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location()) - testtime := now.AddDate(0, -1, 1) - fmt.Printf("beginTime is: %s", beginTime) - fmt.Printf("endTime is: %s", endTime) - fmt.Printf("testtime is:%s", testtime) ctx.JSON(http.StatusOK, cloudbrainsPeriodData) @@ -141,79 +272,6 @@ func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) { ctx.JSON(http.StatusOK, cloudbrainsPeriodData) } -func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { - - recordBeginTime, err := getRecordBeginTime() - if err != nil { - log.Error("Can not get record begin time", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return - } - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("Parameter is wrong", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) - return - } - q := ctx.QueryTrim("q") - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pageSize := 1000 - orderBy := getOrderBy(ctx) - - _, latestDate, err := models.GetRepoStatLastUpdatedTime() - if err != nil { - log.Error("Can not query the last updated time.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) - return - } - - countSql := generateCountSql(beginTime, endTime, latestDate, q) - total, err := models.CountRepoStatByRawSql(countSql) - if err != nil { - log.Error("Can not query total count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) - return - } - var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") - fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) - - totalPage := getTotalPage(total, pageSize) - - f := excelize.NewFile() - - index := f.NewSheet(projectAnalysis) - f.DeleteSheet("Sheet1") - - for k, v := range allProjectsPeroidHeader(ctx) { - f.SetCellValue(projectAnalysis, k, v) - } - - var row = 2 - for i := 0; i <= totalPage; i++ { - - pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) - for _, record := range pageRecords { - - for k, v := range allProjectsPeroidValues(row, record, ctx) { - f.SetCellValue(projectAnalysis, k, v) - } - row++ - - } - - } - f.SetActiveSheet(index) - - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - - f.WriteTo(ctx.Resp) - -} - func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { queryType := ctx.QueryTrim("type") now := time.Now() @@ -292,136 +350,190 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t } -func getCloudbrainTrendTime(ctx *context.Context, recordBeginTime time.Time) ([]models.TimeCloudbrainNum, time.Time, time.Time, error) { - queryType := ctx.QueryTrim("type") - now := time.Now() - recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1) - - beginTimeStr := ctx.QueryTrim("beginTime") - endTimeStr := ctx.QueryTrim("endTime") - var beginTime time.Time - var endTime time.Time - var err error - timeCloudbrainNum := make([]models.TimeCloudbrainNum, 0) - if queryType != "" { - - if queryType == "all" { - beginTime = recordBeginTimeTemp - endTime = now - } else if queryType == "yesterday" { - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) - - } else if queryType == "current_week" { - beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday - beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) - endTime = now - } else if queryType == "current_month" { - endTime = now - beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) - endTime1 := beginTime.AddDate(0, 0, 1) - for endTime1.Before(endTime) { - debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err := getCloudbrainCount(beginTime, endTime1) - if err != nil { - log.Error("Can not query debugOneCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) - } - timeCloudbrainNum = append(timeCloudbrainNum, models.TimeCloudbrainNum{ - Time: beginTime, - DebugOneCount: debugOneCount, - BenchmarkOneCount: benchmarkOneCount, - TrainOneCount: trainOneCount, - DebugTwoCount: debugTwoCount, - TrainTwoCount: trainTwoCount, - InferenceTwoCount: inferenceTwoCount, - }) - beginTime = endTime1 - endTime1 = beginTime.AddDate(0, 0, 1) - } - return timeCloudbrainNum, beginTime, endTime, nil - - } else if queryType == "monthly" { - endTime = now - beginTime = now.AddDate(0, -1, 1) - beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) - - } else if queryType == "current_year" { - endTime = now - beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location()) - - } else if queryType == "last_month" { - - lastMonthTime := now.AddDate(0, -1, 0) - beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location()) - endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location()) - - } else { - return timeCloudbrainNum, now, now, fmt.Errorf("The value of type parameter is wrong.") - - } - - } else { - if beginTimeStr == "" || endTimeStr == "" { - //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 - beginTime = recordBeginTimeTemp - endTime = now - - } else { - - beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) - if err != nil { - return timeCloudbrainNum, now, now, err - } - - endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) - if err != nil { - return timeCloudbrainNum, now, now, err - } - - beginTime = beginTime.AddDate(0, 0, 1) - endTime = endTime.AddDate(0, 0, 1) - } - - } - - if beginTime.Before(recordBeginTimeTemp) { - beginTime = recordBeginTimeTemp - } - - return timeCloudbrainNum, beginTime, endTime, nil - -} - -func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, error) { +func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) { debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime) if err != nil { log.Error("Can not query debugOneCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime) if err != nil { log.Error("Can not query benchmarkCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime) if err != nil { log.Error("Can not query trainOneCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query debugTwoCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query DebugOneTotal count.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime) if err != nil { log.Error("Can not query inferenceTwoCount.", err) - return 0, 0, 0, 0, 0, 0, err + return 0, 0, 0, 0, 0, 0, 0, 0, 0, err } - return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, err + cloudbrainOneCount := debugOneCount + benchmarkOneCount + trainOneCount + cloudbrainTwoCount := debugTwoCount + trainTwoCount + inferenceTwoCount + cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount + return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err +} +func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + dayCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format(time.RFC3339), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.Add(time.Hour) + } + return dayCloudbrainNum, nil +} +func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + dayCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006-01-02"), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 0, 1) + } + return dayCloudbrainNum, nil +} +func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + yearCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) { + debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + if err != nil { + log.Error("Can not query getCloudbrainCount.", err) + return nil, err + } + yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006-01"), + DebugOneCount: debugOneCount, + BenchmarkOneCount: benchmarkOneCount, + TrainOneCount: trainOneCount, + DebugTwoCount: debugTwoCount, + TrainTwoCount: trainTwoCount, + InferenceTwoCount: inferenceTwoCount, + CloudbrainOneCount: cloudbrainOneCount, + CloudbrainTwoCount: cloudbrainTwoCount, + CloudbrainCount: cloudbrainCount, + }) + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 1, 0) + } + return yearCloudbrainNum, nil +} +func getBrainRecordBeginTime() (time.Time, error) { + return time.ParseInLocation(DATE_FORMAT, setting.BrainRecordBeginTime, time.Local) +} + +func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 1000 + orderBy := getOrderBy(ctx) + + _, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + countSql := generateCountSql(beginTime, endTime, latestDate, q) + total, err := models.CountRepoStatByRawSql(countSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + var projectAnalysis = ctx.Tr("repo.repo_stat_inspect") + fileName := getFileName(ctx, beginTime, endTime, projectAnalysis) + + totalPage := getTotalPage(total, pageSize) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeroidHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var row = 2 + for i := 0; i <= totalPage; i++ { + + pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + for _, record := range pageRecords { + + for k, v := range allProjectsPeroidValues(row, record, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + + } + + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + } From 05ab115942601cc6716acb905e6f2a3ae482573c Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 8 Apr 2022 18:25:35 +0800 Subject: [PATCH 11/87] update --- models/cloudbrain.go | 65 +- options/locale/locale_zh-CN.ini | 2 + routers/api/v1/api.go | 5 +- routers/api/v1/repo/cloudbrain_dashboard.go | 123 +++ web_src/js/components/BrainAnalysis.vue | 1008 +++++++++++++++++++ web_src/js/components/DataAnalysis.vue | 10 + 6 files changed, 1210 insertions(+), 3 deletions(-) create mode 100644 web_src/js/components/BrainAnalysis.vue diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1662dcd96..7f7b9bcd6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1,13 +1,14 @@ package models import ( - "code.gitea.io/gitea/modules/util" "encoding/json" "fmt" "strconv" "strings" "time" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" "xorm.io/xorm" @@ -1564,3 +1565,65 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { return nil } + +func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"cloudbrain.type": opts.Type}, + ) + } + + var count int64 + var err error + condition := "cloudbrain.user_id = `user`.id" + if len(opts.Keyword) == 0 { + count, err = sess.Where(cond).Count(new(Cloudbrain)) + } else { + lowerKeyWord := strings.ToLower(opts.Keyword) + + cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) + count, err = sess.Table(&Cloudbrain{}).Where(cond). + Join("left", "`user`", condition).Count(new(CloudbrainInfo)) + + } + + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Join("left", "`user`", condition). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + if opts.NeedRepoInfo { + var ids []int64 + for _, task := range cloudbrains { + ids = append(ids, task.RepoID) + } + repositoryMap, err := GetRepositoriesMapByIDs(ids) + if err == nil { + for _, task := range cloudbrains { + task.Repo = repositoryMap[task.RepoID] + } + } + + } + + return cloudbrains, count, nil +} diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index d26065363..06ea4bd7f 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1005,6 +1005,8 @@ modelarts.train_job.job_status=任务状态 modelarts.train_job.job_name=任务名称 modelarts.train_job.version=任务版本 modelarts.train_job.start_time=开始时间 +modelarts.train_job.end_time=结束时间 +modelarts.train_job.wait_time=等待时间 modelarts.train_job.dura_time=运行时长 modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c03ffbe02..14bd2e2d1 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -557,10 +557,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) //cloudbrain board m.Group("/cloudbrainboard", func() { - m.Get("/downloadtable", repo.ServeCloudbrainPeriodStatisticsFile) + m.Get("/downloadStatistics", repo.ServeCloudbrainPeriodStatisticsFile) + m.Get("/downloadAll", repo.DownloadCloudBrainBoard) m.Group("/cloudbrain", func() { m.Get("/trend", repo.GetAllCloudbrainsTrend) - m.Get("", repo.GetAllCloudbrainsPeriodStatistics) + m.Get("/statistics", repo.GetAllCloudbrainsPeriodStatistics) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 85bea2189..ab26d7dfc 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -537,3 +537,126 @@ func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) { f.WriteTo(ctx.Resp) } +func DownloadCloudBrainBoard(ctx *context.Context) { + + page := 1 + + pageSize := 300 + + var cloudBrain = ctx.Tr("repo.cloudbrain") + fileName := getCloudbrainFileName(cloudBrain) + + _, total, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: 1, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: false, + }) + + if err != nil { + log.Warn("Can not get cloud brain info", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) + return + } + + totalPage := getTotalPage(total, pageSize) + fmt.Printf("total:%v", total) + fmt.Printf("totalPage:%v", totalPage) + + f := excelize.NewFile() + + index := f.NewSheet(cloudBrain) + f.DeleteSheet("Sheet1") + + for k, v := range allHeader(ctx) { + f.SetCellValue(cloudBrain, k, v) + } + + var row = 2 + for i := 0; i < totalPage; i++ { + + pageRecords, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + Type: models.TypeCloudBrainAll, + NeedRepoInfo: true, + }) + if err != nil { + log.Warn("Can not get cloud brain info", err) + continue + } + for _, record := range pageRecords { + + for k, v := range allValues(row, record, ctx) { + f.SetCellValue(cloudBrain, k, v) + } + row++ + + } + + page++ + } + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) +} + +func getCloudbrainFileName(baseName string) string { + return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" + +} +func allHeader(ctx *context.Context) map[string]string { + + return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), + "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.dura_time"), "F1": ctx.Tr("repo.modelarts.computing_resources"), + "G1": ctx.Tr("repo.cloudbrain_creator"), "H1": ctx.Tr("repo.repo_name"), "I1": ctx.Tr("repo.cloudbrain_task_name"), "J1": ctx.Tr("repo.modelarts.train_job.start_time"), + "K1": ctx.Tr("repo.modelarts.train_job.end_time"), "L1": ctx.Tr("repo.modelarts.train_job.wait_time")} + +} +func allValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { + return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, + getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): rs.TrainJobDuration, + getCellName("F", row): rs.ComputeResource, getCellName("G", row): rs.Name, getCellName("H", row): getRepoPathName(rs), + getCellName("I", row): rs.JobName, getCellName("J", row): getBrainStartTime(rs), + getCellName("K", row): getBrainEndTime(rs), getCellName("L", row): getBrainWaitTime(rs), + } +} +func getRepoPathName(rs *models.CloudbrainInfo) string { + if rs.Repo != nil { + return rs.Repo.OwnerName + "/" + rs.Repo.Alias + } + return "" +} +func getBrainStartTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.StartTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainEndTime(rs *models.CloudbrainInfo) string { + timeString := time.Unix(int64(rs.Cloudbrain.EndTime), 0).Format(CREATE_TIME_FORMAT) + if timeString != "1970/01/01 08:00:00" { + return timeString + } else { + return "0" + } + +} +func getBrainWaitTime(rs *models.CloudbrainInfo) string { + waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix + if waitTime <= 0 { + return "0" + } else { + return models.ConvertDurationToStr(int64(waitTime)) + } +} diff --git a/web_src/js/components/BrainAnalysis.vue b/web_src/js/components/BrainAnalysis.vue new file mode 100644 index 000000000..4f096e024 --- /dev/null +++ b/web_src/js/components/BrainAnalysis.vue @@ -0,0 +1,1008 @@ + + + + + diff --git a/web_src/js/components/DataAnalysis.vue b/web_src/js/components/DataAnalysis.vue index ae536db28..6f80c3481 100755 --- a/web_src/js/components/DataAnalysis.vue +++ b/web_src/js/components/DataAnalysis.vue @@ -26,6 +26,14 @@ + + + + + + 云脑分析 + + @@ -33,12 +41,14 @@