From 0111ef5d923ccbe1a46dcb0bff30a606ccb734eb Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 6 Dec 2022 08:52:00 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 ++-- modules/cron/tasks_basic.go | 4 +++- modules/setting/setting.go | 25 ++++++++++++++++++++----- services/cloudbrain/clear.go | 29 +++++++++++++++++------------ 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e492d9eb9..abb804eef 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2050,9 +2050,9 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) { +func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) - endTimeBefore := time.Now().Unix() - days*24*3600 + endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 return cloudbrains, x.Cols("id,job_name,job_id"). In("status", diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 9b74c1637..1813b1d50 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,6 +5,7 @@ package cron import ( + "code.gitea.io/gitea/modules/setting" "context" "time" @@ -196,7 +197,7 @@ func registerHandleClearCloudbrainResult() { RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "* 0,30 2-8 * * ? *", + Schedule: setting.ClearStrategy.Cron, }, func(ctx context.Context, _ *models.User, _ Config) error { cloudbrainService.ClearCloudbrainResultSpace() return nil @@ -319,6 +320,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleClearCloudbrainResult() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2b2ded678..b5ec488a8 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,8 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - ResultSaveDays int64 - ResultBatchSize int + //benchmark config IsBenchmarkEnabled bool @@ -615,6 +614,15 @@ var ( UsageRateBeginTime string }{} + ClearStrategy= struct { + Enabled bool + ResultSaveDays int + ResultBatchSize int + BatchSize int + TrashSaveDays int + Cron string + }{} + C2NetInfos *C2NetSqInfos CenterInfos *AiCenterInfos C2NetMapInfo map[string]*C2NetSequenceInfo @@ -1482,9 +1490,6 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) - ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) - sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") @@ -1684,6 +1689,16 @@ func getModelartsCDConfig() { getNotebookFlavorInfos() } +func getClearStrategy(){ + + sec := Cfg.Section("clear_strategy") + ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) + ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) + ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) + ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) + ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") +} + func getGrampusConfig() { sec := Cfg.Section("grampus") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 52e5c5cf1..f8dfae30e 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -14,7 +14,7 @@ import ( func ClearCloudbrainResultSpace() { - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,17 +32,20 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 files, err := ioutil.ReadDir(setting.JobPath) + processCount:=0 if err != nil { log.Warn("Can not browser local job path.") } else { SortModTimeAscend(files) for _, file := range files { - //清理4个月前的任务 - if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { - //os.RemoveAll(setting.JobPath + file.Name()) - } else { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + } + if processCount==setting.ClearStrategy.ResultBatchSize{ break } } @@ -50,20 +53,22 @@ func ClearCloudbrainResultSpace() { } minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - + processCount=0 if err != nil { log.Warn("Can not browser minio job path.") } else { SortModTimeAscendForMinio(minioFiles) for _, file := range minioFiles { - //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { dirPath := setting.CBCodePathPrefix + file.FileName + "/" log.Info(dirPath) - //storage.Attachments.DeleteDir(dirPath) - } else { - break + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount==setting.ClearStrategy.ResultBatchSize{ + break + } } }