From 85d03023bd7baaa53f3e8addad8368e758313585 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 2 Dec 2022 11:13:09 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cron/tasks_basic.go | 17 +++++- modules/setting/setting.go | 5 ++ services/cloudbrain/clear.go | 108 +++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 services/cloudbrain/clear.go diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 985a82cdb..9b74c1637 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,10 +5,12 @@ package cron import ( - "code.gitea.io/gitea/modules/urfs_client/urchin" "context" "time" + "code.gitea.io/gitea/modules/urfs_client/urchin" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward" @@ -190,6 +192,17 @@ func registerHandleRepoAndUserStatistic() { }) } +func registerHandleClearCloudbrainResult() { + RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "* 0,30 2-8 * * ? *", + }, func(ctx context.Context, _ *models.User, _ Config) error { + cloudbrainService.ClearCloudbrainResultSpace() + return nil + }) +} + func registerHandleSummaryStatistic() { RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ Enabled: true, @@ -317,6 +330,6 @@ func initBasicTasks() { registerHandleModelSafetyTask() - registerHandleScheduleRecord() + registerHandleScheduleRecord() registerHandleCloudbrainDurationStatistic() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a473dad6a..2b2ded678 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,6 +518,8 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string + ResultSaveDays int64 + ResultBatchSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,6 +1482,9 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) + ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) + sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go new file mode 100644 index 000000000..52e5c5cf1 --- /dev/null +++ b/services/cloudbrain/clear.go @@ -0,0 +1,108 @@ +package cloudbrain + +import ( + "io/ioutil" + "os" + "sort" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +func ClearCloudbrainResultSpace() { + + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + + if err != nil { + log.Warn("Failed to get cloudbrain, clear result failed.", err) + return + } + var ids []int64 + for _, task := range tasks { + err := DeleteCloudbrainOneJobStorage(task.JobName) + if err == nil { + ids = append(ids, task.ID) + } + } + + err = models.UpdateCloudBrainRecordsCleared(ids) + if err != nil { + log.Warn("Failed to set cloudbrain cleared status", err) + } + if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + files, err := ioutil.ReadDir(setting.JobPath) + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理4个月前的任务 + if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { + //os.RemoveAll(setting.JobPath + file.Name()) + } else { + break + } + } + + } + + minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) + + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscendForMinio(minioFiles) + for _, file := range minioFiles { + //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) + if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + dirPath := setting.CBCodePathPrefix + file.FileName + "/" + log.Info(dirPath) + //storage.Attachments.DeleteDir(dirPath) + } else { + break + } + } + + } + + } + +} + +func SortModTimeAscend(files []os.FileInfo) { + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime().Before(files[j].ModTime()) + }) +} +func SortModTimeAscendForMinio(files []storage.FileInfo) { + sort.Slice(files, func(i, j int) bool { + timeI, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + timeJ, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + return timeI.Before(timeJ) + }) +} + +func DeleteCloudbrainOneJobStorage(jobName string) error { + //delete local + localJobPath := setting.JobPath + jobName + err := os.RemoveAll(localJobPath) + if err != nil { + log.Error("RemoveAll(%s) failed:%v", localJobPath, err) + } + + dirPath := setting.CBCodePathPrefix + jobName + "/" + err1 := storage.Attachments.DeleteDir(dirPath) + + if err1 != nil { + log.Error("DeleteDir(%s) failed:%v", localJobPath, err) + } + if err == nil { + err = err1 + } + + return err +}