提交代码

models/cloudbrain.go

@@ -2050,9 +2050,9 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) {
 		Find(&cloudbrains)
 }
 
-func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) {
+func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
 	cloudbrains := make([]*Cloudbrain, 0, 10)
-	endTimeBefore := time.Now().Unix() - days*24*3600
+	endTimeBefore := time.Now().Unix() - int64(days)*24*3600
 	missEndTimeBefore := endTimeBefore - 24*3600
 	return cloudbrains, x.Cols("id,job_name,job_id").
 		In("status",

modules/cron/tasks_basic.go

@@ -5,6 +5,7 @@
 package cron
 
 import (
+	"code.gitea.io/gitea/modules/setting"
 	"context"
 	"time"
 
@@ -196,7 +197,7 @@ func registerHandleClearCloudbrainResult() {
 	RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{
 		Enabled:    true,
 		RunAtStart: false,
-		Schedule:   "* 0,30 2-8 * * ? *",
+		Schedule:   setting.ClearStrategy.Cron,
 	}, func(ctx context.Context, _ *models.User, _ Config) error {
 		cloudbrainService.ClearCloudbrainResultSpace()
 		return nil
@@ -319,6 +320,7 @@ func initBasicTasks() {
 
 	registerHandleRepoAndUserStatistic()
 	registerHandleSummaryStatistic()
+	registerHandleClearCloudbrainResult()
 
 	registerSyncCloudbrainStatus()
 	registerHandleOrgStatistic()

modules/setting/setting.go

@@ -518,8 +518,7 @@ var (
 	MaxDatasetNum          int
 	CullIdleTimeout        string
 	CullInterval           string
-	ResultSaveDays         int64
-	ResultBatchSize        int
+
 
 	//benchmark config
 	IsBenchmarkEnabled     bool
@@ -615,6 +614,15 @@ var (
 		UsageRateBeginTime      string
 	}{}
 
+	ClearStrategy= struct {
+		Enabled bool
+		ResultSaveDays         int
+		ResultBatchSize        int
+		BatchSize int
+		TrashSaveDays int
+		Cron string
+	}{}
+
 	C2NetInfos                 *C2NetSqInfos
 	CenterInfos                *AiCenterInfos
 	C2NetMapInfo               map[string]*C2NetSequenceInfo
@@ -1482,9 +1490,6 @@ func NewContext() {
 	CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900")
 	CullInterval = sec.Key("CULL_INTERVAL").MustString("60")
 
-	ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30)
-	ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500)
-
 	sec = Cfg.Section("benchmark")
 	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
 	BenchmarkOwner = sec.Key("OWNER").MustString("")
@@ -1684,6 +1689,16 @@ func getModelartsCDConfig() {
 	getNotebookFlavorInfos()
 }
 
+func getClearStrategy(){
+
+	sec := Cfg.Section("clear_strategy")
+    ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false)
+	ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30)
+	ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500)
+	ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90)
+	ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *")
+}
+
 func getGrampusConfig() {
 	sec := Cfg.Section("grampus")
 

services/cloudbrain/clear.go

@@ -14,7 +14,7 @@ import (
 
 func ClearCloudbrainResultSpace() {
 
-	tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize)
+	tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize)
 
 	if err != nil {
 		log.Warn("Failed to get cloudbrain, clear result failed.", err)
@@ -32,17 +32,20 @@ func ClearCloudbrainResultSpace() {
 	if err != nil {
 		log.Warn("Failed to set cloudbrain cleared status", err)
 	}
-	if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在
+	if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在
 		files, err := ioutil.ReadDir(setting.JobPath)
+		processCount:=0
 		if err != nil {
 			log.Warn("Can not browser local job path.")
 		} else {
 			SortModTimeAscend(files)
 			for _, file := range files {
-				//清理4个月前的任务
-				if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) {
-					//os.RemoveAll(setting.JobPath + file.Name())
-				} else {
+				//清理n天前的历史垃圾数据，清理job目录
+				if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
+					os.RemoveAll(setting.JobPath + file.Name())
+					processCount++
+				}
+				if processCount==setting.ClearStrategy.ResultBatchSize{
 					break
 				}
 			}
@@ -50,20 +53,22 @@ func ClearCloudbrainResultSpace() {
 		}
 
 		minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix)
-
+		processCount=0
 		if err != nil {
 			log.Warn("Can not browser minio job path.")
 		} else {
 			SortModTimeAscendForMinio(minioFiles)
 			for _, file := range minioFiles {
-				//清理4个月前的任务
+
 				timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime)
-				if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) {
+				if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
 					dirPath := setting.CBCodePathPrefix + file.FileName + "/"
 					log.Info(dirPath)
-					//storage.Attachments.DeleteDir(dirPath)
-				} else {
-					break
+					storage.Attachments.DeleteDir(dirPath)
+					processCount++
+					if processCount==setting.ClearStrategy.ResultBatchSize{
+						break
+					}
 				}
 			}