提交代码

modules/cron/tasks_basic.go

@@ -5,10 +5,12 @@
 package cron
 
 import (
-	"code.gitea.io/gitea/modules/urfs_client/urchin"
 	"context"
 	"time"
 
+	"code.gitea.io/gitea/modules/urfs_client/urchin"
+	cloudbrainService "code.gitea.io/gitea/services/cloudbrain"
+
 	"code.gitea.io/gitea/modules/modelarts"
 	"code.gitea.io/gitea/services/cloudbrain/resource"
 	"code.gitea.io/gitea/services/reward"
@@ -190,6 +192,17 @@ func registerHandleRepoAndUserStatistic() {
 	})
 }
 
+func registerHandleClearCloudbrainResult() {
+	RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "* 0,30 2-8 * * ? *",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		cloudbrainService.ClearCloudbrainResultSpace()
+		return nil
+	})
+}
+
 func registerHandleSummaryStatistic() {
 	RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
 		Enabled:    true,
@@ -317,6 +330,6 @@ func initBasicTasks() {
 
 	registerHandleModelSafetyTask()
 
-	registerHandleScheduleRecord()	
+	registerHandleScheduleRecord()
 	registerHandleCloudbrainDurationStatistic()
 }

modules/setting/setting.go

@@ -518,6 +518,8 @@ var (
 	MaxDatasetNum          int
 	CullIdleTimeout        string
 	CullInterval           string
+	ResultSaveDays         int64
+	ResultBatchSize        int
 
 	//benchmark config
 	IsBenchmarkEnabled     bool
@@ -1480,6 +1482,9 @@ func NewContext() {
 	CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900")
 	CullInterval = sec.Key("CULL_INTERVAL").MustString("60")
 
+	ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30)
+	ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500)
+
 	sec = Cfg.Section("benchmark")
 	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
 	BenchmarkOwner = sec.Key("OWNER").MustString("")

services/cloudbrain/clear.go

@@ -0,0 +1,108 @@
+package cloudbrain
+
+import (
+	"io/ioutil"
+	"os"
+	"sort"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+)
+
+func ClearCloudbrainResultSpace() {
+
+	tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize)
+
+	if err != nil {
+		log.Warn("Failed to get cloudbrain, clear result failed.", err)
+		return
+	}
+	var ids []int64
+	for _, task := range tasks {
+		err := DeleteCloudbrainOneJobStorage(task.JobName)
+		if err == nil {
+			ids = append(ids, task.ID)
+		}
+	}
+
+	err = models.UpdateCloudBrainRecordsCleared(ids)
+	if err != nil {
+		log.Warn("Failed to set cloudbrain cleared status", err)
+	}
+	if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在
+		files, err := ioutil.ReadDir(setting.JobPath)
+		if err != nil {
+			log.Warn("Can not browser local job path.")
+		} else {
+			SortModTimeAscend(files)
+			for _, file := range files {
+				//清理4个月前的任务
+				if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) {
+					//os.RemoveAll(setting.JobPath + file.Name())
+				} else {
+					break
+				}
+			}
+
+		}
+
+		minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix)
+
+		if err != nil {
+			log.Warn("Can not browser minio job path.")
+		} else {
+			SortModTimeAscendForMinio(minioFiles)
+			for _, file := range minioFiles {
+				//清理4个月前的任务
+				timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime)
+				if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) {
+					dirPath := setting.CBCodePathPrefix + file.FileName + "/"
+					log.Info(dirPath)
+					//storage.Attachments.DeleteDir(dirPath)
+				} else {
+					break
+				}
+			}
+
+		}
+
+	}
+
+}
+
+func SortModTimeAscend(files []os.FileInfo) {
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].ModTime().Before(files[j].ModTime())
+	})
+}
+func SortModTimeAscendForMinio(files []storage.FileInfo) {
+	sort.Slice(files, func(i, j int) bool {
+		timeI, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime)
+		timeJ, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime)
+		return timeI.Before(timeJ)
+	})
+}
+
+func DeleteCloudbrainOneJobStorage(jobName string) error {
+	//delete local
+	localJobPath := setting.JobPath + jobName
+	err := os.RemoveAll(localJobPath)
+	if err != nil {
+		log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
+	}
+
+	dirPath := setting.CBCodePathPrefix + jobName + "/"
+	err1 := storage.Attachments.DeleteDir(dirPath)
+
+	if err1 != nil {
+		log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
+	}
+	if err == nil {
+		err = err1
+	}
+
+	return err
+}