S3分批清理无用文件

common/pkgs/storage/s3/base_store.go

@@ -14,6 +14,7 @@ import (
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/utils/io2"
+	"gitlink.org.cn/cloudream/common/utils/math2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
@@ -173,24 +174,25 @@ func (s *BaseStore) CleanTemps() {
 		marker = resp.NextMarker
 	}
 
-	if len(deletes) == 0 {
-		return
-	}
+	for len(deletes) > 0 {
+		cnt := math2.Min(500, len(deletes))
+		resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
+			Bucket: aws.String(s.Bucket),
+			Delete: &s3types.Delete{
+				Objects: deletes[:cnt],
+			},
+		})
+		if err != nil {
+			log.Warnf("delete temp files: %v", err)
+			return
+		}
 
-	resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
-		Bucket: aws.String(s.Bucket),
-		Delete: &s3types.Delete{
-			Objects: deletes,
-		},
-	})
-	if err != nil {
-		log.Warnf("delete temp files: %v", err)
-		return
-	}
+		for _, del := range resp.Deleted {
+			obj := deleteObjs[*del.Key]
+			log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
+		}
 
-	for _, del := range resp.Deleted {
-		obj := deleteObjs[*del.Key]
-		log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
+		deletes = deletes[cnt:]
 	}
 }
 

common/pkgs/storage/s3/shard_store.go

@@ -8,6 +8,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/utils/math2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
@@ -181,12 +182,14 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
 		marker = resp.NextMarker
 	}
 
-	cnt := 0
-	if len(deletes) > 0 {
-		resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
+	totalCnt := len(deletes)
+	for len(deletes) > 0 {
+		cnt := math2.Min(500, len(deletes))
+
+		_, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
 			Bucket: aws.String(s.Bucket),
 			Delete: &s3types.Delete{
-				Objects: deletes,
+				Objects: deletes[:cnt],
 			},
 		})
 		if err != nil {
@@ -194,10 +197,10 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
 			return err
 		}
 
-		cnt = len(resp.Deleted)
+		deletes = deletes[cnt:]
 	}
 
-	s.getLogger().Infof("purge %d files", cnt)
+	s.getLogger().Infof("purge %d files", totalCnt)
 	// TODO 无法保证原子性，所以删除失败只打日志
 	return nil
 }