diff --git a/common/pkgs/storage/s3/base_store.go b/common/pkgs/storage/s3/base_store.go index 516ddc5..38ac555 100644 --- a/common/pkgs/storage/s3/base_store.go +++ b/common/pkgs/storage/s3/base_store.go @@ -14,6 +14,7 @@ import ( s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/math2" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" ) @@ -173,24 +174,25 @@ func (s *BaseStore) CleanTemps() { marker = resp.NextMarker } - if len(deletes) == 0 { - return - } + for len(deletes) > 0 { + cnt := math2.Min(500, len(deletes)) + resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ + Bucket: aws.String(s.Bucket), + Delete: &s3types.Delete{ + Objects: deletes[:cnt], + }, + }) + if err != nil { + log.Warnf("delete temp files: %v", err) + return + } - resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ - Bucket: aws.String(s.Bucket), - Delete: &s3types.Delete{ - Objects: deletes, - }, - }) - if err != nil { - log.Warnf("delete temp files: %v", err) - return - } + for _, del := range resp.Deleted { + obj := deleteObjs[*del.Key] + log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified) + } - for _, del := range resp.Deleted { - obj := deleteObjs[*del.Key] - log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified) + deletes = deletes[cnt:] } } diff --git a/common/pkgs/storage/s3/shard_store.go b/common/pkgs/storage/s3/shard_store.go index 12e376f..f4d38f8 100644 --- a/common/pkgs/storage/s3/shard_store.go +++ b/common/pkgs/storage/s3/shard_store.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/utils/math2" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" ) @@ -181,12 +182,14 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error { marker = resp.NextMarker } - cnt := 0 - if len(deletes) > 0 { - resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ + totalCnt := len(deletes) + for len(deletes) > 0 { + cnt := math2.Min(500, len(deletes)) + + _, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ Bucket: aws.String(s.Bucket), Delete: &s3types.Delete{ - Objects: deletes, + Objects: deletes[:cnt], }, }) if err != nil { @@ -194,10 +197,10 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error { return err } - cnt = len(resp.Deleted) + deletes = deletes[cnt:] } - s.getLogger().Infof("purge %d files", cnt) + s.getLogger().Infof("purge %d files", totalCnt) // TODO 无法保证原子性,所以删除失败只打日志 return nil }