You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

create_package.go 4.1 kB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. package task
  2. /*
  3. import (
  4. "fmt"
  5. "path/filepath"
  6. "time"
  7. "github.com/samber/lo"
  8. "gitlink.org.cn/cloudream/common/pkgs/logger"
  9. "gitlink.org.cn/cloudream/common/pkgs/task"
  10. cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
  11. stgglb "gitlink.org.cn/cloudream/storage2/common/globals"
  12. "gitlink.org.cn/cloudream/storage2/common/pkgs/iterator"
  13. "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator"
  14. )
  15. // CreatePackageResult 定义创建包的结果结构
  16. // 包含包的ID和上传的对象列表
  17. type CreatePackageResult struct {
  18. PackageID cdssdk.PackageID
  19. Objects []cdssdk.Object
  20. }
  21. // CreatePackage 定义创建包的任务结构
  22. // 包含用户ID、存储桶ID、包名称、上传对象的迭代器、节点亲和性以及任务结果
  23. type CreatePackage struct {
  24. userID cdssdk.UserID
  25. bucketID cdssdk.BucketID
  26. name string
  27. objIter iterator.UploadingObjectIterator
  28. stgAffinity cdssdk.StorageID
  29. Result CreatePackageResult
  30. }
  31. // NewCreatePackage 创建一个新的CreatePackage实例
  32. // userID: 用户ID
  33. // bucketID: 存储桶ID
  34. // name: 包名称
  35. // objIter: 上传对象的迭代器
  36. // stgAffinity: 节点亲和性,指定包应该创建在哪个节点上(可选)
  37. // 返回CreatePackage实例的指针
  38. func NewCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, objIter iterator.UploadingObjectIterator, stgAffinity cdssdk.StorageID) *CreatePackage {
  39. return &CreatePackage{
  40. userID: userID,
  41. bucketID: bucketID,
  42. name: name,
  43. objIter: objIter,
  44. stgAffinity: stgAffinity,
  45. }
  46. }
  47. // Execute 执行创建包的任务
  48. // task: 任务实例,携带任务上下文
  49. // ctx: 任务上下文,包含分布式锁和网络连接性等信息
  50. // complete: 任务完成的回调函数
  51. func (t *CreatePackage) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
  52. // 获取任务日志记录器
  53. log := logger.WithType[CreatePackage]("Task")
  54. log.Debugf("begin")
  55. defer log.Debugf("end")
  56. // 从MQ池中获取协调器客户端
  57. coorCli, err := stgglb.CoordinatorMQPool.Acquire()
  58. if err != nil {
  59. err = fmt.Errorf("new coordinator client: %w", err)
  60. log.Warn(err.Error())
  61. // 完成任务并设置移除延迟
  62. complete(err, CompleteOption{
  63. RemovingDelay: time.Minute,
  64. })
  65. return
  66. }
  67. defer stgglb.CoordinatorMQPool.Release(coorCli)
  68. // 向协调器创建包
  69. createResp, err := coorCli.CreatePackage(coordinator.NewCreatePackage(t.userID, t.bucketID, t.name))
  70. if err != nil {
  71. err = fmt.Errorf("creating package: %w", err)
  72. log.Error(err.Error())
  73. // 完成任务并设置移除延迟
  74. complete(err, CompleteOption{
  75. RemovingDelay: time.Minute,
  76. })
  77. return
  78. }
  79. up, err := ctx.uploader.BeginUpdate(t.userID, createResp.Package.PackageID, t.stgAffinity, nil, nil)
  80. if err != nil {
  81. err = fmt.Errorf("begin update: %w", err)
  82. log.Error(err.Error())
  83. // 完成任务并设置移除延迟
  84. complete(err, CompleteOption{
  85. RemovingDelay: time.Minute,
  86. })
  87. return
  88. }
  89. defer up.Abort()
  90. for {
  91. obj, err := t.objIter.MoveNext()
  92. if err == iterator.ErrNoMoreItem {
  93. break
  94. }
  95. if err != nil {
  96. log.Error(err.Error())
  97. // 完成任务并设置移除延迟
  98. complete(err, CompleteOption{
  99. RemovingDelay: time.Minute,
  100. })
  101. return
  102. }
  103. path := filepath.ToSlash(obj.Path)
  104. // 上传对象
  105. err = up.Upload(path, obj.File)
  106. if err != nil {
  107. err = fmt.Errorf("uploading object: %w", err)
  108. log.Error(err.Error())
  109. // 完成任务并设置移除延迟
  110. complete(err, CompleteOption{
  111. RemovingDelay: time.Minute,
  112. })
  113. return
  114. }
  115. }
  116. // 结束上传
  117. uploadRet, err := up.Commit()
  118. if err != nil {
  119. err = fmt.Errorf("uploading objects: %w", err)
  120. log.Error(err.Error())
  121. // 完成任务并设置移除延迟
  122. complete(err, CompleteOption{
  123. RemovingDelay: time.Minute,
  124. })
  125. return
  126. }
  127. t.Result.PackageID = createResp.Package.PackageID
  128. t.Result.Objects = lo.Values(uploadRet.Objects)
  129. // 完成任务并设置移除延迟
  130. complete(nil, CompleteOption{
  131. RemovingDelay: time.Minute,
  132. })
  133. }
  134. */

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。