You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

object.go 22 kB

11 months ago
11 months ago
2 years ago
11 months ago
11 months ago
11 months ago
11 months ago
2 years ago
11 months ago
11 months ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. package mq
  2. import (
  3. "errors"
  4. "fmt"
  5. "time"
  6. "gitlink.org.cn/cloudream/storage/common/pkgs/db2"
  7. "gitlink.org.cn/cloudream/storage/common/pkgs/db2/model"
  8. "gorm.io/gorm"
  9. "github.com/samber/lo"
  10. "gitlink.org.cn/cloudream/common/consts/errorcode"
  11. "gitlink.org.cn/cloudream/common/pkgs/logger"
  12. "gitlink.org.cn/cloudream/common/pkgs/mq"
  13. cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
  14. "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
  15. "gitlink.org.cn/cloudream/common/utils/sort2"
  16. stgmod "gitlink.org.cn/cloudream/storage/common/models"
  17. coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
  18. )
  19. func (svc *Service) GetObjects(msg *coormq.GetObjects) (*coormq.GetObjectsResp, *mq.CodeMessage) {
  20. var ret []*cdssdk.Object
  21. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  22. // TODO 应该检查用户是否有每一个Object所在Package的权限
  23. objs, err := svc.db2.Object().BatchGet(tx, msg.ObjectIDs)
  24. if err != nil {
  25. return err
  26. }
  27. objMp := make(map[cdssdk.ObjectID]cdssdk.Object)
  28. for _, obj := range objs {
  29. objMp[obj.ObjectID] = obj
  30. }
  31. for _, objID := range msg.ObjectIDs {
  32. o, ok := objMp[objID]
  33. if ok {
  34. ret = append(ret, &o)
  35. } else {
  36. ret = append(ret, nil)
  37. }
  38. }
  39. return err
  40. })
  41. if err != nil {
  42. logger.WithField("UserID", msg.UserID).
  43. Warn(err.Error())
  44. return nil, mq.Failed(errorcode.OperationFailed, "get objects failed")
  45. }
  46. return mq.ReplyOK(coormq.RespGetObjects(ret))
  47. }
  48. func (svc *Service) GetObjectsByPath(msg *coormq.GetObjectsByPath) (*coormq.GetObjectsByPathResp, *mq.CodeMessage) {
  49. var objs []cdssdk.Object
  50. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  51. var err error
  52. _, err = svc.db2.Package().GetUserPackage(tx, msg.UserID, msg.PackageID)
  53. if err != nil {
  54. return fmt.Errorf("getting package by id: %w", err)
  55. }
  56. if msg.IsPrefix {
  57. objs, err = svc.db2.Object().GetWithPathPrefix(tx, msg.PackageID, msg.Path)
  58. if err != nil {
  59. return fmt.Errorf("getting objects with prefix: %w", err)
  60. }
  61. } else {
  62. objs, err = svc.db2.Object().GetByPath(tx, msg.PackageID, msg.Path)
  63. if err != nil {
  64. return fmt.Errorf("getting object by path: %w", err)
  65. }
  66. }
  67. return nil
  68. })
  69. if err != nil {
  70. logger.WithField("PathPrefix", msg.Path).Warn(err.Error())
  71. return nil, mq.Failed(errorcode.OperationFailed, "get objects with prefix failed")
  72. }
  73. return mq.ReplyOK(coormq.RespGetObjectsByPath(objs))
  74. }
  75. func (svc *Service) GetPackageObjects(msg *coormq.GetPackageObjects) (*coormq.GetPackageObjectsResp, *mq.CodeMessage) {
  76. var objs []cdssdk.Object
  77. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  78. _, err := svc.db2.Package().GetUserPackage(tx, msg.UserID, msg.PackageID)
  79. if err != nil {
  80. return fmt.Errorf("getting package by id: %w", err)
  81. }
  82. objs, err = svc.db2.Object().GetPackageObjects(tx, msg.PackageID)
  83. if err != nil {
  84. return fmt.Errorf("getting package objects: %w", err)
  85. }
  86. return nil
  87. })
  88. if err != nil {
  89. logger.WithField("UserID", msg.UserID).WithField("PackageID", msg.PackageID).
  90. Warn(err.Error())
  91. return nil, mq.Failed(errorcode.OperationFailed, "get package objects failed")
  92. }
  93. return mq.ReplyOK(coormq.RespGetPackageObjects(objs))
  94. }
  95. func (svc *Service) GetPackageObjectDetails(msg *coormq.GetPackageObjectDetails) (*coormq.GetPackageObjectDetailsResp, *mq.CodeMessage) {
  96. var details []stgmod.ObjectDetail
  97. // 必须放在事务里进行,因为GetPackageBlockDetails是由多次数据库操作组成,必须保证数据的一致性
  98. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  99. var err error
  100. _, err = svc.db2.Package().GetByID(tx, msg.PackageID)
  101. if err != nil {
  102. return fmt.Errorf("getting package by id: %w", err)
  103. }
  104. details, err = svc.db2.Object().GetPackageObjectDetails(tx, msg.PackageID)
  105. if err != nil {
  106. return fmt.Errorf("getting package block details: %w", err)
  107. }
  108. return nil
  109. })
  110. if err != nil {
  111. logger.WithField("PackageID", msg.PackageID).Warn(err.Error())
  112. return nil, mq.Failed(errorcode.OperationFailed, "get package object block details failed")
  113. }
  114. return mq.ReplyOK(coormq.RespPackageObjectDetails(details))
  115. }
  116. func (svc *Service) GetObjectDetails(msg *coormq.GetObjectDetails) (*coormq.GetObjectDetailsResp, *mq.CodeMessage) {
  117. detailsMp := make(map[cdssdk.ObjectID]*stgmod.ObjectDetail)
  118. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  119. var err error
  120. msg.ObjectIDs = sort2.SortAsc(msg.ObjectIDs)
  121. // 根据ID依次查询Object,ObjectBlock,PinnedObject,并根据升序的特点进行合并
  122. objs, err := svc.db2.Object().BatchGet(tx, msg.ObjectIDs)
  123. if err != nil {
  124. return fmt.Errorf("batch get objects: %w", err)
  125. }
  126. for _, obj := range objs {
  127. detailsMp[obj.ObjectID] = &stgmod.ObjectDetail{
  128. Object: obj,
  129. }
  130. }
  131. // 查询合并
  132. blocks, err := svc.db2.ObjectBlock().BatchGetByObjectID(tx, msg.ObjectIDs)
  133. if err != nil {
  134. return fmt.Errorf("batch get object blocks: %w", err)
  135. }
  136. for _, block := range blocks {
  137. d := detailsMp[block.ObjectID]
  138. d.Blocks = append(d.Blocks, block)
  139. }
  140. // 查询合并
  141. pinneds, err := svc.db2.PinnedObject().BatchGetByObjectID(tx, msg.ObjectIDs)
  142. if err != nil {
  143. return fmt.Errorf("batch get pinned objects: %w", err)
  144. }
  145. for _, pinned := range pinneds {
  146. d := detailsMp[pinned.ObjectID]
  147. d.PinnedAt = append(d.PinnedAt, pinned.StorageID)
  148. }
  149. return nil
  150. })
  151. if err != nil {
  152. logger.Warn(err.Error())
  153. return nil, mq.Failed(errorcode.OperationFailed, "get object details failed")
  154. }
  155. details := make([]*stgmod.ObjectDetail, len(msg.ObjectIDs))
  156. for i, objID := range msg.ObjectIDs {
  157. details[i] = detailsMp[objID]
  158. }
  159. return mq.ReplyOK(coormq.RespGetObjectDetails(details))
  160. }
  161. func (svc *Service) UpdateObjectRedundancy(msg *coormq.UpdateObjectRedundancy) (*coormq.UpdateObjectRedundancyResp, *mq.CodeMessage) {
  162. err := svc.db2.DoTx(func(ctx db2.SQLContext) error {
  163. db := svc.db2
  164. objs := msg.Updatings
  165. nowTime := time.Now()
  166. objIDs := make([]cdssdk.ObjectID, 0, len(objs))
  167. for _, obj := range objs {
  168. objIDs = append(objIDs, obj.ObjectID)
  169. }
  170. avaiIDs, err := db.Object().BatchTestObjectID(ctx, objIDs)
  171. if err != nil {
  172. return fmt.Errorf("batch test object id: %w", err)
  173. }
  174. // 过滤掉已经不存在的对象。
  175. // 注意,objIDs没有被过滤,因为后续逻辑不过滤也不会出错
  176. objs = lo.Filter(objs, func(obj coormq.UpdatingObjectRedundancy, _ int) bool {
  177. return avaiIDs[obj.ObjectID]
  178. })
  179. dummyObjs := make([]cdssdk.Object, 0, len(objs))
  180. for _, obj := range objs {
  181. dummyObjs = append(dummyObjs, cdssdk.Object{
  182. ObjectID: obj.ObjectID,
  183. Redundancy: obj.Redundancy,
  184. CreateTime: nowTime, // 实际不会更新,只因为不能是0值
  185. UpdateTime: nowTime,
  186. })
  187. }
  188. err = db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"Redundancy", "UpdateTime"})
  189. if err != nil {
  190. return fmt.Errorf("batch update object redundancy: %w", err)
  191. }
  192. // 删除原本所有的编码块记录,重新添加
  193. err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objIDs)
  194. if err != nil {
  195. return fmt.Errorf("batch delete object blocks: %w", err)
  196. }
  197. // 删除原本Pin住的Object。暂不考虑FileHash没有变化的情况
  198. err = db.PinnedObject().BatchDeleteByObjectID(ctx, objIDs)
  199. if err != nil {
  200. return fmt.Errorf("batch delete pinned object: %w", err)
  201. }
  202. blocks := make([]stgmod.ObjectBlock, 0, len(objs))
  203. for _, obj := range objs {
  204. blocks = append(blocks, obj.Blocks...)
  205. }
  206. err = db.ObjectBlock().BatchCreate(ctx, blocks)
  207. if err != nil {
  208. return fmt.Errorf("batch create object blocks: %w", err)
  209. }
  210. caches := make([]model.Cache, 0, len(objs))
  211. for _, obj := range objs {
  212. for _, blk := range obj.Blocks {
  213. caches = append(caches, model.Cache{
  214. FileHash: blk.FileHash,
  215. StorageID: blk.StorageID,
  216. CreateTime: nowTime,
  217. Priority: 0,
  218. })
  219. }
  220. }
  221. err = db.Cache().BatchCreate(ctx, caches)
  222. if err != nil {
  223. return fmt.Errorf("batch create object caches: %w", err)
  224. }
  225. pinneds := make([]cdssdk.PinnedObject, 0, len(objs))
  226. for _, obj := range objs {
  227. for _, p := range obj.PinnedAt {
  228. pinneds = append(pinneds, cdssdk.PinnedObject{
  229. ObjectID: obj.ObjectID,
  230. StorageID: p,
  231. CreateTime: nowTime,
  232. })
  233. }
  234. }
  235. err = db.PinnedObject().BatchTryCreate(ctx, pinneds)
  236. if err != nil {
  237. return fmt.Errorf("batch create pinned objects: %w", err)
  238. }
  239. return nil
  240. })
  241. if err != nil {
  242. logger.Warnf("batch updating redundancy: %s", err.Error())
  243. return nil, mq.Failed(errorcode.OperationFailed, "batch update redundancy failed")
  244. }
  245. return mq.ReplyOK(coormq.RespUpdateObjectRedundancy())
  246. }
  247. func (svc *Service) UpdateObjectInfos(msg *coormq.UpdateObjectInfos) (*coormq.UpdateObjectInfosResp, *mq.CodeMessage) {
  248. var sucs []cdssdk.ObjectID
  249. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  250. msg.Updatings = sort2.Sort(msg.Updatings, func(o1, o2 cdsapi.UpdatingObject) int {
  251. return sort2.Cmp(o1.ObjectID, o2.ObjectID)
  252. })
  253. objIDs := make([]cdssdk.ObjectID, len(msg.Updatings))
  254. for i, obj := range msg.Updatings {
  255. objIDs[i] = obj.ObjectID
  256. }
  257. oldObjs, err := svc.db2.Object().BatchGet(tx, objIDs)
  258. if err != nil {
  259. return fmt.Errorf("batch getting objects: %w", err)
  260. }
  261. oldObjIDs := make([]cdssdk.ObjectID, len(oldObjs))
  262. for i, obj := range oldObjs {
  263. oldObjIDs[i] = obj.ObjectID
  264. }
  265. avaiUpdatings, notExistsObjs := pickByObjectIDs(msg.Updatings, oldObjIDs, func(obj cdsapi.UpdatingObject) cdssdk.ObjectID { return obj.ObjectID })
  266. if len(notExistsObjs) > 0 {
  267. // TODO 部分对象已经不存在
  268. }
  269. newObjs := make([]cdssdk.Object, len(avaiUpdatings))
  270. for i := range newObjs {
  271. newObjs[i] = oldObjs[i]
  272. avaiUpdatings[i].ApplyTo(&newObjs[i])
  273. }
  274. err = svc.db2.Object().BatchUpdate(tx, newObjs)
  275. if err != nil {
  276. return fmt.Errorf("batch create or update: %w", err)
  277. }
  278. sucs = lo.Map(newObjs, func(obj cdssdk.Object, _ int) cdssdk.ObjectID { return obj.ObjectID })
  279. return nil
  280. })
  281. if err != nil {
  282. logger.Warnf("batch updating objects: %s", err.Error())
  283. return nil, mq.Failed(errorcode.OperationFailed, "batch update objects failed")
  284. }
  285. return mq.ReplyOK(coormq.RespUpdateObjectInfos(sucs))
  286. }
  287. // 根据objIDs从objs中挑选Object。
  288. // len(objs) >= len(objIDs)
  289. func pickByObjectIDs[T any](objs []T, objIDs []cdssdk.ObjectID, getID func(T) cdssdk.ObjectID) (picked []T, notFound []T) {
  290. objIdx := 0
  291. idIdx := 0
  292. for idIdx < len(objIDs) && objIdx < len(objs) {
  293. if getID(objs[objIdx]) < objIDs[idIdx] {
  294. notFound = append(notFound, objs[objIdx])
  295. objIdx++
  296. continue
  297. }
  298. picked = append(picked, objs[objIdx])
  299. objIdx++
  300. idIdx++
  301. }
  302. return
  303. }
  304. func (svc *Service) MoveObjects(msg *coormq.MoveObjects) (*coormq.MoveObjectsResp, *mq.CodeMessage) {
  305. var sucs []cdssdk.ObjectID
  306. var evt []*stgmod.BodyObjectInfoUpdated
  307. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  308. msg.Movings = sort2.Sort(msg.Movings, func(o1, o2 cdsapi.MovingObject) int {
  309. return sort2.Cmp(o1.ObjectID, o2.ObjectID)
  310. })
  311. objIDs := make([]cdssdk.ObjectID, len(msg.Movings))
  312. for i, obj := range msg.Movings {
  313. objIDs[i] = obj.ObjectID
  314. }
  315. oldObjs, err := svc.db2.Object().BatchGet(tx, objIDs)
  316. if err != nil {
  317. return fmt.Errorf("batch getting objects: %w", err)
  318. }
  319. oldObjIDs := make([]cdssdk.ObjectID, len(oldObjs))
  320. for i, obj := range oldObjs {
  321. oldObjIDs[i] = obj.ObjectID
  322. }
  323. // 找出仍在数据库的Object
  324. avaiMovings, notExistsObjs := pickByObjectIDs(msg.Movings, oldObjIDs, func(obj cdsapi.MovingObject) cdssdk.ObjectID { return obj.ObjectID })
  325. if len(notExistsObjs) > 0 {
  326. // TODO 部分对象已经不存在
  327. }
  328. // 筛选出PackageID变化、Path变化的对象,这两种对象要检测改变后是否有冲突
  329. var pkgIDChangedObjs []cdssdk.Object
  330. var pathChangedObjs []cdssdk.Object
  331. for i := range avaiMovings {
  332. if avaiMovings[i].PackageID != oldObjs[i].PackageID {
  333. newObj := oldObjs[i]
  334. avaiMovings[i].ApplyTo(&newObj)
  335. pkgIDChangedObjs = append(pkgIDChangedObjs, newObj)
  336. } else if avaiMovings[i].Path != oldObjs[i].Path {
  337. newObj := oldObjs[i]
  338. avaiMovings[i].ApplyTo(&newObj)
  339. pathChangedObjs = append(pathChangedObjs, newObj)
  340. }
  341. }
  342. var newObjs []cdssdk.Object
  343. // 对于PackageID发生变化的对象,需要检查目标Package内是否存在同Path的对象
  344. checkedObjs, err := svc.checkPackageChangedObjects(tx, msg.UserID, pkgIDChangedObjs)
  345. if err != nil {
  346. return err
  347. }
  348. newObjs = append(newObjs, checkedObjs...)
  349. // 对于只有Path发生变化的对象,则检查同Package内有没有同Path的对象
  350. checkedObjs, err = svc.checkPathChangedObjects(tx, msg.UserID, pathChangedObjs)
  351. if err != nil {
  352. return err
  353. }
  354. newObjs = append(newObjs, checkedObjs...)
  355. err = svc.db2.Object().BatchUpdate(tx, newObjs)
  356. if err != nil {
  357. return fmt.Errorf("batch create or update: %w", err)
  358. }
  359. sucs = lo.Map(newObjs, func(obj cdssdk.Object, _ int) cdssdk.ObjectID { return obj.ObjectID })
  360. evt = lo.Map(newObjs, func(obj cdssdk.Object, _ int) *stgmod.BodyObjectInfoUpdated {
  361. return &stgmod.BodyObjectInfoUpdated{
  362. Object: obj,
  363. }
  364. })
  365. return nil
  366. })
  367. if err != nil {
  368. logger.Warn(err.Error())
  369. return nil, mq.Failed(errorcode.OperationFailed, "move objects failed")
  370. }
  371. for _, e := range evt {
  372. svc.evtPub.Publish(e)
  373. }
  374. return mq.ReplyOK(coormq.RespMoveObjects(sucs))
  375. }
  376. func (svc *Service) checkPackageChangedObjects(tx db2.SQLContext, userID cdssdk.UserID, objs []cdssdk.Object) ([]cdssdk.Object, error) {
  377. if len(objs) == 0 {
  378. return nil, nil
  379. }
  380. type PackageObjects struct {
  381. PackageID cdssdk.PackageID
  382. ObjectByPath map[string]*cdssdk.Object
  383. }
  384. packages := make(map[cdssdk.PackageID]*PackageObjects)
  385. for _, obj := range objs {
  386. pkg, ok := packages[obj.PackageID]
  387. if !ok {
  388. pkg = &PackageObjects{
  389. PackageID: obj.PackageID,
  390. ObjectByPath: make(map[string]*cdssdk.Object),
  391. }
  392. packages[obj.PackageID] = pkg
  393. }
  394. if pkg.ObjectByPath[obj.Path] == nil {
  395. o := obj
  396. pkg.ObjectByPath[obj.Path] = &o
  397. } else {
  398. // TODO 有两个对象移动到同一个路径,有冲突
  399. }
  400. }
  401. var willUpdateObjs []cdssdk.Object
  402. for _, pkg := range packages {
  403. _, err := svc.db2.Package().GetUserPackage(tx, userID, pkg.PackageID)
  404. if errors.Is(err, gorm.ErrRecordNotFound) {
  405. continue
  406. }
  407. if err != nil {
  408. return nil, fmt.Errorf("getting user package by id: %w", err)
  409. }
  410. existsObjs, err := svc.db2.Object().BatchGetByPackagePath(tx, pkg.PackageID, lo.Keys(pkg.ObjectByPath))
  411. if err != nil {
  412. return nil, fmt.Errorf("batch getting objects by package path: %w", err)
  413. }
  414. // 标记冲突的对象
  415. for _, obj := range existsObjs {
  416. pkg.ObjectByPath[obj.Path] = nil
  417. // TODO 目标Package内有冲突的对象
  418. }
  419. for _, obj := range pkg.ObjectByPath {
  420. if obj == nil {
  421. continue
  422. }
  423. willUpdateObjs = append(willUpdateObjs, *obj)
  424. }
  425. }
  426. return willUpdateObjs, nil
  427. }
  428. func (svc *Service) checkPathChangedObjects(tx db2.SQLContext, userID cdssdk.UserID, objs []cdssdk.Object) ([]cdssdk.Object, error) {
  429. if len(objs) == 0 {
  430. return nil, nil
  431. }
  432. objByPath := make(map[string]*cdssdk.Object)
  433. for _, obj := range objs {
  434. if objByPath[obj.Path] == nil {
  435. o := obj
  436. objByPath[obj.Path] = &o
  437. } else {
  438. // TODO 有两个对象移动到同一个路径,有冲突
  439. }
  440. }
  441. _, err := svc.db2.Package().GetUserPackage(tx, userID, objs[0].PackageID)
  442. if errors.Is(err, gorm.ErrRecordNotFound) {
  443. return nil, nil
  444. }
  445. if err != nil {
  446. return nil, fmt.Errorf("getting user package by id: %w", err)
  447. }
  448. existsObjs, err := svc.db2.Object().BatchGetByPackagePath(tx, objs[0].PackageID, lo.Map(objs, func(obj cdssdk.Object, idx int) string { return obj.Path }))
  449. if err != nil {
  450. return nil, fmt.Errorf("batch getting objects by package path: %w", err)
  451. }
  452. // 不支持两个对象交换位置的情况,因为数据库不支持
  453. for _, obj := range existsObjs {
  454. objByPath[obj.Path] = nil
  455. }
  456. var willMoveObjs []cdssdk.Object
  457. for _, obj := range objByPath {
  458. if obj == nil {
  459. continue
  460. }
  461. willMoveObjs = append(willMoveObjs, *obj)
  462. }
  463. return willMoveObjs, nil
  464. }
  465. func (svc *Service) DeleteObjects(msg *coormq.DeleteObjects) (*coormq.DeleteObjectsResp, *mq.CodeMessage) {
  466. var sucs []cdssdk.ObjectID
  467. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  468. avaiIDs, err := svc.db2.Object().BatchTestObjectID(tx, msg.ObjectIDs)
  469. if err != nil {
  470. return fmt.Errorf("batch testing object id: %w", err)
  471. }
  472. sucs = lo.Keys(avaiIDs)
  473. err = svc.db2.Object().BatchDelete(tx, msg.ObjectIDs)
  474. if err != nil {
  475. return fmt.Errorf("batch deleting objects: %w", err)
  476. }
  477. err = svc.db2.ObjectBlock().BatchDeleteByObjectID(tx, msg.ObjectIDs)
  478. if err != nil {
  479. return fmt.Errorf("batch deleting object blocks: %w", err)
  480. }
  481. err = svc.db2.PinnedObject().BatchDeleteByObjectID(tx, msg.ObjectIDs)
  482. if err != nil {
  483. return fmt.Errorf("batch deleting pinned objects: %w", err)
  484. }
  485. err = svc.db2.ObjectAccessStat().BatchDeleteByObjectID(tx, msg.ObjectIDs)
  486. if err != nil {
  487. return fmt.Errorf("batch deleting object access stats: %w", err)
  488. }
  489. return nil
  490. })
  491. if err != nil {
  492. logger.Warnf("batch deleting objects: %s", err.Error())
  493. return nil, mq.Failed(errorcode.OperationFailed, "batch delete objects failed")
  494. }
  495. for _, objID := range sucs {
  496. svc.evtPub.Publish(&stgmod.BodyObjectDeleted{
  497. ObjectID: objID,
  498. })
  499. }
  500. return mq.ReplyOK(coormq.RespDeleteObjects(sucs))
  501. }
  502. func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjectsResp, *mq.CodeMessage) {
  503. type CloningObject struct {
  504. Cloning cdsapi.CloningObject
  505. OrgIndex int
  506. }
  507. type PackageClonings struct {
  508. PackageID cdssdk.PackageID
  509. Clonings map[string]CloningObject
  510. }
  511. var evt []*stgmod.BodyNewOrUpdateObject
  512. // TODO 要检查用户是否有Object、Package的权限
  513. clonings := make(map[cdssdk.PackageID]*PackageClonings)
  514. for i, cloning := range msg.Clonings {
  515. pkg, ok := clonings[cloning.NewPackageID]
  516. if !ok {
  517. pkg = &PackageClonings{
  518. PackageID: cloning.NewPackageID,
  519. Clonings: make(map[string]CloningObject),
  520. }
  521. clonings[cloning.NewPackageID] = pkg
  522. }
  523. pkg.Clonings[cloning.NewPath] = CloningObject{
  524. Cloning: cloning,
  525. OrgIndex: i,
  526. }
  527. }
  528. ret := make([]*cdssdk.Object, len(msg.Clonings))
  529. err := svc.db2.DoTx(func(tx db2.SQLContext) error {
  530. // 剔除掉新路径已经存在的对象
  531. for _, pkg := range clonings {
  532. exists, err := svc.db2.Object().BatchGetByPackagePath(tx, pkg.PackageID, lo.Keys(pkg.Clonings))
  533. if err != nil {
  534. return fmt.Errorf("batch getting objects by package path: %w", err)
  535. }
  536. for _, obj := range exists {
  537. delete(pkg.Clonings, obj.Path)
  538. }
  539. }
  540. // 删除目的Package不存在的对象
  541. newPkg, err := svc.db2.Package().BatchTestPackageID(tx, lo.Keys(clonings))
  542. if err != nil {
  543. return fmt.Errorf("batch testing package id: %w", err)
  544. }
  545. for _, pkg := range clonings {
  546. if !newPkg[pkg.PackageID] {
  547. delete(clonings, pkg.PackageID)
  548. }
  549. }
  550. var avaiClonings []CloningObject
  551. var avaiObjIDs []cdssdk.ObjectID
  552. for _, pkg := range clonings {
  553. for _, cloning := range pkg.Clonings {
  554. avaiClonings = append(avaiClonings, cloning)
  555. avaiObjIDs = append(avaiObjIDs, cloning.Cloning.ObjectID)
  556. }
  557. }
  558. avaiDetails, err := svc.db2.Object().BatchGetDetails(tx, avaiObjIDs)
  559. if err != nil {
  560. return fmt.Errorf("batch getting object details: %w", err)
  561. }
  562. avaiDetailsMap := make(map[cdssdk.ObjectID]stgmod.ObjectDetail)
  563. for _, detail := range avaiDetails {
  564. avaiDetailsMap[detail.Object.ObjectID] = detail
  565. }
  566. oldAvaiClonings := avaiClonings
  567. avaiClonings = nil
  568. var newObjs []cdssdk.Object
  569. for _, cloning := range oldAvaiClonings {
  570. // 进一步剔除原始对象不存在的情况
  571. detail, ok := avaiDetailsMap[cloning.Cloning.ObjectID]
  572. if !ok {
  573. continue
  574. }
  575. avaiClonings = append(avaiClonings, cloning)
  576. newObj := detail.Object
  577. newObj.ObjectID = 0
  578. newObj.Path = cloning.Cloning.NewPath
  579. newObj.PackageID = cloning.Cloning.NewPackageID
  580. newObjs = append(newObjs, newObj)
  581. }
  582. // 先创建出新对象
  583. err = svc.db2.Object().BatchCreate(tx, &newObjs)
  584. if err != nil {
  585. return fmt.Errorf("batch creating objects: %w", err)
  586. }
  587. // 创建了新对象就能拿到新对象ID,再创建新对象块
  588. var newBlks []stgmod.ObjectBlock
  589. for i, cloning := range avaiClonings {
  590. oldBlks := avaiDetailsMap[cloning.Cloning.ObjectID].Blocks
  591. for _, blk := range oldBlks {
  592. newBlk := blk
  593. newBlk.ObjectID = newObjs[i].ObjectID
  594. newBlks = append(newBlks, newBlk)
  595. }
  596. }
  597. err = svc.db2.ObjectBlock().BatchCreate(tx, newBlks)
  598. if err != nil {
  599. return fmt.Errorf("batch creating object blocks: %w", err)
  600. }
  601. for i, cloning := range avaiClonings {
  602. ret[cloning.OrgIndex] = &newObjs[i]
  603. }
  604. for i, cloning := range avaiClonings {
  605. var evtBlks []stgmod.BlockDistributionObjectInfo
  606. blkType := getBlockTypeFromRed(newObjs[i].Redundancy)
  607. oldBlks := avaiDetailsMap[cloning.Cloning.ObjectID].Blocks
  608. for _, blk := range oldBlks {
  609. evtBlks = append(evtBlks, stgmod.BlockDistributionObjectInfo{
  610. BlockType: blkType,
  611. Index: blk.Index,
  612. StorageID: blk.StorageID,
  613. })
  614. }
  615. evt = append(evt, &stgmod.BodyNewOrUpdateObject{
  616. Info: newObjs[i],
  617. BlockDistribution: evtBlks,
  618. })
  619. }
  620. return nil
  621. })
  622. if err != nil {
  623. logger.Warnf("cloning objects: %s", err.Error())
  624. return nil, mq.Failed(errorcode.OperationFailed, err.Error())
  625. }
  626. for _, e := range evt {
  627. svc.evtPub.Publish(e)
  628. }
  629. return mq.ReplyOK(coormq.RespCloneObjects(ret))
  630. }

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。