You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

object.go 21 kB

7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
1 year ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
1 year ago
7 months ago
1 year ago
7 months ago
1 year ago
7 months ago
1 year ago
7 months ago
1 year ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768
  1. package services
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "time"
  7. "github.com/samber/lo"
  8. "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
  9. "gitlink.org.cn/cloudream/common/pkgs/logger"
  10. "gitlink.org.cn/cloudream/common/utils/sort2"
  11. "gitlink.org.cn/cloudream/storage2/client/internal/db"
  12. "gitlink.org.cn/cloudream/storage2/client/internal/downloader"
  13. "gitlink.org.cn/cloudream/storage2/client/sdk/api"
  14. "gitlink.org.cn/cloudream/storage2/client/types"
  15. "gitlink.org.cn/cloudream/storage2/common/models/datamap"
  16. "gitlink.org.cn/cloudream/storage2/common/pkgs/ioswitch2/ops2"
  17. "gitlink.org.cn/cloudream/storage2/common/pkgs/ioswitch2/plans"
  18. "gorm.io/gorm"
  19. )
  20. // ObjectService 定义了对象服务,负责管理对象的上传、下载等操作。
  21. type ObjectService struct {
  22. *Service
  23. }
  24. // ObjectSvc 返回一个ObjectService的实例。
  25. func (svc *Service) ObjectSvc() *ObjectService {
  26. return &ObjectService{Service: svc}
  27. }
  28. func (svc *ObjectService) GetByPath(req api.ObjectListByPath) (api.ObjectListByPathResp, error) {
  29. var resp api.ObjectListByPathResp
  30. maxKeys := 1000
  31. if req.MaxKeys > 0 {
  32. maxKeys = req.MaxKeys
  33. }
  34. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  35. var err error
  36. _, err = svc.DB.Package().GetByID(tx, req.PackageID)
  37. if err != nil {
  38. return fmt.Errorf("getting package by id: %w", err)
  39. }
  40. if !req.IsPrefix {
  41. obj, err := svc.DB.Object().GetByPath(tx, req.PackageID, req.Path)
  42. if err != nil {
  43. return fmt.Errorf("getting object by path: %w", err)
  44. }
  45. resp.Objects = append(resp.Objects, obj)
  46. return nil
  47. }
  48. if !req.NoRecursive {
  49. resp.Objects, err = svc.DB.Object().GetWithPathPrefixPaged(tx, req.PackageID, req.Path, req.ContinuationToken, maxKeys)
  50. if err != nil {
  51. return fmt.Errorf("getting objects with prefix: %w", err)
  52. }
  53. if len(resp.Objects) > 0 {
  54. resp.NextContinuationToken = resp.Objects[len(resp.Objects)-1].Path
  55. }
  56. return nil
  57. }
  58. resp.Objects, resp.CommonPrefixes, resp.NextContinuationToken, err = svc.DB.Object().GetByPrefixGroupedPaged(tx, req.PackageID, req.Path, req.ContinuationToken, maxKeys)
  59. return err
  60. })
  61. return resp, err
  62. }
  63. func (svc *ObjectService) GetByIDs(objectIDs []types.ObjectID) ([]*types.Object, error) {
  64. var ret []*types.Object
  65. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  66. // TODO 应该检查用户是否有每一个Object所在Package的权限
  67. objs, err := svc.DB.Object().BatchGet(tx, objectIDs)
  68. if err != nil {
  69. return err
  70. }
  71. objMp := make(map[types.ObjectID]types.Object)
  72. for _, obj := range objs {
  73. objMp[obj.ObjectID] = obj
  74. }
  75. for _, objID := range objectIDs {
  76. o, ok := objMp[objID]
  77. if ok {
  78. ret = append(ret, &o)
  79. } else {
  80. ret = append(ret, nil)
  81. }
  82. }
  83. return err
  84. })
  85. return ret, err
  86. }
  87. func (svc *ObjectService) UpdateInfo(updatings []api.UpdatingObject) ([]types.ObjectID, error) {
  88. var sucs []types.ObjectID
  89. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  90. updatings = sort2.Sort(updatings, func(o1, o2 api.UpdatingObject) int {
  91. return sort2.Cmp(o1.ObjectID, o2.ObjectID)
  92. })
  93. objIDs := make([]types.ObjectID, len(updatings))
  94. for i, obj := range updatings {
  95. objIDs[i] = obj.ObjectID
  96. }
  97. oldObjs, err := svc.DB.Object().BatchGet(tx, objIDs)
  98. if err != nil {
  99. return fmt.Errorf("batch getting objects: %w", err)
  100. }
  101. oldObjIDs := make([]types.ObjectID, len(oldObjs))
  102. for i, obj := range oldObjs {
  103. oldObjIDs[i] = obj.ObjectID
  104. }
  105. avaiUpdatings, notExistsObjs := pickByObjectIDs(updatings, oldObjIDs, func(obj api.UpdatingObject) types.ObjectID { return obj.ObjectID })
  106. if len(notExistsObjs) > 0 {
  107. // TODO 部分对象已经不存在
  108. }
  109. newObjs := make([]types.Object, len(avaiUpdatings))
  110. for i := range newObjs {
  111. newObjs[i] = oldObjs[i]
  112. avaiUpdatings[i].ApplyTo(&newObjs[i])
  113. }
  114. err = svc.DB.Object().BatchUpdate(tx, newObjs)
  115. if err != nil {
  116. return fmt.Errorf("batch create or update: %w", err)
  117. }
  118. sucs = lo.Map(newObjs, func(obj types.Object, _ int) types.ObjectID { return obj.ObjectID })
  119. return nil
  120. })
  121. return sucs, err
  122. }
  123. // 根据objIDs从objs中挑选Object。
  124. // len(objs) >= len(objIDs)
  125. func pickByObjectIDs[T any](objs []T, objIDs []types.ObjectID, getID func(T) types.ObjectID) (picked []T, notFound []T) {
  126. objIdx := 0
  127. idIdx := 0
  128. for idIdx < len(objIDs) && objIdx < len(objs) {
  129. if getID(objs[objIdx]) < objIDs[idIdx] {
  130. notFound = append(notFound, objs[objIdx])
  131. objIdx++
  132. continue
  133. }
  134. picked = append(picked, objs[objIdx])
  135. objIdx++
  136. idIdx++
  137. }
  138. return
  139. }
  140. func (svc *ObjectService) Move(movings []api.MovingObject) ([]types.ObjectID, error) {
  141. var sucs []types.ObjectID
  142. var evt []*datamap.BodyObjectInfoUpdated
  143. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  144. movings = sort2.Sort(movings, func(o1, o2 api.MovingObject) int {
  145. return sort2.Cmp(o1.ObjectID, o2.ObjectID)
  146. })
  147. objIDs := make([]types.ObjectID, len(movings))
  148. for i, obj := range movings {
  149. objIDs[i] = obj.ObjectID
  150. }
  151. oldObjs, err := svc.DB.Object().BatchGet(tx, objIDs)
  152. if err != nil {
  153. return fmt.Errorf("batch getting objects: %w", err)
  154. }
  155. oldObjIDs := make([]types.ObjectID, len(oldObjs))
  156. for i, obj := range oldObjs {
  157. oldObjIDs[i] = obj.ObjectID
  158. }
  159. // 找出仍在数据库的Object
  160. avaiMovings, notExistsObjs := pickByObjectIDs(movings, oldObjIDs, func(obj api.MovingObject) types.ObjectID { return obj.ObjectID })
  161. if len(notExistsObjs) > 0 {
  162. // TODO 部分对象已经不存在
  163. }
  164. // 筛选出PackageID变化、Path变化的对象,这两种对象要检测改变后是否有冲突
  165. var pkgIDChangedObjs []types.Object
  166. var pathChangedObjs []types.Object
  167. for i := range avaiMovings {
  168. if avaiMovings[i].PackageID != oldObjs[i].PackageID {
  169. newObj := oldObjs[i]
  170. avaiMovings[i].ApplyTo(&newObj)
  171. pkgIDChangedObjs = append(pkgIDChangedObjs, newObj)
  172. } else if avaiMovings[i].Path != oldObjs[i].Path {
  173. newObj := oldObjs[i]
  174. avaiMovings[i].ApplyTo(&newObj)
  175. pathChangedObjs = append(pathChangedObjs, newObj)
  176. }
  177. }
  178. var newObjs []types.Object
  179. // 对于PackageID发生变化的对象,需要检查目标Package内是否存在同Path的对象
  180. checkedObjs, err := svc.checkPackageChangedObjects(tx, pkgIDChangedObjs)
  181. if err != nil {
  182. return err
  183. }
  184. newObjs = append(newObjs, checkedObjs...)
  185. // 对于只有Path发生变化的对象,则检查同Package内有没有同Path的对象
  186. checkedObjs, err = svc.checkPathChangedObjects(tx, pathChangedObjs)
  187. if err != nil {
  188. return err
  189. }
  190. newObjs = append(newObjs, checkedObjs...)
  191. err = svc.DB.Object().BatchUpdate(tx, newObjs)
  192. if err != nil {
  193. return fmt.Errorf("batch create or update: %w", err)
  194. }
  195. sucs = lo.Map(newObjs, func(obj types.Object, _ int) types.ObjectID { return obj.ObjectID })
  196. evt = lo.Map(newObjs, func(obj types.Object, _ int) *datamap.BodyObjectInfoUpdated {
  197. return &datamap.BodyObjectInfoUpdated{
  198. Object: obj,
  199. }
  200. })
  201. return nil
  202. })
  203. if err != nil {
  204. logger.Warn(err.Error())
  205. return nil, err
  206. }
  207. for _, e := range evt {
  208. svc.EvtPub.Publish(e)
  209. }
  210. return sucs, nil
  211. }
  212. func (svc *ObjectService) Download(req downloader.DownloadReqeust) (*downloader.Downloading, error) {
  213. // TODO 检查用户ID
  214. iter := svc.Downloader.DownloadObjects([]downloader.DownloadReqeust{req})
  215. // 初始化下载过程
  216. downloading, err := iter.MoveNext()
  217. if downloading == nil {
  218. return nil, fmt.Errorf("object %v not found", req.ObjectID)
  219. }
  220. if err != nil {
  221. return nil, err
  222. }
  223. return downloading, nil
  224. }
  225. func (svc *Service) checkPackageChangedObjects(tx db.SQLContext, objs []types.Object) ([]types.Object, error) {
  226. if len(objs) == 0 {
  227. return nil, nil
  228. }
  229. type PackageObjects struct {
  230. PackageID types.PackageID
  231. ObjectByPath map[string]*types.Object
  232. }
  233. packages := make(map[types.PackageID]*PackageObjects)
  234. for _, obj := range objs {
  235. pkg, ok := packages[obj.PackageID]
  236. if !ok {
  237. pkg = &PackageObjects{
  238. PackageID: obj.PackageID,
  239. ObjectByPath: make(map[string]*types.Object),
  240. }
  241. packages[obj.PackageID] = pkg
  242. }
  243. if pkg.ObjectByPath[obj.Path] == nil {
  244. o := obj
  245. pkg.ObjectByPath[obj.Path] = &o
  246. } else {
  247. // TODO 有两个对象移动到同一个路径,有冲突
  248. }
  249. }
  250. var willUpdateObjs []types.Object
  251. for _, pkg := range packages {
  252. _, err := svc.DB.Package().GetByID(tx, pkg.PackageID)
  253. if errors.Is(err, gorm.ErrRecordNotFound) {
  254. continue
  255. }
  256. if err != nil {
  257. return nil, fmt.Errorf("getting package by id: %w", err)
  258. }
  259. existsObjs, err := svc.DB.Object().BatchGetByPackagePath(tx, pkg.PackageID, lo.Keys(pkg.ObjectByPath))
  260. if err != nil {
  261. return nil, fmt.Errorf("batch getting objects by package path: %w", err)
  262. }
  263. // 标记冲突的对象
  264. for _, obj := range existsObjs {
  265. pkg.ObjectByPath[obj.Path] = nil
  266. // TODO 目标Package内有冲突的对象
  267. }
  268. for _, obj := range pkg.ObjectByPath {
  269. if obj == nil {
  270. continue
  271. }
  272. willUpdateObjs = append(willUpdateObjs, *obj)
  273. }
  274. }
  275. return willUpdateObjs, nil
  276. }
  277. func (svc *Service) checkPathChangedObjects(tx db.SQLContext, objs []types.Object) ([]types.Object, error) {
  278. if len(objs) == 0 {
  279. return nil, nil
  280. }
  281. objByPath := make(map[string]*types.Object)
  282. for _, obj := range objs {
  283. if objByPath[obj.Path] == nil {
  284. o := obj
  285. objByPath[obj.Path] = &o
  286. } else {
  287. // TODO 有两个对象移动到同一个路径,有冲突
  288. }
  289. }
  290. _, err := svc.DB.Package().GetByID(tx, objs[0].PackageID)
  291. if errors.Is(err, gorm.ErrRecordNotFound) {
  292. return nil, nil
  293. }
  294. if err != nil {
  295. return nil, fmt.Errorf("getting package by id: %w", err)
  296. }
  297. existsObjs, err := svc.DB.Object().BatchGetByPackagePath(tx, objs[0].PackageID, lo.Map(objs, func(obj types.Object, idx int) string { return obj.Path }))
  298. if err != nil {
  299. return nil, fmt.Errorf("batch getting objects by package path: %w", err)
  300. }
  301. // 不支持两个对象交换位置的情况,因为数据库不支持
  302. for _, obj := range existsObjs {
  303. objByPath[obj.Path] = nil
  304. }
  305. var willMoveObjs []types.Object
  306. for _, obj := range objByPath {
  307. if obj == nil {
  308. continue
  309. }
  310. willMoveObjs = append(willMoveObjs, *obj)
  311. }
  312. return willMoveObjs, nil
  313. }
  314. func (svc *ObjectService) Delete(objectIDs []types.ObjectID) error {
  315. var sucs []types.ObjectID
  316. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  317. avaiIDs, err := svc.DB.Object().BatchTestObjectID(tx, objectIDs)
  318. if err != nil {
  319. return fmt.Errorf("batch testing object id: %w", err)
  320. }
  321. sucs = lo.Keys(avaiIDs)
  322. err = svc.DB.Object().BatchDelete(tx, objectIDs)
  323. if err != nil {
  324. return fmt.Errorf("batch deleting objects: %w", err)
  325. }
  326. err = svc.DB.ObjectBlock().BatchDeleteByObjectID(tx, objectIDs)
  327. if err != nil {
  328. return fmt.Errorf("batch deleting object blocks: %w", err)
  329. }
  330. err = svc.DB.PinnedObject().BatchDeleteByObjectID(tx, objectIDs)
  331. if err != nil {
  332. return fmt.Errorf("batch deleting pinned objects: %w", err)
  333. }
  334. err = svc.DB.ObjectAccessStat().BatchDeleteByObjectID(tx, objectIDs)
  335. if err != nil {
  336. return fmt.Errorf("batch deleting object access stats: %w", err)
  337. }
  338. return nil
  339. })
  340. if err != nil {
  341. return err
  342. }
  343. for _, objID := range sucs {
  344. svc.EvtPub.Publish(&datamap.BodyObjectDeleted{
  345. ObjectID: objID,
  346. })
  347. }
  348. return nil
  349. }
  350. func (svc *ObjectService) Clone(clonings []api.CloningObject) ([]*types.Object, error) {
  351. type CloningObject struct {
  352. Cloning api.CloningObject
  353. OrgIndex int
  354. }
  355. type PackageClonings struct {
  356. PackageID types.PackageID
  357. Clonings map[string]CloningObject
  358. }
  359. var evt []*datamap.BodyNewOrUpdateObject
  360. // TODO 要检查用户是否有Object、Package的权限
  361. cloningMap := make(map[types.PackageID]*PackageClonings)
  362. for i, cloning := range clonings {
  363. pkg, ok := cloningMap[cloning.NewPackageID]
  364. if !ok {
  365. pkg = &PackageClonings{
  366. PackageID: cloning.NewPackageID,
  367. Clonings: make(map[string]CloningObject),
  368. }
  369. cloningMap[cloning.NewPackageID] = pkg
  370. }
  371. pkg.Clonings[cloning.NewPath] = CloningObject{
  372. Cloning: cloning,
  373. OrgIndex: i,
  374. }
  375. }
  376. ret := make([]*types.Object, len(cloningMap))
  377. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  378. // 剔除掉新路径已经存在的对象
  379. for _, pkg := range cloningMap {
  380. exists, err := svc.DB.Object().BatchGetByPackagePath(tx, pkg.PackageID, lo.Keys(pkg.Clonings))
  381. if err != nil {
  382. return fmt.Errorf("batch getting objects by package path: %w", err)
  383. }
  384. for _, obj := range exists {
  385. delete(pkg.Clonings, obj.Path)
  386. }
  387. }
  388. // 删除目的Package不存在的对象
  389. newPkg, err := svc.DB.Package().BatchTestPackageID(tx, lo.Keys(cloningMap))
  390. if err != nil {
  391. return fmt.Errorf("batch testing package id: %w", err)
  392. }
  393. for _, pkg := range cloningMap {
  394. if !newPkg[pkg.PackageID] {
  395. delete(cloningMap, pkg.PackageID)
  396. }
  397. }
  398. var avaiClonings []CloningObject
  399. var avaiObjIDs []types.ObjectID
  400. for _, pkg := range cloningMap {
  401. for _, cloning := range pkg.Clonings {
  402. avaiClonings = append(avaiClonings, cloning)
  403. avaiObjIDs = append(avaiObjIDs, cloning.Cloning.ObjectID)
  404. }
  405. }
  406. avaiDetails, err := svc.DB.Object().BatchGetDetails(tx, avaiObjIDs)
  407. if err != nil {
  408. return fmt.Errorf("batch getting object details: %w", err)
  409. }
  410. avaiDetailsMap := make(map[types.ObjectID]types.ObjectDetail)
  411. for _, detail := range avaiDetails {
  412. avaiDetailsMap[detail.Object.ObjectID] = detail
  413. }
  414. oldAvaiClonings := avaiClonings
  415. avaiClonings = nil
  416. var newObjs []types.Object
  417. for _, cloning := range oldAvaiClonings {
  418. // 进一步剔除原始对象不存在的情况
  419. detail, ok := avaiDetailsMap[cloning.Cloning.ObjectID]
  420. if !ok {
  421. continue
  422. }
  423. avaiClonings = append(avaiClonings, cloning)
  424. newObj := detail.Object
  425. newObj.ObjectID = 0
  426. newObj.Path = cloning.Cloning.NewPath
  427. newObj.PackageID = cloning.Cloning.NewPackageID
  428. newObjs = append(newObjs, newObj)
  429. }
  430. // 先创建出新对象
  431. err = svc.DB.Object().BatchCreate(tx, &newObjs)
  432. if err != nil {
  433. return fmt.Errorf("batch creating objects: %w", err)
  434. }
  435. // 创建了新对象就能拿到新对象ID,再创建新对象块
  436. var newBlks []types.ObjectBlock
  437. for i, cloning := range avaiClonings {
  438. oldBlks := avaiDetailsMap[cloning.Cloning.ObjectID].Blocks
  439. for _, blk := range oldBlks {
  440. newBlk := blk
  441. newBlk.ObjectID = newObjs[i].ObjectID
  442. newBlks = append(newBlks, newBlk)
  443. }
  444. }
  445. err = svc.DB.ObjectBlock().BatchCreate(tx, newBlks)
  446. if err != nil {
  447. return fmt.Errorf("batch creating object blocks: %w", err)
  448. }
  449. for i, cloning := range avaiClonings {
  450. ret[cloning.OrgIndex] = &newObjs[i]
  451. }
  452. for i, cloning := range avaiClonings {
  453. var evtBlks []datamap.BlockDistributionObjectInfo
  454. blkType := getBlockTypeFromRed(newObjs[i].Redundancy)
  455. oldBlks := avaiDetailsMap[cloning.Cloning.ObjectID].Blocks
  456. for _, blk := range oldBlks {
  457. evtBlks = append(evtBlks, datamap.BlockDistributionObjectInfo{
  458. BlockType: blkType,
  459. Index: blk.Index,
  460. UserSpaceID: blk.UserSpaceID,
  461. })
  462. }
  463. evt = append(evt, &datamap.BodyNewOrUpdateObject{
  464. Info: newObjs[i],
  465. BlockDistribution: evtBlks,
  466. })
  467. }
  468. return nil
  469. })
  470. if err != nil {
  471. logger.Warnf("cloning objects: %s", err.Error())
  472. return nil, err
  473. }
  474. for _, e := range evt {
  475. svc.EvtPub.Publish(e)
  476. }
  477. return ret, nil
  478. }
  479. // GetPackageObjects 获取包中的对象列表。
  480. // userID: 用户ID。
  481. // packageID: 包ID。
  482. // 返回值: 对象列表和错误信息。
  483. func (svc *ObjectService) GetPackageObjects(packageID types.PackageID) ([]types.Object, error) {
  484. return svc.DB.Object().GetPackageObjects(svc.DB.DefCtx(), packageID)
  485. }
  486. func (svc *ObjectService) GetObjectDetails(objectIDs []types.ObjectID) ([]*types.ObjectDetail, error) {
  487. detailsMp := make(map[types.ObjectID]*types.ObjectDetail)
  488. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  489. var err error
  490. objectIDs = sort2.SortAsc(objectIDs)
  491. // 根据ID依次查询Object,ObjectBlock,PinnedObject,并根据升序的特点进行合并
  492. objs, err := svc.DB.Object().BatchGet(tx, objectIDs)
  493. if err != nil {
  494. return fmt.Errorf("batch get objects: %w", err)
  495. }
  496. for _, obj := range objs {
  497. detailsMp[obj.ObjectID] = &types.ObjectDetail{
  498. Object: obj,
  499. }
  500. }
  501. // 查询合并
  502. blocks, err := svc.DB.ObjectBlock().BatchGetByObjectID(tx, objectIDs)
  503. if err != nil {
  504. return fmt.Errorf("batch get object blocks: %w", err)
  505. }
  506. for _, block := range blocks {
  507. d := detailsMp[block.ObjectID]
  508. d.Blocks = append(d.Blocks, block)
  509. }
  510. // 查询合并
  511. pinneds, err := svc.DB.PinnedObject().BatchGetByObjectID(tx, objectIDs)
  512. if err != nil {
  513. return fmt.Errorf("batch get pinned objects: %w", err)
  514. }
  515. for _, pinned := range pinneds {
  516. d := detailsMp[pinned.ObjectID]
  517. d.PinnedAt = append(d.PinnedAt, pinned.UserSpaceID)
  518. }
  519. return nil
  520. })
  521. if err != nil {
  522. logger.Warn(err.Error())
  523. return nil, err
  524. }
  525. details := make([]*types.ObjectDetail, len(objectIDs))
  526. for i, objID := range objectIDs {
  527. details[i] = detailsMp[objID]
  528. }
  529. return details, nil
  530. }
  531. func (svc *ObjectService) NewMultipartUploadObject(packageID types.PackageID, path string) (types.Object, error) {
  532. var obj types.Object
  533. err := svc.DB.DoTx(func(tx db.SQLContext) error {
  534. oldObj, err := svc.DB.Object().GetByPath(tx, packageID, path)
  535. if err == nil {
  536. obj = oldObj
  537. err := svc.DB.ObjectBlock().DeleteByObjectID(tx, obj.ObjectID)
  538. if err != nil {
  539. return fmt.Errorf("delete object blocks: %w", err)
  540. }
  541. obj.FileHash = types.EmptyHash
  542. obj.Size = 0
  543. obj.Redundancy = types.NewMultipartUploadRedundancy()
  544. obj.UpdateTime = time.Now()
  545. err = svc.DB.Object().BatchUpdate(tx, []types.Object{obj})
  546. if err != nil {
  547. return fmt.Errorf("update object: %w", err)
  548. }
  549. return nil
  550. }
  551. obj = types.Object{
  552. PackageID: packageID,
  553. Path: path,
  554. FileHash: types.EmptyHash,
  555. Size: 0,
  556. Redundancy: types.NewMultipartUploadRedundancy(),
  557. CreateTime: time.Now(),
  558. UpdateTime: time.Now(),
  559. }
  560. objID, err := svc.DB.Object().Create(tx, obj)
  561. if err != nil {
  562. return fmt.Errorf("create object: %w", err)
  563. }
  564. obj.ObjectID = objID
  565. return nil
  566. })
  567. if err != nil {
  568. logger.Warnf("new multipart upload object: %s", err.Error())
  569. return types.Object{}, err
  570. }
  571. return obj, nil
  572. }
  573. func (svc *ObjectService) CompleteMultipartUpload(objectID types.ObjectID, indexes []int) (types.Object, error) {
  574. if len(indexes) == 0 {
  575. return types.Object{}, fmt.Errorf("no block indexes specified")
  576. }
  577. objDe, err := db.DoTx11(svc.DB, svc.DB.Object().GetDetail, objectID)
  578. if err != nil {
  579. return types.Object{}, err
  580. }
  581. _, ok := objDe.Object.Redundancy.(*types.MultipartUploadRedundancy)
  582. if !ok {
  583. return types.Object{}, fmt.Errorf("object %v is not a multipart upload", objectID)
  584. }
  585. if len(objDe.Blocks) == 0 {
  586. return types.Object{}, fmt.Errorf("object %v has no blocks", objectID)
  587. }
  588. objBlkMap := make(map[int]types.ObjectBlock)
  589. for _, blk := range objDe.Blocks {
  590. objBlkMap[blk.Index] = blk
  591. }
  592. var compBlks []types.ObjectBlock
  593. var compBlkSpaces []types.UserSpaceDetail
  594. var targetSpace types.UserSpaceDetail
  595. for i, idx := range indexes {
  596. blk, ok := objBlkMap[idx]
  597. if !ok {
  598. return types.Object{}, fmt.Errorf("block %d not found in object %v", idx, objectID)
  599. }
  600. stg := svc.UserSpaceMeta.Get(blk.UserSpaceID)
  601. if stg == nil {
  602. return types.Object{}, fmt.Errorf("storage of user space %d not found", blk.UserSpaceID)
  603. }
  604. compBlks = append(compBlks, blk)
  605. compBlkSpaces = append(compBlkSpaces, *stg)
  606. if i == 0 {
  607. targetSpace = *stg
  608. }
  609. }
  610. bld := exec.NewPlanBuilder()
  611. err = plans.CompleteMultipart(compBlks, compBlkSpaces, targetSpace, "shard", bld)
  612. if err != nil {
  613. return types.Object{}, err
  614. }
  615. exeCtx := exec.NewExecContext()
  616. ret, err := bld.Execute(exeCtx).Wait(context.Background())
  617. if err != nil {
  618. return types.Object{}, err
  619. }
  620. shardInfo := ret["shard"].(*ops2.ShardInfoValue)
  621. err = db.DoTx10(svc.DB, svc.DB.Object().BatchUpdateRedundancy, []db.UpdatingObjectRedundancy{
  622. {
  623. ObjectID: objectID,
  624. FileHash: shardInfo.Hash,
  625. Size: shardInfo.Size,
  626. Redundancy: types.NewNoneRedundancy(),
  627. Blocks: []types.ObjectBlock{{
  628. ObjectID: objectID,
  629. Index: 0,
  630. UserSpaceID: targetSpace.UserSpace.UserSpaceID,
  631. FileHash: shardInfo.Hash,
  632. Size: shardInfo.Size,
  633. }},
  634. },
  635. })
  636. if err != nil {
  637. return types.Object{}, err
  638. }
  639. obj, err := svc.DB.Object().GetByID(svc.DB.DefCtx(), objectID)
  640. if err != nil {
  641. return types.Object{}, err
  642. }
  643. return obj, nil
  644. }

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。