You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

object.go 23 kB

7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
7 months ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721
  1. package db
  2. import (
  3. "fmt"
  4. "strings"
  5. "time"
  6. "github.com/samber/lo"
  7. "gorm.io/gorm"
  8. "gorm.io/gorm/clause"
  9. "gitlink.org.cn/cloudream/common/utils/sort2"
  10. jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
  11. )
  12. type ObjectDB struct {
  13. *DB
  14. }
  15. func (db *DB) Object() *ObjectDB {
  16. return &ObjectDB{DB: db}
  17. }
  18. func (db *ObjectDB) GetByID(ctx SQLContext, objectID jcstypes.ObjectID) (jcstypes.Object, error) {
  19. var ret jcstypes.Object
  20. err := ctx.Table("Object").Where("ObjectID = ?", objectID).First(&ret).Error
  21. return ret, err
  22. }
  23. func (db *ObjectDB) GetByPath(ctx SQLContext, packageID jcstypes.PackageID, path string) (jcstypes.Object, error) {
  24. var ret jcstypes.Object
  25. err := ctx.Table("Object").Where("PackageID = ? AND Path = ?", packageID, path).First(&ret).Error
  26. return ret, err
  27. }
  28. func (db *ObjectDB) GetByFullPath(ctx SQLContext, bktName string, pkgName string, path string) (jcstypes.Object, error) {
  29. var ret jcstypes.Object
  30. err := ctx.Table("Object").
  31. Joins("join Package on Package.PackageID = Object.PackageID and Package.Name = ?", pkgName).
  32. Joins("join Bucket on Bucket.BucketID = Package.BucketID and Bucket.Name = ?", bktName).
  33. Where("Object.Path = ?", path).First(&ret).Error
  34. return ret, err
  35. }
  36. func (db *ObjectDB) GetWithPathPrefix(ctx SQLContext, packageID jcstypes.PackageID, pathPrefix string) ([]jcstypes.Object, error) {
  37. var ret []jcstypes.Object
  38. err := ctx.Table("Object").Where("PackageID = ? AND Path LIKE ?", packageID, escapeLike("", "%", pathPrefix)).Order("ObjectID ASC").Find(&ret).Error
  39. return ret, err
  40. }
  41. // 查询结果将按照Path升序,而不是ObjectID升序
  42. func (db *ObjectDB) GetWithPathPrefixPaged(ctx SQLContext, packageID jcstypes.PackageID, pathPrefix string, startPath string, limit int) ([]jcstypes.Object, error) {
  43. var ret []jcstypes.Object
  44. err := ctx.Table("Object").Where("PackageID = ? AND Path > ? AND Path LIKE ?", packageID, startPath, pathPrefix+"%").Order("Path ASC").Limit(limit).Find(&ret).Error
  45. return ret, err
  46. }
  47. func (db *ObjectDB) GetByPrefixGrouped(ctx SQLContext, packageID jcstypes.PackageID, pathPrefix string) (objs []jcstypes.Object, commonPrefixes []string, err error) {
  48. type ObjectOrDir struct {
  49. jcstypes.Object
  50. IsObject bool `gorm:"IsObject"`
  51. Prefix string `gorm:"Prefix"`
  52. }
  53. sepCnt := strings.Count(pathPrefix, jcstypes.ObjectPathSeparator) + 1
  54. prefixStatm := fmt.Sprintf("Substring_Index(Path, '%s', %d)", jcstypes.ObjectPathSeparator, sepCnt)
  55. grouping := ctx.Table("Object").
  56. Select(fmt.Sprintf("%s as Prefix, Max(ObjectID) as ObjectID, %s = Path as IsObject", prefixStatm, prefixStatm)).
  57. Where("PackageID = ?", packageID).
  58. Where("Path like ?", pathPrefix+"%").
  59. Group("Prefix, IsObject").
  60. Order("Prefix ASC")
  61. var ret []ObjectOrDir
  62. err = ctx.Table("Object").
  63. Select("Grouped.IsObject, Grouped.Prefix, Object.*").
  64. Joins("right join (?) as Grouped on Object.ObjectID = Grouped.ObjectID and Grouped.IsObject = 1", grouping).
  65. Find(&ret).Error
  66. if err != nil {
  67. return
  68. }
  69. for _, o := range ret {
  70. if o.IsObject {
  71. objs = append(objs, o.Object)
  72. } else {
  73. commonPrefixes = append(commonPrefixes, o.Prefix+jcstypes.ObjectPathSeparator)
  74. }
  75. }
  76. return
  77. }
  78. func (db *ObjectDB) GetByPrefixGroupedPaged(ctx SQLContext, packageID jcstypes.PackageID, pathPrefix string, startPath string, limit int) (objs []jcstypes.Object, commonPrefixes []string, nextStartPath string, err error) {
  79. type ObjectOrDir struct {
  80. jcstypes.Object
  81. IsObject bool `gorm:"IsObject"`
  82. Prefix string `gorm:"Prefix"`
  83. }
  84. sepCnt := strings.Count(pathPrefix, jcstypes.ObjectPathSeparator) + 1
  85. prefixStatm := fmt.Sprintf("Substring_Index(Path, '%s', %d)", jcstypes.ObjectPathSeparator, sepCnt)
  86. grouping := ctx.Table("Object").
  87. Select(fmt.Sprintf("%s as Prefix, Max(ObjectID) as ObjectID, %s = Path as IsObject", prefixStatm, prefixStatm)).
  88. Where("PackageID = ?", packageID).
  89. Where("Path like ?", pathPrefix+"%").
  90. Group("Prefix, IsObject").
  91. Having("Prefix > ?", startPath).
  92. Limit(limit).
  93. Order("Prefix ASC")
  94. var ret []ObjectOrDir
  95. err = ctx.Table("Object").
  96. Select("Grouped.IsObject, Grouped.Prefix, Object.*").
  97. Joins("right join (?) as Grouped on Object.ObjectID = Grouped.ObjectID and Grouped.IsObject = 1", grouping).
  98. Find(&ret).Error
  99. if err != nil {
  100. return
  101. }
  102. for _, o := range ret {
  103. if o.IsObject {
  104. objs = append(objs, o.Object)
  105. } else {
  106. commonPrefixes = append(commonPrefixes, o.Prefix+jcstypes.ObjectPathSeparator)
  107. }
  108. nextStartPath = o.Prefix
  109. }
  110. return
  111. }
  112. // 返回gorm.ErrRecordNotFound表示没有对象,nil表示有对象
  113. func (db *ObjectDB) HasObjectWithPrefix(ctx SQLContext, packageID jcstypes.PackageID, pathPrefix string) error {
  114. var obj jcstypes.Object
  115. return ctx.Table("Object").Where("PackageID = ? AND Path LIKE ?", packageID, escapeLike("", "%", pathPrefix)).First(&obj).Error
  116. }
  117. func (db *ObjectDB) BatchTestObjectID(ctx SQLContext, objectIDs []jcstypes.ObjectID) (map[jcstypes.ObjectID]bool, error) {
  118. if len(objectIDs) == 0 {
  119. return make(map[jcstypes.ObjectID]bool), nil
  120. }
  121. var avaiIDs []jcstypes.ObjectID
  122. err := ctx.Table("Object").Where("ObjectID IN ?", objectIDs).Pluck("ObjectID", &avaiIDs).Error
  123. if err != nil {
  124. return nil, err
  125. }
  126. avaiIDMap := make(map[jcstypes.ObjectID]bool)
  127. for _, pkgID := range avaiIDs {
  128. avaiIDMap[pkgID] = true
  129. }
  130. return avaiIDMap, nil
  131. }
  132. func (db *ObjectDB) BatchGet(ctx SQLContext, objectIDs []jcstypes.ObjectID) ([]jcstypes.Object, error) {
  133. if len(objectIDs) == 0 {
  134. return nil, nil
  135. }
  136. var objs []jcstypes.Object
  137. err := ctx.Table("Object").Where("ObjectID IN ?", objectIDs).Order("ObjectID ASC").Find(&objs).Error
  138. if err != nil {
  139. return nil, err
  140. }
  141. return objs, nil
  142. }
  143. func (db *ObjectDB) BatchGetByPackagePath(ctx SQLContext, pkgID jcstypes.PackageID, pathes []string) ([]jcstypes.Object, error) {
  144. if len(pathes) == 0 {
  145. return nil, nil
  146. }
  147. var objs []jcstypes.Object
  148. err := ctx.Table("Object").Where("PackageID = ? AND Path IN ?", pkgID, pathes).Find(&objs).Error
  149. if err != nil {
  150. return nil, err
  151. }
  152. return objs, nil
  153. }
  154. func (db *ObjectDB) GetDetail(ctx SQLContext, objectID jcstypes.ObjectID) (jcstypes.ObjectDetail, error) {
  155. var obj jcstypes.Object
  156. err := ctx.Table("Object").Where("ObjectID = ?", objectID).First(&obj).Error
  157. if err != nil {
  158. return jcstypes.ObjectDetail{}, fmt.Errorf("getting object: %w", err)
  159. }
  160. // 获取所有的 ObjectBlock
  161. var allBlocks []jcstypes.ObjectBlock
  162. err = ctx.Table("ObjectBlock").Where("ObjectID = ?", objectID).Order("`Index` ASC").Find(&allBlocks).Error
  163. if err != nil {
  164. return jcstypes.ObjectDetail{}, fmt.Errorf("getting all object blocks: %w", err)
  165. }
  166. // 获取所有的 PinnedObject
  167. var allPinnedObjs []jcstypes.PinnedObject
  168. err = ctx.Table("PinnedObject").Where("ObjectID = ?", objectID).Order("ObjectID ASC").Find(&allPinnedObjs).Error
  169. if err != nil {
  170. return jcstypes.ObjectDetail{}, fmt.Errorf("getting all pinned objects: %w", err)
  171. }
  172. pinnedAt := make([]jcstypes.UserSpaceID, len(allPinnedObjs))
  173. for i, po := range allPinnedObjs {
  174. pinnedAt[i] = po.UserSpaceID
  175. }
  176. return jcstypes.ObjectDetail{
  177. Object: obj,
  178. Blocks: allBlocks,
  179. PinnedAt: pinnedAt,
  180. }, nil
  181. }
  182. // 仅返回查询到的对象
  183. func (db *ObjectDB) BatchGetDetails(ctx SQLContext, objectIDs []jcstypes.ObjectID) ([]jcstypes.ObjectDetail, error) {
  184. var objs []jcstypes.Object
  185. err := ctx.Table("Object").Where("ObjectID IN ?", objectIDs).Order("ObjectID ASC").Find(&objs).Error
  186. if err != nil {
  187. return nil, err
  188. }
  189. // 获取所有的 ObjectBlock
  190. var allBlocks []jcstypes.ObjectBlock
  191. err = ctx.Table("ObjectBlock").Where("ObjectID IN ?", objectIDs).Order("ObjectID, `Index` ASC").Find(&allBlocks).Error
  192. if err != nil {
  193. return nil, err
  194. }
  195. // 获取所有的 PinnedObject
  196. var allPinnedObjs []jcstypes.PinnedObject
  197. err = ctx.Table("PinnedObject").Where("ObjectID IN ?", objectIDs).Order("ObjectID ASC").Find(&allPinnedObjs).Error
  198. if err != nil {
  199. return nil, err
  200. }
  201. details := make([]jcstypes.ObjectDetail, len(objs))
  202. for i, obj := range objs {
  203. details[i] = jcstypes.ObjectDetail{
  204. Object: obj,
  205. }
  206. }
  207. jcstypes.DetailsFillObjectBlocks(details, allBlocks)
  208. jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
  209. return details, nil
  210. }
  211. func (db *ObjectDB) BatchGetDetailsPaged(ctx SQLContext, pkgID jcstypes.PackageID, lastObjID jcstypes.ObjectID, maxCnt int) ([]jcstypes.ObjectDetail, error) {
  212. var objs []jcstypes.Object
  213. err := ctx.Table("Object").Where("ObjectID > ? ORDER BY ObjectID ASC LIMIT ?", lastObjID, maxCnt).Find(&objs).Error
  214. if err != nil {
  215. return nil, err
  216. }
  217. objIDs := make([]jcstypes.ObjectID, len(objs))
  218. for i, obj := range objs {
  219. objIDs[i] = obj.ObjectID
  220. }
  221. // 获取所有的 ObjectBlock
  222. var allBlocks []jcstypes.ObjectBlock
  223. err = ctx.Table("ObjectBlock").Where("ObjectID IN ?", objIDs).Order("ObjectID, `Index` ASC").Find(&allBlocks).Error
  224. if err != nil {
  225. return nil, err
  226. }
  227. // 获取所有的 PinnedObject
  228. var allPinnedObjs []jcstypes.PinnedObject
  229. err = ctx.Table("PinnedObject").Where("ObjectID IN ?", objIDs).Order("ObjectID ASC").Find(&allPinnedObjs).Error
  230. if err != nil {
  231. return nil, err
  232. }
  233. details := make([]jcstypes.ObjectDetail, len(objs))
  234. for i, obj := range objs {
  235. details[i] = jcstypes.ObjectDetail{
  236. Object: obj,
  237. }
  238. }
  239. jcstypes.DetailsFillObjectBlocks(details, allBlocks)
  240. jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
  241. return details, nil
  242. }
  243. func (db *ObjectDB) Create(ctx SQLContext, obj jcstypes.Object) (jcstypes.ObjectID, error) {
  244. err := ctx.Table("Object").Create(&obj).Error
  245. if err != nil {
  246. return 0, fmt.Errorf("insert object failed, err: %w", err)
  247. }
  248. return obj.ObjectID, nil
  249. }
  250. // 批量创建对象,创建完成后会填充ObjectID。
  251. func (db *ObjectDB) BatchCreate(ctx SQLContext, objs *[]jcstypes.Object) error {
  252. if len(*objs) == 0 {
  253. return nil
  254. }
  255. return ctx.Table("Object").Create(objs).Error
  256. }
  257. // 批量更新对象所有属性,objs中的对象必须包含ObjectID
  258. func (db *ObjectDB) BatchUpdate(ctx SQLContext, objs []jcstypes.Object) error {
  259. if len(objs) == 0 {
  260. return nil
  261. }
  262. return ctx.Clauses(clause.OnConflict{
  263. Columns: []clause.Column{{Name: "ObjectID"}},
  264. UpdateAll: true,
  265. }).Create(objs).Error
  266. }
  267. // 批量更新对象指定属性,objs中的对象只需设置需要更新的属性即可,但:
  268. // 1. 必须包含ObjectID
  269. // 2. 日期类型属性不能设置为0值
  270. func (db *ObjectDB) BatchUpdateColumns(ctx SQLContext, objs []jcstypes.Object, columns []string) error {
  271. if len(objs) == 0 {
  272. return nil
  273. }
  274. return ctx.Clauses(clause.OnConflict{
  275. Columns: []clause.Column{{Name: "ObjectID"}},
  276. DoUpdates: clause.AssignmentColumns(columns),
  277. }).Create(objs).Error
  278. }
  279. func (db *ObjectDB) GetPackageObjects(ctx SQLContext, packageID jcstypes.PackageID) ([]jcstypes.Object, error) {
  280. var ret []jcstypes.Object
  281. err := ctx.Table("Object").Where("PackageID = ?", packageID).Order("ObjectID ASC").Find(&ret).Error
  282. return ret, err
  283. }
  284. func (db *ObjectDB) GetPackageObjectDetails(ctx SQLContext, packageID jcstypes.PackageID) ([]jcstypes.ObjectDetail, error) {
  285. var objs []jcstypes.Object
  286. err := ctx.Table("Object").Where("PackageID = ?", packageID).Order("ObjectID ASC").Find(&objs).Error
  287. if err != nil {
  288. return nil, fmt.Errorf("getting objects: %w", err)
  289. }
  290. // 获取所有的 ObjectBlock
  291. var allBlocks []jcstypes.ObjectBlock
  292. err = ctx.Table("ObjectBlock").
  293. Select("ObjectBlock.*").
  294. Joins("JOIN Object ON ObjectBlock.ObjectID = Object.ObjectID").
  295. Where("Object.PackageID = ?", packageID).
  296. Order("ObjectBlock.ObjectID, `Index` ASC").
  297. Find(&allBlocks).Error
  298. if err != nil {
  299. return nil, fmt.Errorf("getting all object blocks: %w", err)
  300. }
  301. // 获取所有的 PinnedObject
  302. var allPinnedObjs []jcstypes.PinnedObject
  303. err = ctx.Table("PinnedObject").
  304. Select("PinnedObject.*").
  305. Joins("JOIN Object ON PinnedObject.ObjectID = Object.ObjectID").
  306. Where("Object.PackageID = ?", packageID).
  307. Order("PinnedObject.ObjectID").
  308. Find(&allPinnedObjs).Error
  309. if err != nil {
  310. return nil, fmt.Errorf("getting all pinned objects: %w", err)
  311. }
  312. details := make([]jcstypes.ObjectDetail, len(objs))
  313. for i, obj := range objs {
  314. details[i] = jcstypes.ObjectDetail{
  315. Object: obj,
  316. }
  317. }
  318. jcstypes.DetailsFillObjectBlocks(details, allBlocks)
  319. jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
  320. return details, nil
  321. }
  322. func (db *ObjectDB) GetObjectsIfAnyBlockOnStorage(ctx SQLContext, spaceID jcstypes.UserSpaceID) ([]jcstypes.Object, error) {
  323. var objs []jcstypes.Object
  324. err := ctx.Table("Object").Where("ObjectID IN (SELECT ObjectID FROM ObjectBlock WHERE UserSpaceID = ?)", spaceID).Order("ObjectID ASC").Find(&objs).Error
  325. if err != nil {
  326. return nil, fmt.Errorf("getting objects: %w", err)
  327. }
  328. return objs, nil
  329. }
  330. type AddObjectEntry struct {
  331. Path string `json:"path"`
  332. Size int64 `json:"size,string"`
  333. FileHash jcstypes.FileHash `json:"fileHash"`
  334. CreateTime time.Time `json:"createTime"` // 开始上传文件的时间
  335. UserSpaceIDs []jcstypes.UserSpaceID `json:"userSpaceIDs"`
  336. }
  337. func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID jcstypes.PackageID, adds []AddObjectEntry) ([]jcstypes.Object, error) {
  338. if len(adds) == 0 {
  339. return nil, nil
  340. }
  341. // 收集所有路径
  342. pathes := make([]string, 0, len(adds))
  343. for _, add := range adds {
  344. pathes = append(pathes, add.Path)
  345. }
  346. // 先查询要更新的对象,不存在也没关系
  347. existsObjs, err := db.BatchGetByPackagePath(ctx, packageID, pathes)
  348. if err != nil {
  349. return nil, fmt.Errorf("batch get object by path: %w", err)
  350. }
  351. existsObjsMap := make(map[string]jcstypes.Object)
  352. for _, obj := range existsObjs {
  353. existsObjsMap[obj.Path] = obj
  354. }
  355. var updatingObjs []jcstypes.Object
  356. var addingObjs []jcstypes.Object
  357. for i := range adds {
  358. o := jcstypes.Object{
  359. PackageID: packageID,
  360. Path: adds[i].Path,
  361. Size: adds[i].Size,
  362. FileHash: adds[i].FileHash,
  363. Redundancy: jcstypes.NewNoneRedundancy(), // 首次上传默认使用不分块的none模式
  364. CreateTime: adds[i].CreateTime,
  365. UpdateTime: adds[i].CreateTime,
  366. }
  367. e, ok := existsObjsMap[adds[i].Path]
  368. if ok {
  369. o.ObjectID = e.ObjectID
  370. o.CreateTime = e.CreateTime
  371. updatingObjs = append(updatingObjs, o)
  372. } else {
  373. addingObjs = append(addingObjs, o)
  374. }
  375. }
  376. // 先进行更新
  377. err = db.BatchUpdate(ctx, updatingObjs)
  378. if err != nil {
  379. return nil, fmt.Errorf("batch update objects: %w", err)
  380. }
  381. // 再执行插入,Create函数插入后会填充ObjectID
  382. err = db.BatchCreate(ctx, &addingObjs)
  383. if err != nil {
  384. return nil, fmt.Errorf("batch create objects: %w", err)
  385. }
  386. // 按照add参数的顺序返回结果
  387. affectedObjsMp := make(map[string]jcstypes.Object)
  388. for _, o := range updatingObjs {
  389. affectedObjsMp[o.Path] = o
  390. }
  391. for _, o := range addingObjs {
  392. affectedObjsMp[o.Path] = o
  393. }
  394. affectedObjs := make([]jcstypes.Object, 0, len(affectedObjsMp))
  395. affectedObjIDs := make([]jcstypes.ObjectID, 0, len(affectedObjsMp))
  396. for i := range adds {
  397. obj := affectedObjsMp[adds[i].Path]
  398. affectedObjs = append(affectedObjs, obj)
  399. affectedObjIDs = append(affectedObjIDs, obj.ObjectID)
  400. }
  401. if len(affectedObjIDs) > 0 {
  402. // 批量删除 ObjectBlock
  403. if err := db.ObjectBlock().BatchDeleteByObjectID(ctx, affectedObjIDs); err != nil {
  404. return nil, fmt.Errorf("batch delete object blocks: %w", err)
  405. }
  406. // 批量删除 PinnedObject
  407. if err := db.PinnedObject().BatchDeleteByObjectID(ctx, affectedObjIDs); err != nil {
  408. return nil, fmt.Errorf("batch delete pinned objects: %w", err)
  409. }
  410. }
  411. // 创建 ObjectBlock
  412. objBlocks := make([]jcstypes.ObjectBlock, 0, len(adds))
  413. for i, add := range adds {
  414. for _, spaceID := range add.UserSpaceIDs {
  415. objBlocks = append(objBlocks, jcstypes.ObjectBlock{
  416. ObjectID: affectedObjIDs[i],
  417. Index: 0,
  418. UserSpaceID: spaceID,
  419. FileHash: add.FileHash,
  420. Size: add.Size,
  421. })
  422. }
  423. }
  424. if err := db.ObjectBlock().BatchCreate(ctx, objBlocks); err != nil {
  425. return nil, fmt.Errorf("batch create object blocks: %w", err)
  426. }
  427. return affectedObjs, nil
  428. }
  429. func (db *ObjectDB) BatchDelete(ctx SQLContext, ids []jcstypes.ObjectID) error {
  430. if len(ids) == 0 {
  431. return nil
  432. }
  433. return ctx.Table("Object").Where("ObjectID IN ?", ids).Delete(&jcstypes.Object{}).Error
  434. }
  435. func (db *ObjectDB) DeleteInPackage(ctx SQLContext, packageID jcstypes.PackageID) error {
  436. return ctx.Table("Object").Where("PackageID = ?", packageID).Delete(&jcstypes.Object{}).Error
  437. }
  438. type UpdatingObjectRedundancy struct {
  439. ObjectID jcstypes.ObjectID `json:"objectID"`
  440. FileHash jcstypes.FileHash `json:"fileHash"`
  441. Size int64 `json:"size"`
  442. Redundancy jcstypes.Redundancy `json:"redundancy"`
  443. PinnedAt []jcstypes.UserSpaceID `json:"pinnedAt"`
  444. Blocks []jcstypes.ObjectBlock `json:"blocks"`
  445. }
  446. func (db *ObjectDB) BatchUpdateRedundancy(ctx SQLContext, updates []UpdatingObjectRedundancy) error {
  447. objs := updates
  448. nowTime := time.Now()
  449. objIDs := make([]jcstypes.ObjectID, 0, len(objs))
  450. for _, obj := range objs {
  451. objIDs = append(objIDs, obj.ObjectID)
  452. }
  453. avaiIDs, err := db.Object().BatchTestObjectID(ctx, objIDs)
  454. if err != nil {
  455. return fmt.Errorf("batch test object id: %w", err)
  456. }
  457. // 过滤掉已经不存在的对象。
  458. // 注意,objIDs没有被过滤,因为后续逻辑不过滤也不会出错
  459. objs = lo.Filter(objs, func(obj UpdatingObjectRedundancy, _ int) bool {
  460. return avaiIDs[obj.ObjectID]
  461. })
  462. dummyObjs := make([]jcstypes.Object, 0, len(objs))
  463. for _, obj := range objs {
  464. dummyObjs = append(dummyObjs, jcstypes.Object{
  465. ObjectID: obj.ObjectID,
  466. FileHash: obj.FileHash,
  467. Size: obj.Size,
  468. Redundancy: obj.Redundancy,
  469. CreateTime: nowTime, // 实际不会更新,只因为不能是0值
  470. UpdateTime: nowTime,
  471. })
  472. }
  473. err = db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"FileHash", "Size", "Redundancy", "UpdateTime"})
  474. if err != nil {
  475. return fmt.Errorf("batch update object redundancy: %w", err)
  476. }
  477. // 删除原本所有的编码块记录,重新添加
  478. err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objIDs)
  479. if err != nil {
  480. return fmt.Errorf("batch delete object blocks: %w", err)
  481. }
  482. // 删除原本Pin住的Object。暂不考虑FileHash没有变化的情况
  483. err = db.PinnedObject().BatchDeleteByObjectID(ctx, objIDs)
  484. if err != nil {
  485. return fmt.Errorf("batch delete pinned object: %w", err)
  486. }
  487. blocks := make([]jcstypes.ObjectBlock, 0, len(objs))
  488. for _, obj := range objs {
  489. blocks = append(blocks, obj.Blocks...)
  490. }
  491. err = db.ObjectBlock().BatchCreate(ctx, blocks)
  492. if err != nil {
  493. return fmt.Errorf("batch create object blocks: %w", err)
  494. }
  495. pinneds := make([]jcstypes.PinnedObject, 0, len(objs))
  496. for _, obj := range objs {
  497. for _, p := range obj.PinnedAt {
  498. pinneds = append(pinneds, jcstypes.PinnedObject{
  499. ObjectID: obj.ObjectID,
  500. UserSpaceID: p,
  501. CreateTime: nowTime,
  502. })
  503. }
  504. }
  505. err = db.PinnedObject().BatchTryCreate(ctx, pinneds)
  506. if err != nil {
  507. return fmt.Errorf("batch create pinned objects: %w", err)
  508. }
  509. return nil
  510. }
  511. func (*ObjectDB) BatchUpdateUpdateTimeByPath(ctx SQLContext, packageID jcstypes.PackageID, pathes []string, updateTimes []time.Time) error {
  512. if len(pathes) != len(updateTimes) {
  513. return fmt.Errorf("pathes and updateTimes must have the same length")
  514. }
  515. if len(pathes) == 0 {
  516. return nil
  517. }
  518. sb := strings.Builder{}
  519. args := make([]any, 0, len(pathes)*2+1)
  520. sb.WriteString("UPDATE Object SET UpdateTime = CASE Path \n")
  521. for i := range pathes {
  522. sb.WriteString("WHEN ? THEN ? \n")
  523. args = append(args, pathes[i], updateTimes[i])
  524. }
  525. sb.WriteString("END WHERE PackageID = ? AND Path IN (?)")
  526. args = append(args, packageID, pathes)
  527. return ctx.Exec(sb.String(), args...).Error
  528. }
  529. func (db *ObjectDB) MoveByPrefix(ctx SQLContext, oldPkgID jcstypes.PackageID, oldPrefix string, newPkgID jcstypes.PackageID, newPrefix string) error {
  530. return ctx.Table("Object").Where("PackageID = ? AND Path LIKE ?", oldPkgID, escapeLike("", "%", oldPrefix)).
  531. Updates(map[string]any{
  532. "PackageID": newPkgID,
  533. "Path": gorm.Expr("concat(?, substring(Path, ?))", newPrefix, len(oldPrefix)+1),
  534. }).Error
  535. }
  536. func (db *ObjectDB) AppendPart(tx SQLContext, block jcstypes.ObjectBlock) error {
  537. obj, err := db.Object().GetByID(tx, block.ObjectID)
  538. if err != nil {
  539. return fmt.Errorf("getting object by id: %w", err)
  540. }
  541. _, ok := obj.Redundancy.(*jcstypes.MultipartUploadRedundancy)
  542. if !ok {
  543. return fmt.Errorf("object is not a multipart upload object")
  544. }
  545. blks, err := db.ObjectBlock().BatchGetByObjectID(tx, []jcstypes.ObjectID{obj.ObjectID})
  546. if err != nil {
  547. return fmt.Errorf("batch getting object blocks: %w", err)
  548. }
  549. blks = lo.Reject(blks, func(blk jcstypes.ObjectBlock, idx int) bool { return blk.Index == block.Index })
  550. blks = append(blks, block)
  551. blks = sort2.Sort(blks, func(a, b jcstypes.ObjectBlock) int { return a.Index - b.Index })
  552. totalSize := int64(0)
  553. var hashes [][]byte
  554. for _, blk := range blks {
  555. totalSize += blk.Size
  556. hashes = append(hashes, blk.FileHash.GetHashBytes())
  557. }
  558. newObjHash := jcstypes.CalculateCompositeHash(hashes)
  559. obj.Size = totalSize
  560. obj.FileHash = newObjHash
  561. obj.UpdateTime = time.Now()
  562. err = db.ObjectBlock().DeleteByObjectIDIndex(tx, block.ObjectID, block.Index)
  563. if err != nil {
  564. return fmt.Errorf("delete object block: %w", err)
  565. }
  566. err = db.ObjectBlock().Create(tx, block.ObjectID, block.Index, block.UserSpaceID, block.FileHash, block.Size)
  567. if err != nil {
  568. return fmt.Errorf("create object block: %w", err)
  569. }
  570. err = db.Object().BatchUpdate(tx, []jcstypes.Object{obj})
  571. if err != nil {
  572. return fmt.Errorf("update object: %w", err)
  573. }
  574. return nil
  575. }
  576. func (db *ObjectDB) BatchDeleteComplete(ctx SQLContext, objectIDs []jcstypes.ObjectID) error {
  577. err := db.Object().BatchDelete(ctx, objectIDs)
  578. if err != nil {
  579. return fmt.Errorf("batch deleting objects: %w", err)
  580. }
  581. err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objectIDs)
  582. if err != nil {
  583. return fmt.Errorf("batch deleting object blocks: %w", err)
  584. }
  585. err = db.PinnedObject().BatchDeleteByObjectID(ctx, objectIDs)
  586. if err != nil {
  587. return fmt.Errorf("batch deleting pinned objects: %w", err)
  588. }
  589. err = db.ObjectAccessStat().BatchDeleteByObjectID(ctx, objectIDs)
  590. if err != nil {
  591. return fmt.Errorf("batch deleting object access stats: %w", err)
  592. }
  593. return nil
  594. }
  595. func (db *ObjectDB) DeleteCompleteByPath(ctx SQLContext, packageID jcstypes.PackageID, path string) error {
  596. obj, err := db.Object().GetByPath(ctx, packageID, path)
  597. if err != nil {
  598. return err
  599. }
  600. return db.BatchDeleteComplete(ctx, []jcstypes.ObjectID{obj.ObjectID})
  601. }

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。