You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

redundancy_recover.go 40 kB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231
  1. package ticktock
  2. import (
  3. "context"
  4. "fmt"
  5. "strconv"
  6. "github.com/samber/lo"
  7. "gitlink.org.cn/cloudream/common/pkgs/logger"
  8. "gitlink.org.cn/cloudream/common/utils/math2"
  9. "gitlink.org.cn/cloudream/common/utils/sort2"
  10. "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
  11. "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
  12. "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
  13. "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
  14. "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
  15. "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
  16. lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
  17. jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
  18. "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
  19. )
  20. func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail) (jcstypes.Redundancy, []*userSpaceUsageInfo) {
  21. switch obj.Object.Redundancy.(type) {
  22. case *jcstypes.NoneRedundancy:
  23. if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
  24. newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
  25. return &jcstypes.DefaultECRedundancy, newStgs
  26. }
  27. return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
  28. case *jcstypes.RepRedundancy:
  29. if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
  30. newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
  31. return &jcstypes.DefaultECRedundancy, newStgs
  32. }
  33. newSpaces := t.rechooseUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
  34. for _, s := range newSpaces {
  35. if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
  36. return &jcstypes.DefaultRepRedundancy, newSpaces
  37. }
  38. }
  39. return nil, nil
  40. case *jcstypes.ECRedundancy:
  41. if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
  42. return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
  43. }
  44. newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcstypes.DefaultECRedundancy)
  45. for i, s := range newSpaces {
  46. if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
  47. return &jcstypes.DefaultECRedundancy, newSpaces
  48. }
  49. }
  50. return nil, nil
  51. case *jcstypes.LRCRedundancy:
  52. newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcstypes.DefaultLRCRedundancy)
  53. for i, s := range newLRCStgs {
  54. if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
  55. return &jcstypes.DefaultLRCRedundancy, newLRCStgs
  56. }
  57. }
  58. return nil, nil
  59. }
  60. return nil, nil
  61. }
  62. func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, newRed jcstypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  63. log := logger.WithType[ChangeRedundancy]("TickTock")
  64. var updating *db.UpdatingObjectRedundancy
  65. var evt datamap.SysEventBody
  66. var err error
  67. switch srcRed := obj.Object.Redundancy.(type) {
  68. case *jcstypes.NoneRedundancy:
  69. switch newRed := newRed.(type) {
  70. case *jcstypes.RepRedundancy:
  71. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
  72. updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)
  73. case *jcstypes.ECRedundancy:
  74. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
  75. updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)
  76. case *jcstypes.LRCRedundancy:
  77. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
  78. updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)
  79. case *jcstypes.SegmentRedundancy:
  80. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
  81. updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
  82. }
  83. case *jcstypes.RepRedundancy:
  84. switch newRed := newRed.(type) {
  85. case *jcstypes.RepRedundancy:
  86. updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)
  87. case *jcstypes.ECRedundancy:
  88. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
  89. updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
  90. }
  91. case *jcstypes.ECRedundancy:
  92. switch newRed := newRed.(type) {
  93. case *jcstypes.RepRedundancy:
  94. log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
  95. updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)
  96. case *jcstypes.ECRedundancy:
  97. updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
  98. }
  99. case *jcstypes.LRCRedundancy:
  100. switch newRed := newRed.(type) {
  101. case *jcstypes.LRCRedundancy:
  102. updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
  103. }
  104. }
  105. return updating, evt, err
  106. }
  107. // 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点
  108. func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcstypes.ObjectDetail, userspaceCnt int) []jcstypes.UserSpaceID {
  109. type stgBlocks struct {
  110. UserSpaceID jcstypes.UserSpaceID
  111. Count int
  112. }
  113. stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
  114. for _, obj := range objs {
  115. shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
  116. if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok && !shouldUseEC {
  117. for _, block := range obj.Blocks {
  118. if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
  119. stgBlocksMap[block.UserSpaceID] = &stgBlocks{
  120. UserSpaceID: block.UserSpaceID,
  121. Count: 0,
  122. }
  123. }
  124. stgBlocksMap[block.UserSpaceID].Count++
  125. }
  126. }
  127. }
  128. userspaces := lo.Values(stgBlocksMap)
  129. sort2.Sort(userspaces, func(left *stgBlocks, right *stgBlocks) int {
  130. return right.Count - left.Count
  131. })
  132. ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
  133. if len(ids) > userspaceCnt {
  134. ids = ids[:userspaceCnt]
  135. }
  136. return ids
  137. }
  138. func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
  139. sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
  140. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  141. })
  142. return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
  143. }
  144. func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
  145. sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
  146. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  147. })
  148. return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
  149. }
  150. func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
  151. sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
  152. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  153. })
  154. return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
  155. }
  156. func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContext, segCount int) []*userSpaceUsageInfo {
  157. sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
  158. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  159. })
  160. return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
  161. }
  162. func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
  163. type rechooseUserSpace struct {
  164. *userSpaceUsageInfo
  165. HasBlock bool
  166. }
  167. var rechooseStgs []*rechooseUserSpace
  168. for _, stg := range ctx.allUserSpaces {
  169. hasBlock := false
  170. for _, id := range ctx.mostBlockStgIDs {
  171. if id == stg.UserSpace.UserSpace.UserSpaceID {
  172. hasBlock = true
  173. break
  174. }
  175. }
  176. rechooseStgs = append(rechooseStgs, &rechooseUserSpace{
  177. userSpaceUsageInfo: stg,
  178. HasBlock: hasBlock,
  179. })
  180. }
  181. sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseUserSpace, right *rechooseUserSpace) int {
  182. // 已经缓存了文件块的节点优先选择
  183. v := sort2.CmpBool(right.HasBlock, left.HasBlock)
  184. if v != 0 {
  185. return v
  186. }
  187. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  188. })
  189. return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
  190. }
  191. func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
  192. type rechooseStg struct {
  193. *userSpaceUsageInfo
  194. CachedBlockIndex int
  195. }
  196. var rechooseStgs []*rechooseStg
  197. for _, stg := range ctx.allUserSpaces {
  198. cachedBlockIndex := -1
  199. for _, block := range obj.Blocks {
  200. if block.UserSpaceID == stg.UserSpace.UserSpace.UserSpaceID {
  201. cachedBlockIndex = block.Index
  202. break
  203. }
  204. }
  205. rechooseStgs = append(rechooseStgs, &rechooseStg{
  206. userSpaceUsageInfo: stg,
  207. CachedBlockIndex: cachedBlockIndex,
  208. })
  209. }
  210. sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseStg, right *rechooseStg) int {
  211. // 已经缓存了文件块的节点优先选择
  212. v := sort2.CmpBool(right.CachedBlockIndex > -1, left.CachedBlockIndex > -1)
  213. if v != 0 {
  214. return v
  215. }
  216. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  217. })
  218. // TODO 可以考虑选择已有块的节点时,能依然按照Index顺序选择
  219. return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
  220. }
  221. func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
  222. type rechooseStg struct {
  223. *userSpaceUsageInfo
  224. CachedBlockIndex int
  225. }
  226. var rechooseStgs []*rechooseStg
  227. for _, stg := range ctx.allUserSpaces {
  228. cachedBlockIndex := -1
  229. for _, block := range obj.Blocks {
  230. if block.UserSpaceID == stg.UserSpace.UserSpace.UserSpaceID {
  231. cachedBlockIndex = block.Index
  232. break
  233. }
  234. }
  235. rechooseStgs = append(rechooseStgs, &rechooseStg{
  236. userSpaceUsageInfo: stg,
  237. CachedBlockIndex: cachedBlockIndex,
  238. })
  239. }
  240. sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseStg, right *rechooseStg) int {
  241. // 已经缓存了文件块的节点优先选择
  242. v := sort2.CmpBool(right.CachedBlockIndex > -1, left.CachedBlockIndex > -1)
  243. if v != 0 {
  244. return v
  245. }
  246. return sort2.Cmp(right.AccessAmount, left.AccessAmount)
  247. })
  248. // TODO 可以考虑选择已有块的节点时,能依然按照Index顺序选择
  249. return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
  250. }
  251. func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUsageInfo) []*userSpaceUsageInfo {
  252. repeateCount := (count + len(stgs) - 1) / len(stgs)
  253. extendStgs := make([]*userSpaceUsageInfo, repeateCount*len(stgs))
  254. // 使用复制的方式将节点数扩充到要求的数量
  255. // 复制之后的结构:ABCD -> AAABBBCCCDDD
  256. for p := 0; p < repeateCount; p++ {
  257. for i, userspace := range stgs {
  258. putIdx := i*repeateCount + p
  259. extendStgs[putIdx] = userspace
  260. }
  261. }
  262. extendStgs = extendStgs[:count]
  263. var chosen []*userSpaceUsageInfo
  264. for len(chosen) < count {
  265. // 在每一轮内都选不同地区的节点,如果节点数不够,那么就再来一轮
  266. chosenLocations := make(map[jcstypes.Location]bool)
  267. for i, stg := range extendStgs {
  268. if stg == nil {
  269. continue
  270. }
  271. if chosenLocations[stg.UserSpace.UserSpace.Storage.GetLocation()] {
  272. continue
  273. }
  274. chosen = append(chosen, stg)
  275. chosenLocations[stg.UserSpace.UserSpace.Storage.GetLocation()] = true
  276. extendStgs[i] = nil
  277. }
  278. }
  279. return chosen
  280. }
  281. func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  282. if len(obj.Blocks) == 0 {
  283. return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
  284. }
  285. srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
  286. if !ok {
  287. return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
  288. }
  289. // 如果选择的备份节点都是同一个,那么就只要上传一次
  290. uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
  291. ft := ioswitch2.NewFromTo()
  292. ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
  293. for i, stg := range uploadStgs {
  294. ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
  295. }
  296. plans := exec.NewPlanBuilder()
  297. err := parser.Parse(ft, plans)
  298. if err != nil {
  299. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  300. }
  301. // TODO 添加依赖
  302. execCtx := exec.NewExecContext()
  303. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  304. ret, err := plans.Execute(execCtx).Wait(context.Background())
  305. if err != nil {
  306. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  307. }
  308. var blocks []jcstypes.ObjectBlock
  309. var blockChgs []datamap.BlockChange
  310. for i, stg := range uploadStgs {
  311. r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  312. blocks = append(blocks, jcstypes.ObjectBlock{
  313. ObjectID: obj.Object.ObjectID,
  314. Index: 0,
  315. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  316. FileHash: r.Hash,
  317. Size: r.Size,
  318. })
  319. blockChgs = append(blockChgs, &datamap.BlockChangeClone{
  320. BlockType: datamap.BlockTypeRaw,
  321. SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
  322. TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  323. TransferBytes: 1,
  324. })
  325. }
  326. // 删除原本的文件块
  327. blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
  328. Index: 0,
  329. UserSpaceID: obj.Blocks[0].UserSpaceID,
  330. })
  331. return &db.UpdatingObjectRedundancy{
  332. ObjectID: obj.Object.ObjectID,
  333. FileHash: obj.Object.FileHash,
  334. Size: obj.Object.Size,
  335. Redundancy: red,
  336. Blocks: blocks,
  337. }, &datamap.BodyBlockTransfer{
  338. ObjectID: obj.Object.ObjectID,
  339. PackageID: obj.Object.PackageID,
  340. BlockChanges: blockChgs,
  341. }, nil
  342. }
  343. func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  344. if len(obj.Blocks) == 0 {
  345. return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
  346. }
  347. srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
  348. if !ok {
  349. return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
  350. }
  351. ft := ioswitch2.NewFromTo()
  352. ft.ECParam = red
  353. ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
  354. for i := 0; i < red.N; i++ {
  355. ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].UserSpace, ioswitch2.ECStream(i), fmt.Sprintf("%d", i)))
  356. }
  357. plans := exec.NewPlanBuilder()
  358. err := parser.Parse(ft, plans)
  359. if err != nil {
  360. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  361. }
  362. execCtx := exec.NewExecContext()
  363. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  364. ioRet, err := plans.Execute(execCtx).Wait(context.Background())
  365. if err != nil {
  366. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  367. }
  368. var blocks []jcstypes.ObjectBlock
  369. var evtTargetBlocks []datamap.Block
  370. var evtBlockTrans []datamap.DataTransfer
  371. for i := 0; i < red.N; i++ {
  372. r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  373. blocks = append(blocks, jcstypes.ObjectBlock{
  374. ObjectID: obj.Object.ObjectID,
  375. Index: i,
  376. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  377. FileHash: r.Hash,
  378. Size: r.Size,
  379. })
  380. evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
  381. BlockType: datamap.BlockTypeEC,
  382. Index: i,
  383. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  384. })
  385. evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
  386. SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
  387. TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  388. TransferBytes: 1,
  389. })
  390. }
  391. return &db.UpdatingObjectRedundancy{
  392. ObjectID: obj.Object.ObjectID,
  393. FileHash: obj.Object.FileHash,
  394. Size: obj.Object.Size,
  395. Redundancy: red,
  396. Blocks: blocks,
  397. },
  398. &datamap.BodyBlockTransfer{
  399. ObjectID: obj.Object.ObjectID,
  400. PackageID: obj.Object.PackageID,
  401. BlockChanges: []datamap.BlockChange{
  402. &datamap.BlockChangeEnDecode{
  403. SourceBlocks: []datamap.Block{{
  404. BlockType: datamap.BlockTypeRaw,
  405. UserSpaceID: obj.Blocks[0].UserSpaceID,
  406. }},
  407. TargetBlocks: evtTargetBlocks,
  408. DataTransfers: evtBlockTrans,
  409. },
  410. // 删除原本的文件块
  411. &datamap.BlockChangeDeleted{
  412. Index: 0,
  413. UserSpaceID: obj.Blocks[0].UserSpaceID,
  414. },
  415. },
  416. }, nil
  417. }
  418. func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  419. if len(obj.Blocks) == 0 {
  420. return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
  421. }
  422. srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
  423. if !ok {
  424. return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
  425. }
  426. var toes []ioswitchlrc.To
  427. for i := 0; i < red.N; i++ {
  428. toes = append(toes, ioswitchlrc.NewToStorage(*uploadStgs[i].UserSpace, i, fmt.Sprintf("%d", i)))
  429. }
  430. plans := exec.NewPlanBuilder()
  431. err := lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *srcStg.UserSpace, -1), toes, plans)
  432. if err != nil {
  433. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  434. }
  435. execCtx := exec.NewExecContext()
  436. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  437. ioRet, err := plans.Execute(execCtx).Wait(context.Background())
  438. if err != nil {
  439. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  440. }
  441. var blocks []jcstypes.ObjectBlock
  442. var evtTargetBlocks []datamap.Block
  443. var evtBlockTrans []datamap.DataTransfer
  444. for i := 0; i < red.N; i++ {
  445. r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  446. blocks = append(blocks, jcstypes.ObjectBlock{
  447. ObjectID: obj.Object.ObjectID,
  448. Index: i,
  449. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  450. FileHash: r.Hash,
  451. Size: r.Size,
  452. })
  453. evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
  454. BlockType: datamap.BlockTypeEC,
  455. Index: i,
  456. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  457. })
  458. evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
  459. SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
  460. TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  461. TransferBytes: 1,
  462. })
  463. }
  464. return &db.UpdatingObjectRedundancy{
  465. ObjectID: obj.Object.ObjectID,
  466. FileHash: obj.Object.FileHash,
  467. Size: obj.Object.Size,
  468. Redundancy: red,
  469. Blocks: blocks,
  470. },
  471. &datamap.BodyBlockTransfer{
  472. ObjectID: obj.Object.ObjectID,
  473. PackageID: obj.Object.PackageID,
  474. BlockChanges: []datamap.BlockChange{
  475. &datamap.BlockChangeEnDecode{
  476. SourceBlocks: []datamap.Block{{
  477. BlockType: datamap.BlockTypeRaw,
  478. UserSpaceID: obj.Blocks[0].UserSpaceID,
  479. }},
  480. TargetBlocks: evtTargetBlocks,
  481. DataTransfers: evtBlockTrans,
  482. },
  483. // 删除原本的文件块
  484. &datamap.BlockChangeDeleted{
  485. Index: 0,
  486. UserSpaceID: obj.Blocks[0].UserSpaceID,
  487. },
  488. },
  489. },
  490. nil
  491. }
  492. func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  493. if len(obj.Blocks) == 0 {
  494. return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
  495. }
  496. srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
  497. if !ok {
  498. return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
  499. }
  500. // 如果选择的备份节点都是同一个,那么就只要上传一次
  501. uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
  502. ft := ioswitch2.NewFromTo()
  503. ft.SegmentParam = red
  504. ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
  505. for i, stg := range uploadStgs {
  506. ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.SegmentStream(i), fmt.Sprintf("%d", i)))
  507. }
  508. plans := exec.NewPlanBuilder()
  509. err := parser.Parse(ft, plans)
  510. if err != nil {
  511. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  512. }
  513. // TODO 添加依赖
  514. execCtx := exec.NewExecContext()
  515. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  516. ret, err := plans.Execute(execCtx).Wait(context.Background())
  517. if err != nil {
  518. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  519. }
  520. var blocks []jcstypes.ObjectBlock
  521. var evtTargetBlocks []datamap.Block
  522. var evtBlockTrans []datamap.DataTransfer
  523. for i, stg := range uploadStgs {
  524. r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  525. blocks = append(blocks, jcstypes.ObjectBlock{
  526. ObjectID: obj.Object.ObjectID,
  527. Index: i,
  528. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  529. FileHash: r.Hash,
  530. Size: r.Size,
  531. })
  532. evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
  533. BlockType: datamap.BlockTypeSegment,
  534. Index: i,
  535. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  536. })
  537. evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
  538. SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
  539. TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  540. TransferBytes: 1,
  541. })
  542. }
  543. return &db.UpdatingObjectRedundancy{
  544. ObjectID: obj.Object.ObjectID,
  545. FileHash: obj.Object.FileHash,
  546. Size: obj.Object.Size,
  547. Redundancy: red,
  548. Blocks: blocks,
  549. },
  550. &datamap.BodyBlockTransfer{
  551. ObjectID: obj.Object.ObjectID,
  552. PackageID: obj.Object.PackageID,
  553. BlockChanges: []datamap.BlockChange{
  554. &datamap.BlockChangeEnDecode{
  555. SourceBlocks: []datamap.Block{{
  556. BlockType: datamap.BlockTypeRaw,
  557. UserSpaceID: obj.Blocks[0].UserSpaceID,
  558. }},
  559. TargetBlocks: evtTargetBlocks,
  560. DataTransfers: evtBlockTrans,
  561. },
  562. // 删除原本的文件块
  563. &datamap.BlockChangeDeleted{
  564. Index: 0,
  565. UserSpaceID: obj.Blocks[0].UserSpaceID,
  566. },
  567. },
  568. },
  569. nil
  570. }
  571. func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  572. if len(obj.Blocks) == 0 {
  573. return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
  574. }
  575. srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
  576. if !ok {
  577. return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
  578. }
  579. // 如果选择的备份节点都是同一个,那么就只要上传一次
  580. uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
  581. ft := ioswitch2.NewFromTo()
  582. ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
  583. for i, stg := range uploadStgs {
  584. ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
  585. }
  586. plans := exec.NewPlanBuilder()
  587. err := parser.Parse(ft, plans)
  588. if err != nil {
  589. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  590. }
  591. // TODO 添加依赖
  592. execCtx := exec.NewExecContext()
  593. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  594. ret, err := plans.Execute(execCtx).Wait(context.Background())
  595. if err != nil {
  596. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  597. }
  598. var blocks []jcstypes.ObjectBlock
  599. var blockChgs []datamap.BlockChange
  600. for i, stg := range uploadStgs {
  601. r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  602. blocks = append(blocks, jcstypes.ObjectBlock{
  603. ObjectID: obj.Object.ObjectID,
  604. Index: 0,
  605. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  606. FileHash: r.Hash,
  607. Size: r.Size,
  608. })
  609. blockChgs = append(blockChgs, &datamap.BlockChangeClone{
  610. BlockType: datamap.BlockTypeRaw,
  611. SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
  612. TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  613. TransferBytes: 1,
  614. })
  615. }
  616. // 删除原本的文件块
  617. blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
  618. Index: 0,
  619. UserSpaceID: obj.Blocks[0].UserSpaceID,
  620. })
  621. return &db.UpdatingObjectRedundancy{
  622. ObjectID: obj.Object.ObjectID,
  623. FileHash: obj.Object.FileHash,
  624. Size: obj.Object.Size,
  625. Redundancy: red,
  626. Blocks: blocks,
  627. },
  628. &datamap.BodyBlockTransfer{
  629. ObjectID: obj.Object.ObjectID,
  630. PackageID: obj.Object.PackageID,
  631. BlockChanges: blockChgs,
  632. },
  633. nil
  634. }
  635. func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  636. return t.noneToEC(ctx, obj, red, uploadUserSpaces)
  637. }
  638. func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  639. var chosenBlocks []jcstypes.GrouppedObjectBlock
  640. var chosenBlockIndexes []int
  641. var chosenBlockStg []jcstypes.UserSpaceDetail
  642. for _, block := range obj.GroupBlocks() {
  643. if len(block.UserSpaceIDs) > 0 {
  644. // TODO 考虑选择最优的节点
  645. stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
  646. if !ok {
  647. continue
  648. }
  649. chosenBlocks = append(chosenBlocks, block)
  650. chosenBlockIndexes = append(chosenBlockIndexes, block.Index)
  651. chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
  652. }
  653. if len(chosenBlocks) == srcRed.K {
  654. break
  655. }
  656. }
  657. if len(chosenBlocks) < srcRed.K {
  658. return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
  659. }
  660. // 如果选择的备份节点都是同一个,那么就只要上传一次
  661. uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
  662. planBlder := exec.NewPlanBuilder()
  663. ft := ioswitch2.NewFromTo()
  664. ft.ECParam = srcRed
  665. for i, block := range chosenBlocks {
  666. ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, chosenBlockStg[i], ioswitch2.ECStream(block.Index)))
  667. }
  668. for i := range uploadStgs {
  669. ft.AddTo(ioswitch2.NewToShardStoreWithRange(*uploadStgs[i].UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i), math2.NewRange(0, obj.Object.Size)))
  670. }
  671. err := parser.Parse(ft, planBlder)
  672. if err != nil {
  673. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  674. }
  675. // TODO 添加依赖
  676. execCtx := exec.NewExecContext()
  677. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  678. ioRet, err := planBlder.Execute(execCtx).Wait(context.Background())
  679. if err != nil {
  680. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  681. }
  682. var blocks []jcstypes.ObjectBlock
  683. for i := range uploadStgs {
  684. r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
  685. blocks = append(blocks, jcstypes.ObjectBlock{
  686. ObjectID: obj.Object.ObjectID,
  687. Index: 0,
  688. UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
  689. FileHash: r.Hash,
  690. Size: r.Size,
  691. })
  692. }
  693. var evtSrcBlocks []datamap.Block
  694. var evtTargetBlocks []datamap.Block
  695. for i2, block := range chosenBlocks {
  696. evtSrcBlocks = append(evtSrcBlocks, datamap.Block{
  697. BlockType: datamap.BlockTypeEC,
  698. Index: block.Index,
  699. UserSpaceID: chosenBlockStg[i2].UserSpace.UserSpaceID,
  700. })
  701. }
  702. for _, stg := range uploadStgs {
  703. evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
  704. BlockType: datamap.BlockTypeRaw,
  705. Index: 0,
  706. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  707. })
  708. }
  709. var evtBlockTrans []datamap.DataTransfer
  710. for _, stg := range uploadStgs {
  711. for i2 := range chosenBlocks {
  712. evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
  713. SourceUserSpaceID: chosenBlockStg[i2].UserSpace.UserSpaceID,
  714. TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  715. TransferBytes: 1,
  716. })
  717. }
  718. }
  719. var blockChgs []datamap.BlockChange
  720. blockChgs = append(blockChgs, &datamap.BlockChangeEnDecode{
  721. SourceBlocks: evtSrcBlocks,
  722. TargetBlocks: evtTargetBlocks,
  723. DataTransfers: evtBlockTrans,
  724. })
  725. for _, block := range obj.Blocks {
  726. blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
  727. Index: block.Index,
  728. UserSpaceID: block.UserSpaceID,
  729. })
  730. }
  731. return &db.UpdatingObjectRedundancy{
  732. ObjectID: obj.Object.ObjectID,
  733. FileHash: obj.Object.FileHash,
  734. Size: obj.Object.Size,
  735. Redundancy: tarRed,
  736. Blocks: blocks,
  737. },
  738. &datamap.BodyBlockTransfer{
  739. ObjectID: obj.Object.ObjectID,
  740. PackageID: obj.Object.PackageID,
  741. BlockChanges: blockChgs,
  742. },
  743. nil
  744. }
  745. func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  746. grpBlocks := obj.GroupBlocks()
  747. var chosenBlocks []jcstypes.GrouppedObjectBlock
  748. var chosenBlockStg []jcstypes.UserSpaceDetail
  749. for _, block := range grpBlocks {
  750. if len(block.UserSpaceIDs) > 0 {
  751. stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
  752. if !ok {
  753. continue
  754. }
  755. chosenBlocks = append(chosenBlocks, block)
  756. chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
  757. }
  758. if len(chosenBlocks) == srcRed.K {
  759. break
  760. }
  761. }
  762. if len(chosenBlocks) < srcRed.K {
  763. return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
  764. }
  765. // 目前EC的参数都相同,所以可以不用重建出完整数据然后再分块,可以直接构建出目的节点需要的块
  766. planBlder := exec.NewPlanBuilder()
  767. var evtSrcBlocks []datamap.Block
  768. var evtTargetBlocks []datamap.Block
  769. ft := ioswitch2.NewFromTo()
  770. ft.ECParam = srcRed
  771. for i, block := range chosenBlocks {
  772. ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, chosenBlockStg[i], ioswitch2.ECStream(block.Index)))
  773. evtSrcBlocks = append(evtSrcBlocks, datamap.Block{
  774. BlockType: datamap.BlockTypeEC,
  775. Index: block.Index,
  776. UserSpaceID: chosenBlockStg[i].UserSpace.UserSpaceID,
  777. })
  778. }
  779. var newBlocks []jcstypes.ObjectBlock
  780. shouldUpdateBlocks := false
  781. for i, stg := range uploadUserSpaces {
  782. newBlock := jcstypes.ObjectBlock{
  783. ObjectID: obj.Object.ObjectID,
  784. Index: i,
  785. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  786. }
  787. grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
  788. // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
  789. if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
  790. newBlock.FileHash = grp.FileHash
  791. newBlock.Size = grp.Size
  792. newBlocks = append(newBlocks, newBlock)
  793. continue
  794. }
  795. shouldUpdateBlocks = true
  796. // 否则就要重建出这个节点需要的块
  797. // 输出只需要自己要保存的那一块
  798. ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.ECStream(i), fmt.Sprintf("%d", i)))
  799. evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
  800. BlockType: datamap.BlockTypeEC,
  801. Index: i,
  802. UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
  803. })
  804. newBlocks = append(newBlocks, newBlock)
  805. }
  806. err := parser.Parse(ft, planBlder)
  807. if err != nil {
  808. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  809. }
  810. // 如果没有任何Plan,Wait会直接返回成功
  811. execCtx := exec.NewExecContext()
  812. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  813. ret, err := planBlder.Execute(execCtx).Wait(context.Background())
  814. if err != nil {
  815. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  816. }
  817. if !shouldUpdateBlocks {
  818. return nil, nil, nil
  819. }
  820. for k, vs := range ret.Stored {
  821. idx, err := strconv.ParseInt(k, 10, 64)
  822. if err != nil {
  823. return nil, nil, fmt.Errorf("parsing result key %s as index: %w", k, err)
  824. }
  825. if len(vs) == 0 {
  826. continue
  827. }
  828. v := vs[0]
  829. r := v.(*ops2.FileInfoValue)
  830. newBlocks[idx].FileHash = r.Hash
  831. newBlocks[idx].Size = r.Size
  832. }
  833. var evtBlockTrans []datamap.DataTransfer
  834. for _, src := range evtSrcBlocks {
  835. for _, tar := range evtTargetBlocks {
  836. evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
  837. SourceUserSpaceID: src.UserSpaceID,
  838. TargetUserSpaceID: tar.UserSpaceID,
  839. TransferBytes: 1,
  840. })
  841. }
  842. }
  843. var blockChgs []datamap.BlockChange
  844. for _, block := range obj.Blocks {
  845. keep := lo.ContainsBy(newBlocks, func(newBlock jcstypes.ObjectBlock) bool {
  846. return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
  847. })
  848. if !keep {
  849. blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
  850. Index: block.Index,
  851. UserSpaceID: block.UserSpaceID,
  852. })
  853. }
  854. }
  855. blockChgs = append(blockChgs, &datamap.BlockChangeEnDecode{
  856. SourceBlocks: evtSrcBlocks,
  857. TargetBlocks: evtTargetBlocks,
  858. DataTransfers: evtBlockTrans,
  859. })
  860. return &db.UpdatingObjectRedundancy{
  861. ObjectID: obj.Object.ObjectID,
  862. FileHash: obj.Object.FileHash,
  863. Size: obj.Object.Size,
  864. Redundancy: tarRed,
  865. Blocks: newBlocks,
  866. },
  867. &datamap.BodyBlockTransfer{
  868. ObjectID: obj.Object.ObjectID,
  869. PackageID: obj.Object.PackageID,
  870. BlockChanges: blockChgs,
  871. },
  872. nil
  873. }
  874. func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.LRCRedundancy, tarRed *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  875. blocksGrpByIndex := obj.GroupBlocks()
  876. var lostBlocks []int
  877. var lostBlockGrps []int
  878. canGroupReconstruct := true
  879. allBlockFlags := make([]bool, srcRed.N)
  880. for _, block := range blocksGrpByIndex {
  881. allBlockFlags[block.Index] = true
  882. }
  883. for i, ok := range allBlockFlags {
  884. grpID := srcRed.FindGroup(i)
  885. if !ok {
  886. if grpID == -1 {
  887. canGroupReconstruct = false
  888. break
  889. }
  890. if len(lostBlocks) > 0 && lostBlockGrps[len(lostBlockGrps)-1] == grpID {
  891. canGroupReconstruct = false
  892. break
  893. }
  894. lostBlocks = append(lostBlocks, i)
  895. lostBlockGrps = append(lostBlockGrps, grpID)
  896. }
  897. }
  898. // TODO 产生BlockTransfer事件
  899. if canGroupReconstruct {
  900. // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadUserSpaces)
  901. }
  902. return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadUserSpaces)
  903. }
  904. /*
  905. TODO2 修复这一块的代码
  906. func (t *ChangeRedundancy) groupReconstructLRC(obj jcstypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
  907. grped := make(map[int]jcstypes.GrouppedObjectBlock)
  908. for _, b := range grpedBlocks {
  909. grped[b.Index] = b
  910. }
  911. plans := exec.NewPlanBuilder()
  912. for i := 0; i < len(lostBlocks); i++ {
  913. var froms []ioswitchlrc.From
  914. grpEles := red.GetGroupElements(lostBlockGrps[i])
  915. for _, ele := range grpEles {
  916. if ele == lostBlocks[i] {
  917. continue
  918. }
  919. froms = append(froms, ioswitchlrc.NewFromUserSpace(grped[ele].FileHash, nil, ele))
  920. }
  921. err := lrcparser.ReconstructGroup(froms, []ioswitchlrc.To{
  922. ioswitchlrc.NewToUserSpace(uploadUserSpaces[i].UserSpace, lostBlocks[i], fmt.Sprintf("%d", lostBlocks[i])),
  923. }, plans)
  924. if err != nil {
  925. return nil, fmt.Errorf("parsing plan: %w", err)
  926. }
  927. }
  928. fmt.Printf("plans: %v\n", plans)
  929. // 如果没有任何Plan,Wait会直接返回成功
  930. // TODO 添加依赖
  931. ret, err := plans.Execute(exec.NewExecContext()).Wait(context.TODO())
  932. if err != nil {
  933. return nil, fmt.Errorf("executing io plan: %w", err)
  934. }
  935. var newBlocks []jcstypes.ObjectBlock
  936. for _, i := range lostBlocks {
  937. newBlocks = append(newBlocks, jcstypes.ObjectBlock{
  938. ObjectID: obj.Object.ObjectID,
  939. Index: i,
  940. UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
  941. FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash,
  942. })
  943. }
  944. for _, b := range grpedBlocks {
  945. for _, hubID := range b.UserSpaceIDs {
  946. newBlocks = append(newBlocks, jcstypes.ObjectBlock{
  947. ObjectID: obj.Object.ObjectID,
  948. Index: b.Index,
  949. UserSpaceID: hubID,
  950. FileHash: b.FileHash,
  951. })
  952. }
  953. }
  954. return &db.UpdatingObjectRedundancy{
  955. ObjectID: obj.Object.ObjectID,
  956. Redundancy: red,
  957. Blocks: newBlocks,
  958. }, nil
  959. }
  960. */
  961. func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, grpBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
  962. var chosenBlocks []jcstypes.GrouppedObjectBlock
  963. var chosenBlockStg []jcstypes.UserSpaceDetail
  964. for _, block := range grpBlocks {
  965. if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
  966. stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
  967. if !ok {
  968. continue
  969. }
  970. chosenBlocks = append(chosenBlocks, block)
  971. chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
  972. }
  973. if len(chosenBlocks) == red.K {
  974. break
  975. }
  976. }
  977. if len(chosenBlocks) < red.K {
  978. return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
  979. }
  980. // 目前LRC的参数都相同,所以可以不用重建出完整数据然后再分块,可以直接构建出目的节点需要的块
  981. planBlder := exec.NewPlanBuilder()
  982. var froms []ioswitchlrc.From
  983. var toes []ioswitchlrc.To
  984. var newBlocks []jcstypes.ObjectBlock
  985. shouldUpdateBlocks := false
  986. for i, userspace := range uploadUserSpaces {
  987. newBlock := jcstypes.ObjectBlock{
  988. ObjectID: obj.Object.ObjectID,
  989. Index: i,
  990. UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
  991. }
  992. grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
  993. // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
  994. if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {
  995. newBlock.FileHash = grp.FileHash
  996. newBlock.Size = grp.Size
  997. newBlocks = append(newBlocks, newBlock)
  998. continue
  999. }
  1000. shouldUpdateBlocks = true
  1001. // 否则就要重建出这个节点需要的块
  1002. for i2, block := range chosenBlocks {
  1003. froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, chosenBlockStg[i2], block.Index))
  1004. }
  1005. // 输出只需要自己要保存的那一块
  1006. toes = append(toes, ioswitchlrc.NewToStorage(*userspace.UserSpace, i, fmt.Sprintf("%d", i)))
  1007. newBlocks = append(newBlocks, newBlock)
  1008. }
  1009. err := lrcparser.ReconstructAny(froms, toes, planBlder)
  1010. if err != nil {
  1011. return nil, nil, fmt.Errorf("parsing plan: %w", err)
  1012. }
  1013. fmt.Printf("plans: %v\n", planBlder)
  1014. // 如果没有任何Plan,Wait会直接返回成功
  1015. execCtx := exec.NewExecContext()
  1016. exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
  1017. ret, err := planBlder.Execute(execCtx).Wait(context.Background())
  1018. if err != nil {
  1019. return nil, nil, fmt.Errorf("executing io plan: %w", err)
  1020. }
  1021. if !shouldUpdateBlocks {
  1022. return nil, nil, nil
  1023. }
  1024. for k, vs := range ret.Stored {
  1025. idx, err := strconv.ParseInt(k, 10, 64)
  1026. if err != nil {
  1027. return nil, nil, fmt.Errorf("parsing result key %s as index: %w", k, err)
  1028. }
  1029. if len(vs) == 0 {
  1030. continue
  1031. }
  1032. v := vs[0]
  1033. r := v.(*ops2.FileInfoValue)
  1034. newBlocks[idx].FileHash = r.Hash
  1035. newBlocks[idx].Size = r.Size
  1036. }
  1037. // TODO 产生系统事件
  1038. return &db.UpdatingObjectRedundancy{
  1039. ObjectID: obj.Object.ObjectID,
  1040. FileHash: obj.Object.FileHash,
  1041. Size: obj.Object.Size,
  1042. Redundancy: red,
  1043. Blocks: newBlocks,
  1044. }, nil, nil
  1045. }

本项目旨在将云际存储公共基础设施化,使个人及企业可低门槛使用高效的云际存储服务(安装开箱即用云际存储客户端即可,无需关注其他组件的部署),同时支持用户灵活便捷定制云际存储的功能细节。