|
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231 |
- package ticktock
-
- import (
- "context"
- "fmt"
- "strconv"
-
- "github.com/samber/lo"
- "gitlink.org.cn/cloudream/common/pkgs/logger"
- "gitlink.org.cn/cloudream/common/utils/math2"
- "gitlink.org.cn/cloudream/common/utils/sort2"
- "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
- "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
- "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
- "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
- "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
- "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
- lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
- jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
- "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
- )
-
- func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail) (jcstypes.Redundancy, []*userSpaceUsageInfo) {
- switch obj.Object.Redundancy.(type) {
- case *jcstypes.NoneRedundancy:
- if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
- newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
- return &jcstypes.DefaultECRedundancy, newStgs
- }
-
- return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
-
- case *jcstypes.RepRedundancy:
- if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
- newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
- return &jcstypes.DefaultECRedundancy, newStgs
- }
-
- newSpaces := t.rechooseUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
- for _, s := range newSpaces {
- if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
- return &jcstypes.DefaultRepRedundancy, newSpaces
- }
- }
-
- return nil, nil
-
- case *jcstypes.ECRedundancy:
- if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
- return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
- }
-
- newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcstypes.DefaultECRedundancy)
- for i, s := range newSpaces {
- if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
- return &jcstypes.DefaultECRedundancy, newSpaces
- }
- }
-
- return nil, nil
-
- case *jcstypes.LRCRedundancy:
- newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcstypes.DefaultLRCRedundancy)
-
- for i, s := range newLRCStgs {
- if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
- return &jcstypes.DefaultLRCRedundancy, newLRCStgs
- }
- }
-
- return nil, nil
- }
- return nil, nil
- }
-
- func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, newRed jcstypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- log := logger.WithType[ChangeRedundancy]("TickTock")
-
- var updating *db.UpdatingObjectRedundancy
- var evt datamap.SysEventBody
- var err error
-
- switch srcRed := obj.Object.Redundancy.(type) {
- case *jcstypes.NoneRedundancy:
- switch newRed := newRed.(type) {
- case *jcstypes.RepRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
- updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)
-
- case *jcstypes.ECRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
- updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)
-
- case *jcstypes.LRCRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
- updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)
-
- case *jcstypes.SegmentRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
- updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
- }
-
- case *jcstypes.RepRedundancy:
- switch newRed := newRed.(type) {
- case *jcstypes.RepRedundancy:
- updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)
-
- case *jcstypes.ECRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
- updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
- }
-
- case *jcstypes.ECRedundancy:
- switch newRed := newRed.(type) {
- case *jcstypes.RepRedundancy:
- log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
- updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)
-
- case *jcstypes.ECRedundancy:
- updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
- }
-
- case *jcstypes.LRCRedundancy:
- switch newRed := newRed.(type) {
- case *jcstypes.LRCRedundancy:
- updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
- }
- }
-
- return updating, evt, err
- }
-
- // 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点
- func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcstypes.ObjectDetail, userspaceCnt int) []jcstypes.UserSpaceID {
- type stgBlocks struct {
- UserSpaceID jcstypes.UserSpaceID
- Count int
- }
-
- stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
- for _, obj := range objs {
- shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
- if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok && !shouldUseEC {
- for _, block := range obj.Blocks {
- if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
- stgBlocksMap[block.UserSpaceID] = &stgBlocks{
- UserSpaceID: block.UserSpaceID,
- Count: 0,
- }
- }
- stgBlocksMap[block.UserSpaceID].Count++
- }
- }
- }
-
- userspaces := lo.Values(stgBlocksMap)
- sort2.Sort(userspaces, func(left *stgBlocks, right *stgBlocks) int {
- return right.Count - left.Count
- })
-
- ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
- if len(ids) > userspaceCnt {
- ids = ids[:userspaceCnt]
- }
- return ids
- }
-
- func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
- sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
- }
-
- func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
- sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
- }
-
- func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
- sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
- }
-
- func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContext, segCount int) []*userSpaceUsageInfo {
- sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
- }
-
- func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
- type rechooseUserSpace struct {
- *userSpaceUsageInfo
- HasBlock bool
- }
-
- var rechooseStgs []*rechooseUserSpace
- for _, stg := range ctx.allUserSpaces {
- hasBlock := false
- for _, id := range ctx.mostBlockStgIDs {
- if id == stg.UserSpace.UserSpace.UserSpaceID {
- hasBlock = true
- break
- }
- }
-
- rechooseStgs = append(rechooseStgs, &rechooseUserSpace{
- userSpaceUsageInfo: stg,
- HasBlock: hasBlock,
- })
- }
-
- sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseUserSpace, right *rechooseUserSpace) int {
- // 已经缓存了文件块的节点优先选择
- v := sort2.CmpBool(right.HasBlock, left.HasBlock)
- if v != 0 {
- return v
- }
-
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
- }
-
- func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
- type rechooseStg struct {
- *userSpaceUsageInfo
- CachedBlockIndex int
- }
-
- var rechooseStgs []*rechooseStg
- for _, stg := range ctx.allUserSpaces {
- cachedBlockIndex := -1
- for _, block := range obj.Blocks {
- if block.UserSpaceID == stg.UserSpace.UserSpace.UserSpaceID {
- cachedBlockIndex = block.Index
- break
- }
- }
-
- rechooseStgs = append(rechooseStgs, &rechooseStg{
- userSpaceUsageInfo: stg,
- CachedBlockIndex: cachedBlockIndex,
- })
- }
-
- sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseStg, right *rechooseStg) int {
- // 已经缓存了文件块的节点优先选择
- v := sort2.CmpBool(right.CachedBlockIndex > -1, left.CachedBlockIndex > -1)
- if v != 0 {
- return v
- }
-
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- // TODO 可以考虑选择已有块的节点时,能依然按照Index顺序选择
- return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
- }
-
- func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
- type rechooseStg struct {
- *userSpaceUsageInfo
- CachedBlockIndex int
- }
-
- var rechooseStgs []*rechooseStg
- for _, stg := range ctx.allUserSpaces {
- cachedBlockIndex := -1
- for _, block := range obj.Blocks {
- if block.UserSpaceID == stg.UserSpace.UserSpace.UserSpaceID {
- cachedBlockIndex = block.Index
- break
- }
- }
-
- rechooseStgs = append(rechooseStgs, &rechooseStg{
- userSpaceUsageInfo: stg,
- CachedBlockIndex: cachedBlockIndex,
- })
- }
-
- sortedStgs := sort2.Sort(rechooseStgs, func(left *rechooseStg, right *rechooseStg) int {
- // 已经缓存了文件块的节点优先选择
- v := sort2.CmpBool(right.CachedBlockIndex > -1, left.CachedBlockIndex > -1)
- if v != 0 {
- return v
- }
-
- return sort2.Cmp(right.AccessAmount, left.AccessAmount)
- })
-
- // TODO 可以考虑选择已有块的节点时,能依然按照Index顺序选择
- return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
- }
-
- func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUsageInfo) []*userSpaceUsageInfo {
- repeateCount := (count + len(stgs) - 1) / len(stgs)
- extendStgs := make([]*userSpaceUsageInfo, repeateCount*len(stgs))
-
- // 使用复制的方式将节点数扩充到要求的数量
- // 复制之后的结构:ABCD -> AAABBBCCCDDD
- for p := 0; p < repeateCount; p++ {
- for i, userspace := range stgs {
- putIdx := i*repeateCount + p
- extendStgs[putIdx] = userspace
- }
- }
- extendStgs = extendStgs[:count]
-
- var chosen []*userSpaceUsageInfo
- for len(chosen) < count {
- // 在每一轮内都选不同地区的节点,如果节点数不够,那么就再来一轮
- chosenLocations := make(map[jcstypes.Location]bool)
- for i, stg := range extendStgs {
- if stg == nil {
- continue
- }
-
- if chosenLocations[stg.UserSpace.UserSpace.Storage.GetLocation()] {
- continue
- }
-
- chosen = append(chosen, stg)
- chosenLocations[stg.UserSpace.UserSpace.Storage.GetLocation()] = true
- extendStgs[i] = nil
- }
- }
-
- return chosen
- }
-
- func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- if len(obj.Blocks) == 0 {
- return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
- }
-
- srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
- if !ok {
- return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
- }
-
- // 如果选择的备份节点都是同一个,那么就只要上传一次
- uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
-
- ft := ioswitch2.NewFromTo()
- ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
- for i, stg := range uploadStgs {
- ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
- }
-
- plans := exec.NewPlanBuilder()
- err := parser.Parse(ft, plans)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- // TODO 添加依赖
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ret, err := plans.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
- var blockChgs []datamap.BlockChange
- for i, stg := range uploadStgs {
- r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: 0,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- blockChgs = append(blockChgs, &datamap.BlockChangeClone{
- BlockType: datamap.BlockTypeRaw,
- SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
- TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
-
- // 删除原本的文件块
- blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
- Index: 0,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- })
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: blocks,
- }, &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: blockChgs,
- }, nil
- }
-
- func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- if len(obj.Blocks) == 0 {
- return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
- }
-
- srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
- if !ok {
- return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
- }
-
- ft := ioswitch2.NewFromTo()
- ft.ECParam = red
- ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
- for i := 0; i < red.N; i++ {
- ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].UserSpace, ioswitch2.ECStream(i), fmt.Sprintf("%d", i)))
- }
- plans := exec.NewPlanBuilder()
- err := parser.Parse(ft, plans)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ioRet, err := plans.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
- var evtTargetBlocks []datamap.Block
- var evtBlockTrans []datamap.DataTransfer
- for i := 0; i < red.N; i++ {
- r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
- BlockType: datamap.BlockTypeEC,
- Index: i,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- })
- evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
- SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
- TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: blocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: []datamap.BlockChange{
- &datamap.BlockChangeEnDecode{
- SourceBlocks: []datamap.Block{{
- BlockType: datamap.BlockTypeRaw,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- }},
- TargetBlocks: evtTargetBlocks,
- DataTransfers: evtBlockTrans,
- },
-
- // 删除原本的文件块
- &datamap.BlockChangeDeleted{
- Index: 0,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- },
- },
- }, nil
- }
-
- func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- if len(obj.Blocks) == 0 {
- return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
- }
-
- srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
- if !ok {
- return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
- }
-
- var toes []ioswitchlrc.To
- for i := 0; i < red.N; i++ {
- toes = append(toes, ioswitchlrc.NewToStorage(*uploadStgs[i].UserSpace, i, fmt.Sprintf("%d", i)))
- }
-
- plans := exec.NewPlanBuilder()
- err := lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *srcStg.UserSpace, -1), toes, plans)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ioRet, err := plans.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
- var evtTargetBlocks []datamap.Block
- var evtBlockTrans []datamap.DataTransfer
- for i := 0; i < red.N; i++ {
- r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
- BlockType: datamap.BlockTypeEC,
- Index: i,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- })
- evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
- SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
- TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: blocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: []datamap.BlockChange{
- &datamap.BlockChangeEnDecode{
- SourceBlocks: []datamap.Block{{
- BlockType: datamap.BlockTypeRaw,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- }},
- TargetBlocks: evtTargetBlocks,
- DataTransfers: evtBlockTrans,
- },
-
- // 删除原本的文件块
- &datamap.BlockChangeDeleted{
- Index: 0,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- },
- },
- },
- nil
- }
-
- func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- if len(obj.Blocks) == 0 {
- return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
- }
-
- srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
- if !ok {
- return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
- }
-
- // 如果选择的备份节点都是同一个,那么就只要上传一次
- uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
-
- ft := ioswitch2.NewFromTo()
- ft.SegmentParam = red
- ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
- for i, stg := range uploadStgs {
- ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.SegmentStream(i), fmt.Sprintf("%d", i)))
- }
-
- plans := exec.NewPlanBuilder()
- err := parser.Parse(ft, plans)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- // TODO 添加依赖
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ret, err := plans.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
- var evtTargetBlocks []datamap.Block
- var evtBlockTrans []datamap.DataTransfer
- for i, stg := range uploadStgs {
- r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
- BlockType: datamap.BlockTypeSegment,
- Index: i,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- })
- evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
- SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
- TargetUserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: blocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: []datamap.BlockChange{
- &datamap.BlockChangeEnDecode{
- SourceBlocks: []datamap.Block{{
- BlockType: datamap.BlockTypeRaw,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- }},
- TargetBlocks: evtTargetBlocks,
- DataTransfers: evtBlockTrans,
- },
-
- // 删除原本的文件块
- &datamap.BlockChangeDeleted{
- Index: 0,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- },
- },
- },
- nil
- }
-
- func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- if len(obj.Blocks) == 0 {
- return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
- }
-
- srcStg, ok := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID]
- if !ok {
- return nil, nil, fmt.Errorf("userspace %v not found", obj.Blocks[0].UserSpaceID)
- }
-
- // 如果选择的备份节点都是同一个,那么就只要上传一次
- uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
-
- ft := ioswitch2.NewFromTo()
- ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
- for i, stg := range uploadStgs {
- ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
- }
-
- plans := exec.NewPlanBuilder()
- err := parser.Parse(ft, plans)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- // TODO 添加依赖
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ret, err := plans.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
- var blockChgs []datamap.BlockChange
- for i, stg := range uploadStgs {
- r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: 0,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- blockChgs = append(blockChgs, &datamap.BlockChangeClone{
- BlockType: datamap.BlockTypeRaw,
- SourceUserSpaceID: obj.Blocks[0].UserSpaceID,
- TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
-
- // 删除原本的文件块
- blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
- Index: 0,
- UserSpaceID: obj.Blocks[0].UserSpaceID,
- })
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: blocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: blockChgs,
- },
- nil
- }
-
- func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- return t.noneToEC(ctx, obj, red, uploadUserSpaces)
- }
-
- func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- var chosenBlocks []jcstypes.GrouppedObjectBlock
- var chosenBlockIndexes []int
- var chosenBlockStg []jcstypes.UserSpaceDetail
- for _, block := range obj.GroupBlocks() {
- if len(block.UserSpaceIDs) > 0 {
- // TODO 考虑选择最优的节点
- stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
- if !ok {
- continue
- }
-
- chosenBlocks = append(chosenBlocks, block)
- chosenBlockIndexes = append(chosenBlockIndexes, block.Index)
- chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
- }
-
- if len(chosenBlocks) == srcRed.K {
- break
- }
- }
-
- if len(chosenBlocks) < srcRed.K {
- return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
- }
-
- // 如果选择的备份节点都是同一个,那么就只要上传一次
- uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
-
- planBlder := exec.NewPlanBuilder()
- ft := ioswitch2.NewFromTo()
- ft.ECParam = srcRed
-
- for i, block := range chosenBlocks {
- ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, chosenBlockStg[i], ioswitch2.ECStream(block.Index)))
- }
-
- for i := range uploadStgs {
- ft.AddTo(ioswitch2.NewToShardStoreWithRange(*uploadStgs[i].UserSpace, ioswitch2.RawStream(), fmt.Sprintf("%d", i), math2.NewRange(0, obj.Object.Size)))
- }
-
- err := parser.Parse(ft, planBlder)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- // TODO 添加依赖
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ioRet, err := planBlder.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var blocks []jcstypes.ObjectBlock
-
- for i := range uploadStgs {
- r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
- blocks = append(blocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: 0,
- UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
- FileHash: r.Hash,
- Size: r.Size,
- })
- }
-
- var evtSrcBlocks []datamap.Block
- var evtTargetBlocks []datamap.Block
- for i2, block := range chosenBlocks {
- evtSrcBlocks = append(evtSrcBlocks, datamap.Block{
- BlockType: datamap.BlockTypeEC,
- Index: block.Index,
- UserSpaceID: chosenBlockStg[i2].UserSpace.UserSpaceID,
- })
- }
-
- for _, stg := range uploadStgs {
- evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
- BlockType: datamap.BlockTypeRaw,
- Index: 0,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- })
- }
-
- var evtBlockTrans []datamap.DataTransfer
- for _, stg := range uploadStgs {
- for i2 := range chosenBlocks {
- evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
- SourceUserSpaceID: chosenBlockStg[i2].UserSpace.UserSpaceID,
- TargetUserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- TransferBytes: 1,
- })
- }
- }
-
- var blockChgs []datamap.BlockChange
- blockChgs = append(blockChgs, &datamap.BlockChangeEnDecode{
- SourceBlocks: evtSrcBlocks,
- TargetBlocks: evtTargetBlocks,
- DataTransfers: evtBlockTrans,
- })
-
- for _, block := range obj.Blocks {
- blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
- Index: block.Index,
- UserSpaceID: block.UserSpaceID,
- })
- }
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: tarRed,
- Blocks: blocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: blockChgs,
- },
- nil
- }
-
- func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- grpBlocks := obj.GroupBlocks()
-
- var chosenBlocks []jcstypes.GrouppedObjectBlock
- var chosenBlockStg []jcstypes.UserSpaceDetail
- for _, block := range grpBlocks {
- if len(block.UserSpaceIDs) > 0 {
- stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
- if !ok {
- continue
- }
- chosenBlocks = append(chosenBlocks, block)
- chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
- }
-
- if len(chosenBlocks) == srcRed.K {
- break
- }
- }
-
- if len(chosenBlocks) < srcRed.K {
- return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
- }
-
- // 目前EC的参数都相同,所以可以不用重建出完整数据然后再分块,可以直接构建出目的节点需要的块
- planBlder := exec.NewPlanBuilder()
-
- var evtSrcBlocks []datamap.Block
- var evtTargetBlocks []datamap.Block
-
- ft := ioswitch2.NewFromTo()
- ft.ECParam = srcRed
-
- for i, block := range chosenBlocks {
- ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, chosenBlockStg[i], ioswitch2.ECStream(block.Index)))
-
- evtSrcBlocks = append(evtSrcBlocks, datamap.Block{
- BlockType: datamap.BlockTypeEC,
- Index: block.Index,
- UserSpaceID: chosenBlockStg[i].UserSpace.UserSpaceID,
- })
- }
-
- var newBlocks []jcstypes.ObjectBlock
- shouldUpdateBlocks := false
- for i, stg := range uploadUserSpaces {
- newBlock := jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- }
-
- grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
-
- // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
- if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
- newBlock.FileHash = grp.FileHash
- newBlock.Size = grp.Size
- newBlocks = append(newBlocks, newBlock)
- continue
- }
-
- shouldUpdateBlocks = true
-
- // 否则就要重建出这个节点需要的块
- // 输出只需要自己要保存的那一块
- ft.AddTo(ioswitch2.NewToShardStore(*stg.UserSpace, ioswitch2.ECStream(i), fmt.Sprintf("%d", i)))
-
- evtTargetBlocks = append(evtTargetBlocks, datamap.Block{
- BlockType: datamap.BlockTypeEC,
- Index: i,
- UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
- })
-
- newBlocks = append(newBlocks, newBlock)
- }
-
- err := parser.Parse(ft, planBlder)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- // 如果没有任何Plan,Wait会直接返回成功
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ret, err := planBlder.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- if !shouldUpdateBlocks {
- return nil, nil, nil
- }
-
- for k, vs := range ret.Stored {
- idx, err := strconv.ParseInt(k, 10, 64)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing result key %s as index: %w", k, err)
- }
- if len(vs) == 0 {
- continue
- }
- v := vs[0]
-
- r := v.(*ops2.FileInfoValue)
- newBlocks[idx].FileHash = r.Hash
- newBlocks[idx].Size = r.Size
- }
-
- var evtBlockTrans []datamap.DataTransfer
- for _, src := range evtSrcBlocks {
- for _, tar := range evtTargetBlocks {
- evtBlockTrans = append(evtBlockTrans, datamap.DataTransfer{
- SourceUserSpaceID: src.UserSpaceID,
- TargetUserSpaceID: tar.UserSpaceID,
- TransferBytes: 1,
- })
- }
- }
-
- var blockChgs []datamap.BlockChange
- for _, block := range obj.Blocks {
- keep := lo.ContainsBy(newBlocks, func(newBlock jcstypes.ObjectBlock) bool {
- return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
- })
- if !keep {
- blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{
- Index: block.Index,
- UserSpaceID: block.UserSpaceID,
- })
- }
- }
- blockChgs = append(blockChgs, &datamap.BlockChangeEnDecode{
- SourceBlocks: evtSrcBlocks,
- TargetBlocks: evtTargetBlocks,
- DataTransfers: evtBlockTrans,
- })
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: tarRed,
- Blocks: newBlocks,
- },
- &datamap.BodyBlockTransfer{
- ObjectID: obj.Object.ObjectID,
- PackageID: obj.Object.PackageID,
- BlockChanges: blockChgs,
- },
- nil
- }
-
- func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.LRCRedundancy, tarRed *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
-
- blocksGrpByIndex := obj.GroupBlocks()
-
- var lostBlocks []int
- var lostBlockGrps []int
- canGroupReconstruct := true
-
- allBlockFlags := make([]bool, srcRed.N)
- for _, block := range blocksGrpByIndex {
- allBlockFlags[block.Index] = true
- }
-
- for i, ok := range allBlockFlags {
- grpID := srcRed.FindGroup(i)
- if !ok {
- if grpID == -1 {
- canGroupReconstruct = false
- break
- }
-
- if len(lostBlocks) > 0 && lostBlockGrps[len(lostBlockGrps)-1] == grpID {
- canGroupReconstruct = false
- break
- }
-
- lostBlocks = append(lostBlocks, i)
- lostBlockGrps = append(lostBlockGrps, grpID)
- }
- }
-
- // TODO 产生BlockTransfer事件
-
- if canGroupReconstruct {
- // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadUserSpaces)
- }
-
- return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadUserSpaces)
- }
-
- /*
- TODO2 修复这一块的代码
-
- func (t *ChangeRedundancy) groupReconstructLRC(obj jcstypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
- grped := make(map[int]jcstypes.GrouppedObjectBlock)
- for _, b := range grpedBlocks {
- grped[b.Index] = b
- }
-
- plans := exec.NewPlanBuilder()
-
- for i := 0; i < len(lostBlocks); i++ {
- var froms []ioswitchlrc.From
- grpEles := red.GetGroupElements(lostBlockGrps[i])
- for _, ele := range grpEles {
- if ele == lostBlocks[i] {
- continue
- }
-
- froms = append(froms, ioswitchlrc.NewFromUserSpace(grped[ele].FileHash, nil, ele))
- }
-
- err := lrcparser.ReconstructGroup(froms, []ioswitchlrc.To{
- ioswitchlrc.NewToUserSpace(uploadUserSpaces[i].UserSpace, lostBlocks[i], fmt.Sprintf("%d", lostBlocks[i])),
- }, plans)
- if err != nil {
- return nil, fmt.Errorf("parsing plan: %w", err)
- }
- }
-
- fmt.Printf("plans: %v\n", plans)
-
- // 如果没有任何Plan,Wait会直接返回成功
- // TODO 添加依赖
- ret, err := plans.Execute(exec.NewExecContext()).Wait(context.TODO())
- if err != nil {
- return nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- var newBlocks []jcstypes.ObjectBlock
- for _, i := range lostBlocks {
- newBlocks = append(newBlocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
- FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash,
- })
- }
- for _, b := range grpedBlocks {
- for _, hubID := range b.UserSpaceIDs {
- newBlocks = append(newBlocks, jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: b.Index,
- UserSpaceID: hubID,
- FileHash: b.FileHash,
- })
- }
- }
-
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- Redundancy: red,
- Blocks: newBlocks,
- }, nil
- }
- */
- func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, grpBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
- var chosenBlocks []jcstypes.GrouppedObjectBlock
- var chosenBlockStg []jcstypes.UserSpaceDetail
-
- for _, block := range grpBlocks {
- if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
- stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
- if !ok {
- continue
- }
-
- chosenBlocks = append(chosenBlocks, block)
- chosenBlockStg = append(chosenBlockStg, *stg.UserSpace)
- }
-
- if len(chosenBlocks) == red.K {
- break
- }
- }
-
- if len(chosenBlocks) < red.K {
- return nil, nil, fmt.Errorf("no enough blocks to reconstruct the original file data")
- }
-
- // 目前LRC的参数都相同,所以可以不用重建出完整数据然后再分块,可以直接构建出目的节点需要的块
- planBlder := exec.NewPlanBuilder()
-
- var froms []ioswitchlrc.From
- var toes []ioswitchlrc.To
- var newBlocks []jcstypes.ObjectBlock
- shouldUpdateBlocks := false
- for i, userspace := range uploadUserSpaces {
- newBlock := jcstypes.ObjectBlock{
- ObjectID: obj.Object.ObjectID,
- Index: i,
- UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
- }
-
- grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })
-
- // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
- if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {
- newBlock.FileHash = grp.FileHash
- newBlock.Size = grp.Size
- newBlocks = append(newBlocks, newBlock)
- continue
- }
-
- shouldUpdateBlocks = true
-
- // 否则就要重建出这个节点需要的块
-
- for i2, block := range chosenBlocks {
- froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, chosenBlockStg[i2], block.Index))
- }
-
- // 输出只需要自己要保存的那一块
- toes = append(toes, ioswitchlrc.NewToStorage(*userspace.UserSpace, i, fmt.Sprintf("%d", i)))
-
- newBlocks = append(newBlocks, newBlock)
- }
-
- err := lrcparser.ReconstructAny(froms, toes, planBlder)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing plan: %w", err)
- }
-
- fmt.Printf("plans: %v\n", planBlder)
-
- // 如果没有任何Plan,Wait会直接返回成功
- execCtx := exec.NewExecContext()
- exec.SetValueByType(execCtx, ctx.ticktock.stgPool)
- ret, err := planBlder.Execute(execCtx).Wait(context.Background())
- if err != nil {
- return nil, nil, fmt.Errorf("executing io plan: %w", err)
- }
-
- if !shouldUpdateBlocks {
- return nil, nil, nil
- }
-
- for k, vs := range ret.Stored {
- idx, err := strconv.ParseInt(k, 10, 64)
- if err != nil {
- return nil, nil, fmt.Errorf("parsing result key %s as index: %w", k, err)
- }
- if len(vs) == 0 {
- continue
- }
- v := vs[0]
-
- r := v.(*ops2.FileInfoValue)
- newBlocks[idx].FileHash = r.Hash
- newBlocks[idx].Size = r.Size
- }
-
- // TODO 产生系统事件
- return &db.UpdatingObjectRedundancy{
- ObjectID: obj.Object.ObjectID,
- FileHash: obj.Object.FileHash,
- Size: obj.Object.Size,
- Redundancy: red,
- Blocks: newBlocks,
- }, nil, nil
- }
|