package ticktock import ( "context" "fmt" "math" "math/rand" "sync" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkgs/bitmap" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/common/consts" "gitlink.org.cn/cloudream/jcs-pub/common/models/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/distlock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" ) func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg clitypes.PackageDetail, objs []clitypes.ObjectDetail, reen *distlock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) { log := logger.WithType[ChangeRedundancy]("TickTock") var readerStgIDs []clitypes.UserSpaceID for _, space := range execCtx.allUserSpaces { // TODO 可以考虑做成配置 if space.AccessAmount >= float64(pkg.ObjectCount/2) { readerStgIDs = append(readerStgIDs, space.UserSpace.UserSpace.UserSpaceID) } } // 只对ec和rep对象进行处理 var ecObjects []clitypes.ObjectDetail var repObjects []clitypes.ObjectDetail for _, obj := range objs { if _, ok := obj.Object.Redundancy.(*clitypes.ECRedundancy); ok { ecObjects = append(ecObjects, obj) } else if _, ok := obj.Object.Redundancy.(*clitypes.RepRedundancy); ok { repObjects = append(repObjects, obj) } } planBld := exec.NewPlanBuilder() planningStgIDs := make(map[clitypes.UserSpaceID]bool) var sysEvents []datamap.SysEventBody // 对于rep对象,统计出所有对象块分布最多的两个节点,用这两个节点代表所有rep对象块的分布,去进行退火算法 var repObjectsUpdating []db.UpdatingObjectRedundancy repMostHubIDs := t.summaryRepObjectBlockNodes(repObjects) solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{ totalBlockCount: 1, minBlockCnt: 1, pinnedAt: repMostHubIDs, blocks: nil, }) for _, obj := range repObjects { repObjectsUpdating = append(repObjectsUpdating, t.makePlansForRepObject(execCtx, solu, obj, planBld, planningStgIDs)) sysEvents = append(sysEvents, t.generateSysEventForRepObject(solu, obj)...) } // 对于ec对象,则每个对象单独进行退火算法 var ecObjectsUpdating []db.UpdatingObjectRedundancy for _, obj := range ecObjects { ecRed := obj.Object.Redundancy.(*clitypes.ECRedundancy) solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{ totalBlockCount: ecRed.N, minBlockCnt: ecRed.K, pinnedAt: obj.PinnedAt, blocks: obj.Blocks, }) ecObjectsUpdating = append(ecObjectsUpdating, t.makePlansForECObject(execCtx, solu, obj, planBld, planningStgIDs)) sysEvents = append(sysEvents, t.generateSysEventForECObject(solu, obj)...) } ioSwRets, err := t.executePlans(execCtx, planBld, planningStgIDs, reen) if err != nil { log.Warn(err.Error()) return nil, nil, fmt.Errorf("execute plans: %w", err) } // 根据按照方案进行调整的结果,填充更新元数据的命令 for i := range ecObjectsUpdating { t.populateECObjectEntry(&ecObjectsUpdating[i], ecObjects[i], ioSwRets) } return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil } func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []clitypes.ObjectDetail) []clitypes.UserSpaceID { type stgBlocks struct { UserSpaceID clitypes.UserSpaceID Count int } stgBlocksMap := make(map[clitypes.UserSpaceID]*stgBlocks) for _, obj := range objs { cacheBlockStgs := make(map[clitypes.UserSpaceID]bool) for _, block := range obj.Blocks { if _, ok := stgBlocksMap[block.UserSpaceID]; !ok { stgBlocksMap[block.UserSpaceID] = &stgBlocks{ UserSpaceID: block.UserSpaceID, Count: 0, } } stgBlocksMap[block.UserSpaceID].Count++ cacheBlockStgs[block.UserSpaceID] = true } for _, hubID := range obj.PinnedAt { if cacheBlockStgs[hubID] { continue } if _, ok := stgBlocksMap[hubID]; !ok { stgBlocksMap[hubID] = &stgBlocks{ UserSpaceID: hubID, Count: 0, } } stgBlocksMap[hubID].Count++ } } stgs := lo.Values(stgBlocksMap) sort2.Sort(stgs, func(left *stgBlocks, right *stgBlocks) int { return right.Count - left.Count }) // 只选出块数超过一半的节点,但要保证至少有两个节点 for i := 2; i < len(stgs); i++ { if stgs[i].Count < len(objs)/2 { stgs = stgs[:i] break } } return lo.Map(stgs, func(item *stgBlocks, idx int) clitypes.UserSpaceID { return item.UserSpaceID }) } type annealingState struct { ctx *changeRedundancyContext readerStgIDs []clitypes.UserSpaceID // 近期可能访问此对象的节点 stgsSortedByReader map[clitypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序 object annealingObject // 进行退火的对象 blockList []objectBlock // 排序后的块分布情况 stgBlockBitmaps map[clitypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块 stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度 maxScore float64 // 搜索过程中得到过的最大分数 maxScoreRmBlocks []bool // 最大分数对应的删除方案 rmBlocks []bool // 当前删除方案 inversedIndex int // 当前删除方案是从上一次的方案改动哪个flag而来的 lastDisasterTolerance float64 // 上一次方案的容灾度 lastSpaceCost float64 // 上一次方案的冗余度 lastMinAccessCost float64 // 上一次方案的最小访问费用 lastScore float64 // 上一次方案的分数 } type objectBlock struct { Index int UserSpaceID clitypes.UserSpaceID HasEntity bool // 节点拥有实际的文件数据块 HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块 FileHash clitypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值 Size int64 // 块大小 } type stgDist struct { UserSpaceID clitypes.UserSpaceID Distance float64 } type combinatorialTree struct { nodes []combinatorialTreeNode blocksMaps map[int]bitmap.Bitmap64 stgIDToLocalStgID map[clitypes.UserSpaceID]int localStgIDToStgID []clitypes.UserSpaceID } type annealingObject struct { totalBlockCount int minBlockCnt int pinnedAt []clitypes.UserSpaceID blocks []clitypes.ObjectBlock } const ( iterActionNone = 0 iterActionSkip = 1 iterActionBreak = 2 ) func newCombinatorialTree(stgBlocksMaps map[clitypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree { tree := combinatorialTree{ blocksMaps: make(map[int]bitmap.Bitmap64), stgIDToLocalStgID: make(map[clitypes.UserSpaceID]int), } tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps))) for id, mp := range stgBlocksMaps { tree.stgIDToLocalStgID[id] = len(tree.localStgIDToStgID) tree.blocksMaps[len(tree.localStgIDToStgID)] = *mp tree.localStgIDToStgID = append(tree.localStgIDToStgID, id) } tree.nodes[0].localHubID = -1 index := 1 tree.initNode(0, &tree.nodes[0], &index) return tree } func (t *combinatorialTree) initNode(minAvaiLocalHubID int, parent *combinatorialTreeNode, index *int) { for i := minAvaiLocalHubID; i < len(t.stgIDToLocalStgID); i++ { curIndex := *index *index++ bitMp := t.blocksMaps[i] bitMp.Or(&parent.blocksBitmap) t.nodes[curIndex] = combinatorialTreeNode{ localHubID: i, parent: parent, blocksBitmap: bitMp, } t.initNode(i+1, &t.nodes[curIndex], index) } } // 获得索引指定的节点所在的层 func (t *combinatorialTree) GetDepth(index int) int { depth := 0 // 反复判断节点在哪个子树。从左到右,子树节点的数量呈现8 4 2的变化,由此可以得到每个子树的索引值的范围 subTreeCount := 1 << len(t.stgIDToLocalStgID) for index > 0 { if index < subTreeCount { // 定位到一个子树后,深度+1,然后进入这个子树,使用同样的方法再进行定位。 // 进入子树后需要将索引值-1,因为要去掉子树的根节点 index-- depth++ } else { // 如果索引值不在这个子树范围内,则将值减去子树的节点数量, // 这样每一次都可以视为使用同样的逻辑对不同大小的树进行判断。 index -= subTreeCount } subTreeCount >>= 1 } return depth } // 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。 // 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点 func (t *combinatorialTree) UpdateBitmap(stgID clitypes.UserSpaceID, mp bitmap.Bitmap64, k int) { t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp // 首先定义两种遍历树节点时的移动方式: // 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1 // 2. 水平移动:从一个节点移动到它右边的兄弟节点。每移动一步,根据它所在的深度,index+8,+4,+2 // LocalID从0开始,将其+1后得到移动步数steps。 // 将移动步数拆成多部分,分配到上述的两种移动方式上,并进行任意组合,且保证第一次为至少进行一次的竖直移动,移动之后的节点都会是同一个计算中心节点。 steps := t.stgIDToLocalStgID[stgID] + 1 for d := 1; d <= steps; d++ { t.iterCombBits(len(t.stgIDToLocalStgID)-1, steps-d, 0, func(i int) { index := d + i node := &t.nodes[index] newMp := t.blocksMaps[node.localHubID] newMp.Or(&node.parent.blocksBitmap) node.blocksBitmap = newMp if newMp.Weight() >= k { return } t.iterChildren(index, func(index, parentIndex, depth int) int { curNode := &t.nodes[index] parentNode := t.nodes[parentIndex] newMp := t.blocksMaps[curNode.localHubID] newMp.Or(&parentNode.blocksBitmap) curNode.blocksBitmap = newMp if newMp.Weight() >= k { return iterActionSkip } return iterActionNone }) }) } } // 遍历树,找到至少拥有K个块的树节点的最大深度 func (t *combinatorialTree) FindKBlocksMaxDepth(k int) int { maxDepth := -1 t.iterChildren(0, func(index, parentIndex, depth int) int { if t.nodes[index].blocksBitmap.Weight() >= k { if maxDepth < depth { maxDepth = depth } return iterActionSkip } // 如果到了叶子节点,还没有找到K个块,那就认为要满足K个块,至少需要再多一个节点,即深度+1。 // 由于遍历时采用的是深度优先的算法,因此遍历到这个叶子节点时,叶子节点再加一个节点的组合已经在前面搜索过, // 所以用当前叶子节点深度+1来作为当前分支的结果就可以,即使当前情况下增加任意一个节点依然不够K块, // 可以使用同样的思路去递推到当前叶子节点增加两个块的情况。 if t.nodes[index].localHubID == len(t.stgIDToLocalStgID)-1 { if maxDepth < depth+1 { maxDepth = depth + 1 } } return iterActionNone }) if maxDepth == -1 || maxDepth > len(t.stgIDToLocalStgID) { return len(t.stgIDToLocalStgID) } return maxDepth } func (t *combinatorialTree) iterCombBits(width int, count int, offset int, callback func(int)) { if count == 0 { callback(offset) return } for b := width; b >= count; b-- { t.iterCombBits(b-1, count-1, offset+(1<>= 1 } } func (t *combinatorialTree) itering(index int, parentIndex int, depth int, do func(index int, parentIndex int, depth int) int) int { act := do(index, parentIndex, depth) if act == iterActionBreak { return act } if act == iterActionSkip { return iterActionNone } curNode := &t.nodes[index] childIndex := index + 1 childCounts := len(t.stgIDToLocalStgID) - 1 - curNode.localHubID if childCounts == 0 { return iterActionNone } childTreeNodeCnt := 1 << (childCounts - 1) for c := 0; c < childCounts; c++ { act = t.itering(childIndex, index, depth+1, do) if act == iterActionBreak { return act } childIndex += childTreeNodeCnt childTreeNodeCnt >>= 1 } return iterActionNone } type combinatorialTreeNode struct { localHubID int parent *combinatorialTreeNode blocksBitmap bitmap.Bitmap64 // 选择了这个中心之后,所有中心一共包含多少种块 } type annealingSolution struct { blockList []objectBlock // 所有节点的块分布情况 rmBlocks []bool // 要删除哪些块 disasterTolerance float64 // 本方案的容灾度 spaceCost float64 // 本方案的冗余度 minAccessCost float64 // 本方案的最小访问费用 } func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []clitypes.UserSpaceID, object annealingObject) annealingSolution { state := &annealingState{ ctx: ctx, readerStgIDs: readerStgIDs, stgsSortedByReader: make(map[clitypes.UserSpaceID][]stgDist), object: object, stgBlockBitmaps: make(map[clitypes.UserSpaceID]*bitmap.Bitmap64), } t.initBlockList(state) if state.blockList == nil { return annealingSolution{} } t.initNodeBlockBitmap(state) t.sortNodeByReaderDistance(state) state.rmBlocks = make([]bool, len(state.blockList)) state.inversedIndex = -1 state.stgCombTree = newCombinatorialTree(state.stgBlockBitmaps) state.lastScore = t.calcScore(state) state.maxScore = state.lastScore state.maxScoreRmBlocks = lo2.ArrayClone(state.rmBlocks) // 模拟退火算法的温度 curTemp := state.lastScore // 结束温度 finalTemp := curTemp * 0.2 // 冷却率 coolingRate := 0.95 for curTemp > finalTemp { state.inversedIndex = rand.Intn(len(state.rmBlocks)) block := state.blockList[state.inversedIndex] state.rmBlocks[state.inversedIndex] = !state.rmBlocks[state.inversedIndex] state.stgBlockBitmaps[block.UserSpaceID].Set(block.Index, !state.rmBlocks[state.inversedIndex]) state.stgCombTree.UpdateBitmap(block.UserSpaceID, *state.stgBlockBitmaps[block.UserSpaceID], state.object.minBlockCnt) curScore := t.calcScore(state) dScore := curScore - state.lastScore // 如果新方案比旧方案得分低,且没有要求强制接受新方案,那么就将变化改回去 if curScore == 0 || (dScore < 0 && !t.alwaysAccept(curTemp, dScore, coolingRate)) { state.rmBlocks[state.inversedIndex] = !state.rmBlocks[state.inversedIndex] state.stgBlockBitmaps[block.UserSpaceID].Set(block.Index, !state.rmBlocks[state.inversedIndex]) state.stgCombTree.UpdateBitmap(block.UserSpaceID, *state.stgBlockBitmaps[block.UserSpaceID], state.object.minBlockCnt) // fmt.Printf("\n") } else { // fmt.Printf(" accept!\n") state.lastScore = curScore if state.maxScore < curScore { state.maxScore = state.lastScore state.maxScoreRmBlocks = lo2.ArrayClone(state.rmBlocks) } } curTemp *= coolingRate } // fmt.Printf("final: %v\n", state.maxScoreRmBlocks) return annealingSolution{ blockList: state.blockList, rmBlocks: state.maxScoreRmBlocks, disasterTolerance: state.lastDisasterTolerance, spaceCost: state.lastSpaceCost, minAccessCost: state.lastMinAccessCost, } } func (t *ChangeRedundancy) initBlockList(ctx *annealingState) { blocksMap := make(map[clitypes.UserSpaceID][]objectBlock) // 先生成所有的影子块 for _, pinned := range ctx.object.pinnedAt { blocks := make([]objectBlock, 0, ctx.object.totalBlockCount) for i := 0; i < ctx.object.totalBlockCount; i++ { blocks = append(blocks, objectBlock{ Index: i, UserSpaceID: pinned, HasShadow: true, }) } blocksMap[pinned] = blocks } // 再填充实际块 for _, b := range ctx.object.blocks { blocks := blocksMap[b.UserSpaceID] has := false for i := range blocks { if blocks[i].Index == b.Index { blocks[i].HasEntity = true blocks[i].FileHash = b.FileHash has = true break } } if has { continue } blocks = append(blocks, objectBlock{ Index: b.Index, UserSpaceID: b.UserSpaceID, HasEntity: true, FileHash: b.FileHash, Size: b.Size, }) blocksMap[b.UserSpaceID] = blocks } var sortedBlocks []objectBlock for _, bs := range blocksMap { sortedBlocks = append(sortedBlocks, bs...) } sortedBlocks = sort2.Sort(sortedBlocks, func(left objectBlock, right objectBlock) int { d := left.UserSpaceID - right.UserSpaceID if d != 0 { return int(d) } return left.Index - right.Index }) ctx.blockList = sortedBlocks } func (t *ChangeRedundancy) initNodeBlockBitmap(state *annealingState) { for _, b := range state.blockList { mp, ok := state.stgBlockBitmaps[b.UserSpaceID] if !ok { nb := bitmap.Bitmap64(0) mp = &nb state.stgBlockBitmaps[b.UserSpaceID] = mp } mp.Set(b.Index, true) } } func (t *ChangeRedundancy) sortNodeByReaderDistance(state *annealingState) { for _, r := range state.readerStgIDs { var nodeDists []stgDist for n := range state.stgBlockBitmaps { if r == n { // 同节点时距离视为0.1 nodeDists = append(nodeDists, stgDist{ UserSpaceID: n, Distance: consts.StorageDistanceSameStorage, }) } else if state.ctx.allUserSpaces[r].UserSpace.MasterHub.LocationID == state.ctx.allUserSpaces[n].UserSpace.MasterHub.LocationID { // 同地区时距离视为1 nodeDists = append(nodeDists, stgDist{ UserSpaceID: n, Distance: consts.StorageDistanceSameLocation, }) } else { // 不同地区时距离视为5 nodeDists = append(nodeDists, stgDist{ UserSpaceID: n, Distance: consts.StorageDistanceOther, }) } } state.stgsSortedByReader[r] = sort2.Sort(nodeDists, func(left, right stgDist) int { return sort2.Cmp(left.Distance, right.Distance) }) } } func (t *ChangeRedundancy) calcScore(state *annealingState) float64 { dt := t.calcDisasterTolerance(state) ac := t.calcMinAccessCost(state) sc := t.calcSpaceCost(state) state.lastDisasterTolerance = dt state.lastMinAccessCost = ac state.lastSpaceCost = sc dtSc := 1.0 if dt < 1 { dtSc = 0 } else if dt >= 2 { dtSc = 1.5 } newSc := 0.0 if dt == 0 || ac == 0 { newSc = 0 } else { newSc = dtSc / (sc * ac) } // fmt.Printf("solu: %v, cur: %v, dt: %v, ac: %v, sc: %v \n", state.rmBlocks, newSc, dt, ac, sc) return newSc } // 计算容灾度 func (t *ChangeRedundancy) calcDisasterTolerance(state *annealingState) float64 { if state.inversedIndex != -1 { node := state.blockList[state.inversedIndex] state.stgCombTree.UpdateBitmap(node.UserSpaceID, *state.stgBlockBitmaps[node.UserSpaceID], state.object.minBlockCnt) } return float64(len(state.stgBlockBitmaps) - state.stgCombTree.FindKBlocksMaxDepth(state.object.minBlockCnt)) } // 计算最小访问数据的代价 func (t *ChangeRedundancy) calcMinAccessCost(state *annealingState) float64 { cost := math.MaxFloat64 for _, reader := range state.readerStgIDs { tarNodes := state.stgsSortedByReader[reader] gotBlocks := bitmap.Bitmap64(0) thisCost := 0.0 for _, tar := range tarNodes { tarNodeMp := state.stgBlockBitmaps[tar.UserSpaceID] // 只需要从目的节点上获得缺少的块 curWeigth := gotBlocks.Weight() // 下面的if会在拿到k个块之后跳出循环,所以or多了块也没关系 gotBlocks.Or(tarNodeMp) // 但是算读取块的消耗时,不能多算,最多算读了k个块的消耗 willGetBlocks := math2.Min(gotBlocks.Weight()-curWeigth, state.object.minBlockCnt-curWeigth) thisCost += float64(willGetBlocks) * float64(tar.Distance) if gotBlocks.Weight() >= state.object.minBlockCnt { break } } if gotBlocks.Weight() >= state.object.minBlockCnt { cost = math.Min(cost, thisCost) } } return cost } // 计算冗余度 func (t *ChangeRedundancy) calcSpaceCost(ctx *annealingState) float64 { blockCount := 0 for i, b := range ctx.blockList { if ctx.rmBlocks[i] { continue } if b.HasEntity { blockCount++ } if b.HasShadow { blockCount++ } } // 所有算力中心上拥有的块的总数 / 一个对象被分成了几个块 return float64(blockCount) / float64(ctx.object.minBlockCnt) } // 如果新方案得分比旧方案小,那么在一定概率内也接受新方案 func (t *ChangeRedundancy) alwaysAccept(curTemp float64, dScore float64, coolingRate float64) bool { v := math.Exp(dScore / curTemp / coolingRate) // fmt.Printf(" -- chance: %v, temp: %v", v, curTemp) return v > rand.Float64() } func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj clitypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[clitypes.UserSpaceID]bool) db.UpdatingObjectRedundancy { entry := db.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, FileHash: obj.Object.FileHash, Size: obj.Object.Size, Redundancy: obj.Object.Redundancy, } ft := ioswitch2.NewFromTo() fromStg := ctx.allUserSpaces[obj.Blocks[0].UserSpaceID].UserSpace ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg.MasterHub, *fromStg, ioswitch2.RawStream())) for i, f := range solu.rmBlocks { hasCache := lo.ContainsBy(obj.Blocks, func(b clitypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) || lo.ContainsBy(obj.PinnedAt, func(n clitypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID }) willRm := f if !willRm { // 如果对象在退火后要保留副本的节点没有副本,则需要在这个节点创建副本 if !hasCache { toStg := ctx.allUserSpaces[solu.blockList[i].UserSpaceID].UserSpace ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, *toStg, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID))) planningHubIDs[solu.blockList[i].UserSpaceID] = true } entry.Blocks = append(entry.Blocks, clitypes.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: solu.blockList[i].Index, UserSpaceID: solu.blockList[i].UserSpaceID, FileHash: obj.Object.FileHash, Size: solu.blockList[i].Size, }) } } err := parser.Parse(ft, planBld) if err != nil { // TODO 错误处理 } return entry } func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj clitypes.ObjectDetail) []datamap.SysEventBody { var blockChgs []datamap.BlockChange for i, f := range solu.rmBlocks { hasCache := lo.ContainsBy(obj.Blocks, func(b clitypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) || lo.ContainsBy(obj.PinnedAt, func(n clitypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID }) willRm := f if !willRm { // 如果对象在退火后要保留副本的节点没有副本,则需要在这个节点创建副本 if !hasCache { blockChgs = append(blockChgs, &datamap.BlockChangeClone{ BlockType: datamap.BlockTypeRaw, SourceUserSpaceID: obj.Blocks[0].UserSpaceID, TargetUserSpaceID: solu.blockList[i].UserSpaceID, }) } } else { blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{ Index: 0, UserSpaceID: solu.blockList[i].UserSpaceID, }) } } transEvt := &datamap.BodyBlockTransfer{ ObjectID: obj.Object.ObjectID, PackageID: obj.Object.PackageID, BlockChanges: blockChgs, } var blockDist []datamap.BlockDistributionObjectInfo for i, f := range solu.rmBlocks { if !f { blockDist = append(blockDist, datamap.BlockDistributionObjectInfo{ BlockType: datamap.BlockTypeRaw, Index: 0, UserSpaceID: solu.blockList[i].UserSpaceID, }) } } distEvt := &datamap.BodyBlockDistribution{ ObjectID: obj.Object.ObjectID, PackageID: obj.Object.PackageID, Path: obj.Object.Path, Size: obj.Object.Size, FileHash: obj.Object.FileHash, FaultTolerance: solu.disasterTolerance, Redundancy: solu.spaceCost, AvgAccessCost: 0, // TODO 计算平均访问代价,从日常访问数据中统计 BlockDistribution: blockDist, // TODO 不好计算传输量 } return []datamap.SysEventBody{transEvt, distEvt} } func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj clitypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[clitypes.UserSpaceID]bool) db.UpdatingObjectRedundancy { entry := db.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, FileHash: obj.Object.FileHash, Size: obj.Object.Size, Redundancy: obj.Object.Redundancy, } reconstrct := make(map[clitypes.UserSpaceID]*[]int) for i, f := range solu.rmBlocks { block := solu.blockList[i] if !f { entry.Blocks = append(entry.Blocks, clitypes.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: block.Index, UserSpaceID: block.UserSpaceID, FileHash: block.FileHash, Size: block.Size, }) // 如果这个块是影子块,那么就要从完整对象里重建这个块 if !block.HasEntity { re, ok := reconstrct[block.UserSpaceID] if !ok { re = &[]int{} reconstrct[block.UserSpaceID] = re } *re = append(*re, block.Index) } } } ecRed := obj.Object.Redundancy.(*clitypes.ECRedundancy) for id, idxs := range reconstrct { // 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本 ft := ioswitch2.NewFromTo() ft.ECParam = ecRed ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *ctx.allUserSpaces[id].UserSpace.MasterHub, *ctx.allUserSpaces[id].UserSpace, ioswitch2.RawStream())) for _, i := range *idxs { ft.AddTo(ioswitch2.NewToShardStore(*ctx.allUserSpaces[id].UserSpace.MasterHub, *ctx.allUserSpaces[id].UserSpace, ioswitch2.ECStream(i), fmt.Sprintf("%d.%d", obj.Object.ObjectID, i))) } err := parser.Parse(ft, planBld) if err != nil { // TODO 错误处理 continue } planningHubIDs[id] = true } return entry } func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj clitypes.ObjectDetail) []datamap.SysEventBody { var blockChgs []datamap.BlockChange reconstrct := make(map[clitypes.UserSpaceID]*[]int) for i, f := range solu.rmBlocks { block := solu.blockList[i] if !f { // 如果这个块是影子块,那么就要从完整对象里重建这个块 if !block.HasEntity { re, ok := reconstrct[block.UserSpaceID] if !ok { re = &[]int{} reconstrct[block.UserSpaceID] = re } *re = append(*re, block.Index) } } else { blockChgs = append(blockChgs, &datamap.BlockChangeDeleted{ Index: block.Index, UserSpaceID: block.UserSpaceID, }) } } // 由于每一个需要被重建的块都是从同中心的副本里构建出来的,所以对于每一个中心都要产生一个BlockChangeEnDecode for id, idxs := range reconstrct { var tarBlocks []datamap.Block for _, idx := range *idxs { tarBlocks = append(tarBlocks, datamap.Block{ BlockType: datamap.BlockTypeEC, Index: idx, UserSpaceID: id, }) } blockChgs = append(blockChgs, &datamap.BlockChangeEnDecode{ SourceBlocks: []datamap.Block{{ BlockType: datamap.BlockTypeRaw, Index: 0, UserSpaceID: id, // 影子块的原始对象就在同一个节点上 }}, TargetBlocks: tarBlocks, // 传输量为0 }) } transEvt := &datamap.BodyBlockTransfer{ ObjectID: obj.Object.ObjectID, PackageID: obj.Object.PackageID, BlockChanges: blockChgs, } var blockDist []datamap.BlockDistributionObjectInfo for i, f := range solu.rmBlocks { if !f { blockDist = append(blockDist, datamap.BlockDistributionObjectInfo{ BlockType: datamap.BlockTypeEC, Index: solu.blockList[i].Index, UserSpaceID: solu.blockList[i].UserSpaceID, }) } } distEvt := &datamap.BodyBlockDistribution{ ObjectID: obj.Object.ObjectID, PackageID: obj.Object.PackageID, Path: obj.Object.Path, Size: obj.Object.Size, FileHash: obj.Object.FileHash, FaultTolerance: solu.disasterTolerance, Redundancy: solu.spaceCost, AvgAccessCost: 0, // TODO 计算平均访问代价,从日常访问数据中统计 BlockDistribution: blockDist, // TODO 不好计算传输量 } return []datamap.SysEventBody{transEvt, distEvt} } func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[clitypes.UserSpaceID]bool, reen *distlock.Reentrant) (map[string]exec.VarValue, error) { reqBlder := reqbuilder.NewBuilder() for id, _ := range planningSpaceIDs { reqBlder.Shard().Buzy(id) } err := reen.Lock(reqBlder.Build()) if err != nil { return nil, fmt.Errorf("locking shard resources: %w", err) } wg := sync.WaitGroup{} // 执行IO计划 var ioSwRets map[string]exec.VarValue var ioSwErr error wg.Add(1) go func() { defer wg.Done() execCtx := exec.NewExecContext() exec.SetValueByType(execCtx, ctx.ticktock.stgPool) ret, err := planBld.Execute(execCtx).Wait(context.TODO()) if err != nil { ioSwErr = fmt.Errorf("executing io switch plan: %w", err) return } ioSwRets = ret }() wg.Wait() if ioSwErr != nil { return nil, ioSwErr } return ioSwRets, nil } func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj clitypes.ObjectDetail, ioRets map[string]exec.VarValue) { for i := range entry.Blocks { if entry.Blocks[i].FileHash != "" { continue } key := fmt.Sprintf("%d.%d", obj.Object.ObjectID, entry.Blocks[i].Index) // 不应该出现key不存在的情况 r := ioRets[key].(*ops2.ShardInfoValue) entry.Blocks[i].FileHash = r.Hash entry.Blocks[i].Size = r.Size } }