From e0ea7cc92f8e189b671dd2fac43751c0653c8535 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 26 Nov 2024 16:08:39 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- magefiles/common.go | 10 ++++- magefiles/targets/targets.go | 5 +++ pkgs/ioswitch/dag/node.go | 50 +++++++++++++++++++++ pkgs/ioswitch/exec/utils.go | 4 ++ pkgs/ioswitch/plan/generate.go | 12 +++++ sdks/storage/models.go | 80 +++++++++++++++++++++++++++++++++- utils/lo2/lo.go | 15 ++++++- utils/math2/math.go | 12 +++++ 8 files changed, 185 insertions(+), 3 deletions(-) diff --git a/magefiles/common.go b/magefiles/common.go index 577cf7c..9f207f7 100644 --- a/magefiles/common.go +++ b/magefiles/common.go @@ -15,6 +15,7 @@ var Global = struct { OS string Arch string BuildRoot string + Debug bool }{ Arch: "amd64", } @@ -50,7 +51,14 @@ func Build(args BuildArgs) error { binPath := filepath.Join(fullOutputDir, args.OutputName+goBuildArgs.OutputExt) fmt.Printf("building to %s\n", binPath) - goCmdArgs := []string{"build", "-o", binPath} + goCmdArgs := []string{"build"} + + if Global.Debug { + goCmdArgs = append(goCmdArgs, "-gcflags", "all=-N -l") + } + + goCmdArgs = append(goCmdArgs, "-o", binPath) + if args.EntryFile != "" { goCmdArgs = append(goCmdArgs, args.EntryFile) } diff --git a/magefiles/targets/targets.go b/magefiles/targets/targets.go index 09c8212..148cd3f 100644 --- a/magefiles/targets/targets.go +++ b/magefiles/targets/targets.go @@ -28,3 +28,8 @@ func ARM64() { func BuildRoot(dir string) { magefiles.Global.BuildRoot = dir } + +// [配置项]关闭编译优化，用于调试 +func Debug() { + magefiles.Global.Debug = true +} diff --git a/pkgs/ioswitch/dag/node.go b/pkgs/ioswitch/dag/node.go index 198cc6f..494fede 100644 --- a/pkgs/ioswitch/dag/node.go +++ b/pkgs/ioswitch/dag/node.go @@ -114,6 +114,31 @@ func (s *InputSlots) EnlargeOne() int { return s.Len() - 1 } +func (s *InputSlots) GetVarIDs() []exec.VarID { + var ids []exec.VarID + for _, v := range s.RawArray() { + if v == nil { + continue + } + ids = append(ids, v.VarID) + } + + return ids +} + +func (s *InputSlots) GetVarIDsRanged(start, end int) []exec.VarID { + var ids []exec.VarID + for i := start; i < end; i++ { + v := s.Get(i) + if v == nil { + continue + } + ids = append(ids, v.VarID) + } + + return ids +} + type OutputSlots struct { VarSlots } @@ -138,6 +163,31 @@ func (s *OutputSlots) SetupNew(my Node, v *Var) { } } +func (s *OutputSlots) GetVarIDs() []exec.VarID { + var ids []exec.VarID + for _, v := range s.RawArray() { + if v == nil { + continue + } + ids = append(ids, v.VarID) + } + + return ids +} + +func (s *OutputSlots) GetVarIDsRanged(start, end int) []exec.VarID { + var ids []exec.VarID + for i := start; i < end; i++ { + v := s.Get(i) + if v == nil { + continue + } + ids = append(ids, v.VarID) + } + + return ids +} + type Slot struct { Var *Var Index int diff --git a/pkgs/ioswitch/exec/utils.go b/pkgs/ioswitch/exec/utils.go index f876f2c..09c48d3 100644 --- a/pkgs/ioswitch/exec/utils.go +++ b/pkgs/ioswitch/exec/utils.go @@ -14,6 +14,10 @@ type Range struct { Length *int64 } +func NewRange(offset int64, length int64) Range { + return Range{Offset: offset, Length: &length} +} + func (r *Range) Extend(other Range) { newOffset := math2.Min(r.Offset, other.Offset) diff --git a/pkgs/ioswitch/plan/generate.go b/pkgs/ioswitch/plan/generate.go index 6d155d9..3b3147b 100644 --- a/pkgs/ioswitch/plan/generate.go +++ b/pkgs/ioswitch/plan/generate.go @@ -119,6 +119,9 @@ func buildPlan(graph *dag.Graph, blder *exec.PlanBuilder) error { graph.Walk(func(node dag.Node) bool { for i := 0; i < node.OutputStreams().Len(); i++ { out := node.OutputStreams().Get(i) + if out == nil { + continue + } if out.VarID > 0 { continue @@ -129,6 +132,9 @@ func buildPlan(graph *dag.Graph, blder *exec.PlanBuilder) error { for i := 0; i < node.InputStreams().Len(); i++ { in := node.InputStreams().Get(i) + if in == nil { + continue + } if in.VarID > 0 { continue @@ -139,6 +145,9 @@ func buildPlan(graph *dag.Graph, blder *exec.PlanBuilder) error { for i := 0; i < node.OutputValues().Len(); i++ { out := node.OutputValues().Get(i) + if out == nil { + continue + } if out.VarID > 0 { continue @@ -149,6 +158,9 @@ func buildPlan(graph *dag.Graph, blder *exec.PlanBuilder) error { for i := 0; i < node.InputValues().Len(); i++ { in := node.InputValues().Get(i) + if in == nil { + continue + } if in.VarID > 0 { continue diff --git a/sdks/storage/models.go b/sdks/storage/models.go index cfc28bd..2b48c9c 100644 --- a/sdks/storage/models.go +++ b/sdks/storage/models.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkgs/types" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -33,7 +34,6 @@ type FileHash string /// TODO 将分散在各处的公共结构体定义集中到这里来 type Redundancy interface { - driver.Valuer } var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Redundancy]( @@ -41,6 +41,7 @@ var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTyp (*RepRedundancy)(nil), (*ECRedundancy)(nil), (*LRCRedundancy)(nil), + (*SegmentRedundancy)(nil), )), "type") type NoneRedundancy struct { @@ -162,6 +163,83 @@ func (b *LRCRedundancy) GetGroupElements(grp int) []int { return idxes } +type SegmentRedundancy struct { + serder.Metadata `union:"segment"` + Type string `json:"type"` + Segments []int64 `json:"segments"` // 每一段的大小 +} + +func NewSegmentRedundancy(totalSize int64, segmentCount int) *SegmentRedundancy { + var segs []int64 + segLen := int64(0) + // 计算每一段的大小。大小不一定都相同，但总和应该等于总大小。 + for i := 0; i < segmentCount; i++ { + curLen := totalSize*int64(i+1)/int64(segmentCount) - segLen + segs = append(segs, curLen) + segLen += curLen + } + + return &SegmentRedundancy{ + Type: "segment", + Segments: segs, + } +} + +func (r *SegmentRedundancy) SegmentCount() int { + return len(r.Segments) +} + +func (r *SegmentRedundancy) CalcSegmentStart(index int) int64 { + return lo.Sum(r.Segments[:index]) +} + +// 计算指定位置取整到最近的段的起始位置。 +func (r *SegmentRedundancy) FloorSegmentPosition(pos int64) int64 { + fpos := int64(0) + for _, segLen := range r.Segments { + segEnd := fpos + segLen + if pos < segEnd { + break + } + fpos += segLen + } + + return fpos +} + +// 计算指定范围内的段索引范围，参数和返回值所代表的范围都是左闭右开的。 +// 如果end == -1，则代表计算从start到最后一个字节的范围。 +func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxStart int, segIdxEnd int) { + segIdxStart = len(b.Segments) + segIdxEnd = len(b.Segments) + + // 找到第一个包含start的段索引 + segStart := int64(0) + for i, segLen := range b.Segments { + segEnd := segStart + segLen + if start < segEnd { + segIdxStart = i + break + } + segStart += segLen + } + + if end != nil { + // 找到第一个包含end的段索引 + segStart = int64(0) + for i, segLen := range b.Segments { + segEnd := segStart + segLen + if *end <= segEnd { + segIdxEnd = i + 1 + break + } + segStart += segLen + } + } + + return +} + const ( PackageStateNormal = "Normal" PackageStateDeleted = "Deleted" diff --git a/utils/lo2/lo.go b/utils/lo2/lo.go index 056f7e7..930f6d3 100644 --- a/utils/lo2/lo.go +++ b/utils/lo2/lo.go @@ -1,6 +1,9 @@ package lo2 -import "github.com/samber/lo" +import ( + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/utils/math2" +) func Remove[T comparable](arr []T, item T) []T { index := lo.IndexOf(arr, item) @@ -25,6 +28,16 @@ func RemoveAt[T any](arr []T, index int) []T { return append(arr[:index], arr[index+1:]...) } +func RemoveRange[T any](arr []T, start int, length int) []T { + if start >= len(arr) { + return arr + } + + length = math2.Min(len(arr), start+length) - start + copy(arr[start:], arr[start+length:]) + return arr[:len(arr)-length] +} + func RemoveAllDefault[T comparable](arr []T) []T { var def T return lo.Filter(arr, func(i T, idx int) bool { diff --git a/utils/math2/math.go b/utils/math2/math.go index 6677d2b..ee65faa 100644 --- a/utils/math2/math.go +++ b/utils/math2/math.go @@ -33,3 +33,15 @@ func CeilDiv[T constraints.Integer](v T, div T) T { func FloorDiv[T constraints.Integer](v T, div T) T { return v / div } + +func Clamp[T constraints.Integer](v, min, max T) T { + if v < min { + return min + } + + if v > max { + return max + } + + return v +}