| @@ -1,6 +1,5 @@ | |||
| package cmdline | |||
| /* | |||
| import ( | |||
| "context" | |||
| "fmt" | |||
| @@ -13,54 +12,31 @@ import ( | |||
| stgglb "gitlink.org.cn/cloudream/storage/common/globals" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc" | |||
| lrcparser "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc/parser" | |||
| coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" | |||
| ) | |||
| func init() { | |||
| rootCmd.AddCommand(&cobra.Command{ | |||
| Use: "test", | |||
| Short: "test", | |||
| // Args: cobra.ExactArgs(1), | |||
| Use: "test2", | |||
| Short: "test2", | |||
| Run: func(cmd *cobra.Command, args []string) { | |||
| // cmdCtx := GetCmdCtx(cmd) | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.AddFrom(ioswitch2.NewFromNode("Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", &nodes.Nodes[0], -1)) | |||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], -1, "asd")) | |||
| le := int64(3) | |||
| toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 5, Length: &le}) | |||
| ft.AddTo(toExec) | |||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 0, "0")) | |||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 1, "1")) | |||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 2, "2")) | |||
| // ft.AddFrom(ioswitch2.NewFromNode("QmS2s8GRYHEurXL7V1zUtKvf2H1BGcQc5NN1T1hiSnWvbd", &nodes.Nodes[0], 1)) | |||
| // ft.AddFrom(ioswitch2.NewFromNode("QmUgUEUMzdnjPNx6xu9PDGXpSyXTk8wzPWvyYZ9zasE1WW", &nodes.Nodes[1], 2)) | |||
| // le := int64(5) | |||
| // toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 3, Length: &le}) | |||
| // toExec, hd := plans.NewToExecutorWithRange(1, plans.Range{Offset: 0, Length: nil}) | |||
| // toExec2, hd2 := plans.NewToExecutorWithRange(2, plans.Range{Offset: 0, Length: nil}) | |||
| // ft.AddTo(toExec) | |||
| // ft.AddTo(toExec2) | |||
| // fromExec, hd := plans.NewFromExecutor(-1) | |||
| // ft.AddFrom(fromExec) | |||
| // ft.AddTo(plans.NewToNode(nodes.Nodes[1], -1, "asd")) | |||
| parser := parser.NewParser(cdssdk.DefaultECRedundancy) | |||
| ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3) | |||
| ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.RawStream())) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0), "0")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1), "1")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2), "2")) | |||
| plans := exec.NewPlanBuilder() | |||
| err = parser.Parse(ft, plans) | |||
| @@ -68,7 +44,9 @@ func init() { | |||
| panic(err) | |||
| } | |||
| exec := plans.Execute() | |||
| fmt.Printf("plans: %v\n", plans) | |||
| exec := plans.Execute(exec.NewExecContext()) | |||
| fut := future.NewSetVoid() | |||
| go func() { | |||
| @@ -77,176 +55,130 @@ func init() { | |||
| panic(err) | |||
| } | |||
| fmt.Printf("mp: %+v\n", mp) | |||
| fmt.Printf("0: %v, 1: %v, 2: %v\n", mp["0"], mp["1"], mp["2"]) | |||
| fut.SetVoid() | |||
| }() | |||
| go func() { | |||
| // exec.BeginWrite(io.NopCloser(bytes.NewBuffer([]byte("hello world"))), hd) | |||
| // if err != nil { | |||
| // panic(err) | |||
| // } | |||
| str, err := exec.BeginRead(hd) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| defer str.Close() | |||
| data, err := io.ReadAll(str) | |||
| if err != nil && err != io.EOF { | |||
| panic(err) | |||
| } | |||
| fmt.Printf("data: %v(%v)\n", string(data), len(data)) | |||
| }() | |||
| fut.Wait(context.TODO()) | |||
| }, | |||
| }) | |||
| // rootCmd.AddCommand(&cobra.Command{ | |||
| // Use: "test", | |||
| // Short: "test", | |||
| // // Args: cobra.ExactArgs(1), | |||
| // Run: func(cmd *cobra.Command, args []string) { | |||
| // cmdCtx := GetCmdCtx(cmd) | |||
| // file, _ := cmdCtx.Cmdline.Svc.ObjectSvc().Download(1, downloader.DownloadReqeust{ | |||
| // ObjectID: 27379, | |||
| // Length: -1, | |||
| // }) | |||
| // data, _ := io.ReadAll(file.File) | |||
| // fmt.Printf("data: %v(%v)\n", string(data), len(data)) | |||
| // }, | |||
| // }) | |||
| rootCmd.AddCommand(&cobra.Command{ | |||
| Use: "test3", | |||
| Short: "test3", | |||
| // Args: cobra.ExactArgs(1), | |||
| Use: "test", | |||
| Short: "test", | |||
| Run: func(cmd *cobra.Command, args []string) { | |||
| // cmdCtx := GetCmdCtx(cmd) | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| red := cdssdk.DefaultLRCRedundancy | |||
| var toes []ioswitchlrc.To | |||
| for i := 0; i < red.N; i++ { | |||
| toes = append(toes, ioswitchlrc.NewToNode(nodes.Nodes[i%2], i, fmt.Sprintf("%d", i))) | |||
| } | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3) | |||
| ft.ECParam = &cdssdk.DefaultECRedundancy | |||
| ft.AddFrom(ioswitch2.NewFromShardstore("22CC59CE3297F78F2D20DC1E33181B77F21E6782097C94E1664F99F129834069", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0))) | |||
| ft.AddFrom(ioswitch2.NewFromShardstore("5EAC20EB3EBC7B5FA176C5BD1C01041FB2A6D14C35D6A232CA83D7F1E4B01ADE", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1))) | |||
| ft.AddFrom(ioswitch2.NewFromShardstore("A9BC1802F37100C80C72A1D6E8F53C0E0B73F85F99153D8C78FB01CEC9D8D903", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2))) | |||
| toDrv, drvStr := ioswitch2.NewToDriverWithRange(ioswitch2.RawStream(), exec.NewRange(500, 500)) | |||
| ft.AddTo(toDrv) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2")) | |||
| plans := exec.NewPlanBuilder() | |||
| err = lrcparser.Encode(ioswitchlrc.NewFromNode("QmNspjDLxQbAsuh37jRXKvLWHE2f7JpqY4HEJ8x7Jgbzqa", &nodes.Nodes[0], -1), toes, plans) | |||
| err = parser.Parse(ft, plans) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||
| } | |||
| fmt.Printf("plans: %v\n", plans) | |||
| fmt.Printf("ioRet: %v\n", ioRet) | |||
| }, | |||
| }) | |||
| exec := plans.Execute(exec.NewExecContext()) | |||
| rootCmd.AddCommand(&cobra.Command{ | |||
| Use: "test4", | |||
| Short: "test4", | |||
| // Args: cobra.ExactArgs(1), | |||
| Run: func(cmd *cobra.Command, args []string) { | |||
| // cmdCtx := GetCmdCtx(cmd) | |||
| fut := future.NewSetVoid() | |||
| go func() { | |||
| mp, err := exec.Wait(context.Background()) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| for k, v := range mp { | |||
| fmt.Printf("%s: %v\n", k, v) | |||
| } | |||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| fut.SetVoid() | |||
| }() | |||
| // red := cdssdk.DefaultLRCRedundancy | |||
| go func() { | |||
| str, err := exec.BeginRead(drvStr) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err = lrcparser.ReconstructGroup([]ioswitchlrc.From{ | |||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 0), | |||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 1), | |||
| }, []ioswitchlrc.To{ | |||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"), | |||
| }, plans) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| data, err := io.ReadAll(str) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||
| } | |||
| fmt.Printf("read(%v): %s\n", len(data), string(data)) | |||
| }() | |||
| fmt.Printf("ioRet: %v\n", ioRet) | |||
| fut.Wait(context.TODO()) | |||
| }, | |||
| }) | |||
| rootCmd.AddCommand(&cobra.Command{ | |||
| Use: "test3", | |||
| Short: "test3", | |||
| // Args: cobra.ExactArgs(1), | |||
| Run: func(cmd *cobra.Command, args []string) { | |||
| // cmdCtx := GetCmdCtx(cmd) | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| // red := cdssdk.DefaultLRCRedundancy | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.ECParam = &cdssdk.DefaultECRedundancy | |||
| ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.RawStream())) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1")) | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2")) | |||
| plans := exec.NewPlanBuilder() | |||
| le := int64(1293) | |||
| err = lrcparser.ReconstructAny([]ioswitchlrc.From{ | |||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[0], 0), | |||
| ioswitchlrc.NewFromNode("QmQBKncEDqxw3BrGr3th3gS3jUC2fizGz1w29ZxxrrKfNv", &nodes.Nodes[0], 2), | |||
| }, []ioswitchlrc.To{ | |||
| ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], -1, "-1", exec.Range{0, &le}), | |||
| ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], 0, "0", exec.Range{10, &le}), | |||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 1, "1"), | |||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 2, "2"), | |||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"), | |||
| }, plans) | |||
| err = parser.Parse(ft, plans) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||
| if err != nil { | |||
| panic(err) | |||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||
| } | |||
| fmt.Printf("plans: %v\n", plans) | |||
| exec := plans.Execute(exec.NewExecContext()) | |||
| fmt.Printf("ioRet: %v\n", ioRet) | |||
| fut := future.NewSetVoid() | |||
| go func() { | |||
| mp, err := exec.Wait(context.Background()) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| for k, v := range mp { | |||
| fmt.Printf("%s: %v\n", k, v) | |||
| } | |||
| fut.SetVoid() | |||
| }() | |||
| fut.Wait(context.TODO()) | |||
| }, | |||
| }) | |||
| } | |||
| */ | |||
| @@ -400,7 +400,7 @@ func (iter *DownloadObjectIterator) downloadFromStorage(stg *stgmod.StorageDetai | |||
| strHandle = handle | |||
| plans := exec.NewPlanBuilder() | |||
| if err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy); err != nil { | |||
| if err := parser.Parse(ft, plans); err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -199,6 +199,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||
| } | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.ECParam = s.red | |||
| for _, b := range s.blocks { | |||
| stg := b.Storage | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(b.Block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(b.Block.Index))) | |||
| @@ -210,7 +211,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||
| ft.AddTo(toExec) | |||
| plans := exec.NewPlanBuilder() | |||
| err := parser.Parse(ft, plans, *s.red) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return 0, err | |||
| } | |||
| @@ -6,7 +6,7 @@ import ( | |||
| ) | |||
| type From interface { | |||
| GetStreamType() StreamType | |||
| GetStreamIndex() StreamIndex | |||
| } | |||
| type To interface { | |||
| @@ -14,60 +14,64 @@ type To interface { | |||
| // 如果DataIndex == -1,则表示在整个文件的范围。 | |||
| // 如果DataIndex >= 0,则表示在文件的某个分片的范围。 | |||
| GetRange() exec.Range | |||
| GetStreamType() StreamType | |||
| GetStreamIndex() StreamIndex | |||
| } | |||
| const ( | |||
| // 未处理的完整文件流 | |||
| StreamTypeRaw = iota | |||
| StreamIndexRaw = iota | |||
| // EC编码的某一块的流 | |||
| StreamTypeEC | |||
| StreamIndexEC | |||
| // 分段编码的某一段的流 | |||
| StreamTypeSegment | |||
| StreamIndexSegment | |||
| ) | |||
| type StreamType struct { | |||
| type StreamIndex struct { | |||
| Type int | |||
| Index int | |||
| } | |||
| func RawStream() StreamType { | |||
| return StreamType{ | |||
| Type: StreamTypeRaw, | |||
| func RawStream() StreamIndex { | |||
| return StreamIndex{ | |||
| Type: StreamIndexRaw, | |||
| } | |||
| } | |||
| func ECSrteam(index int) StreamType { | |||
| return StreamType{ | |||
| Type: StreamTypeEC, | |||
| func ECSrteam(index int) StreamIndex { | |||
| return StreamIndex{ | |||
| Type: StreamIndexEC, | |||
| Index: index, | |||
| } | |||
| } | |||
| func SegmentStream(index int) StreamType { | |||
| return StreamType{ | |||
| Type: StreamTypeSegment, | |||
| func SegmentStream(index int) StreamIndex { | |||
| return StreamIndex{ | |||
| Type: StreamIndexSegment, | |||
| Index: index, | |||
| } | |||
| } | |||
| func (s StreamType) IsRaw() bool { | |||
| return s.Type == StreamTypeRaw | |||
| func (s StreamIndex) IsRaw() bool { | |||
| return s.Type == StreamIndexRaw | |||
| } | |||
| func (s StreamType) IsEC() bool { | |||
| return s.Type == StreamTypeEC | |||
| func (s StreamIndex) IsEC() bool { | |||
| return s.Type == StreamIndexEC | |||
| } | |||
| func (s StreamType) IsSegment() bool { | |||
| return s.Type == StreamTypeSegment | |||
| func (s StreamIndex) IsSegment() bool { | |||
| return s.Type == StreamIndexSegment | |||
| } | |||
| type FromTos []FromTo | |||
| type FromTo struct { | |||
| Froms []From | |||
| Toes []To | |||
| // 如果输入或者输出用到了EC编码的流,则需要提供EC参数。 | |||
| ECParam *cdssdk.ECRedundancy | |||
| // 同上 | |||
| SegmentParam *cdssdk.SegmentRedundancy | |||
| Froms []From | |||
| Toes []To | |||
| } | |||
| func NewFromTo() FromTo { | |||
| @@ -85,69 +89,69 @@ func (ft *FromTo) AddTo(to To) *FromTo { | |||
| } | |||
| type FromDriver struct { | |||
| Handle *exec.DriverWriteStream | |||
| StreamType StreamType | |||
| Handle *exec.DriverWriteStream | |||
| StreamIndex StreamIndex | |||
| } | |||
| func NewFromDriver(strType StreamType) (*FromDriver, *exec.DriverWriteStream) { | |||
| func NewFromDriver(strIdx StreamIndex) (*FromDriver, *exec.DriverWriteStream) { | |||
| handle := &exec.DriverWriteStream{ | |||
| RangeHint: &exec.Range{}, | |||
| } | |||
| return &FromDriver{ | |||
| Handle: handle, | |||
| StreamType: strType, | |||
| Handle: handle, | |||
| StreamIndex: strIdx, | |||
| }, handle | |||
| } | |||
| func (f *FromDriver) GetStreamType() StreamType { | |||
| return f.StreamType | |||
| func (f *FromDriver) GetStreamIndex() StreamIndex { | |||
| return f.StreamIndex | |||
| } | |||
| type FromShardstore struct { | |||
| FileHash cdssdk.FileHash | |||
| Hub cdssdk.Hub | |||
| Storage cdssdk.Storage | |||
| StreamType StreamType | |||
| FileHash cdssdk.FileHash | |||
| Hub cdssdk.Hub | |||
| Storage cdssdk.Storage | |||
| StreamIndex StreamIndex | |||
| } | |||
| func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strType StreamType) *FromShardstore { | |||
| func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strIdx StreamIndex) *FromShardstore { | |||
| return &FromShardstore{ | |||
| FileHash: fileHash, | |||
| Hub: hub, | |||
| Storage: storage, | |||
| StreamType: strType, | |||
| FileHash: fileHash, | |||
| Hub: hub, | |||
| Storage: storage, | |||
| StreamIndex: strIdx, | |||
| } | |||
| } | |||
| func (f *FromShardstore) GetStreamType() StreamType { | |||
| return f.StreamType | |||
| func (f *FromShardstore) GetStreamIndex() StreamIndex { | |||
| return f.StreamIndex | |||
| } | |||
| type ToDriver struct { | |||
| Handle *exec.DriverReadStream | |||
| StreamType StreamType | |||
| Range exec.Range | |||
| Handle *exec.DriverReadStream | |||
| StreamIndex StreamIndex | |||
| Range exec.Range | |||
| } | |||
| func NewToDriver(strType StreamType) (*ToDriver, *exec.DriverReadStream) { | |||
| func NewToDriver(strIdx StreamIndex) (*ToDriver, *exec.DriverReadStream) { | |||
| str := exec.DriverReadStream{} | |||
| return &ToDriver{ | |||
| Handle: &str, | |||
| StreamType: strType, | |||
| Handle: &str, | |||
| StreamIndex: strIdx, | |||
| }, &str | |||
| } | |||
| func NewToDriverWithRange(strType StreamType, rng exec.Range) (*ToDriver, *exec.DriverReadStream) { | |||
| func NewToDriverWithRange(strIdx StreamIndex, rng exec.Range) (*ToDriver, *exec.DriverReadStream) { | |||
| str := exec.DriverReadStream{} | |||
| return &ToDriver{ | |||
| Handle: &str, | |||
| StreamType: strType, | |||
| Range: rng, | |||
| Handle: &str, | |||
| StreamIndex: strIdx, | |||
| Range: rng, | |||
| }, &str | |||
| } | |||
| func (t *ToDriver) GetStreamType() StreamType { | |||
| return t.StreamType | |||
| func (t *ToDriver) GetStreamIndex() StreamIndex { | |||
| return t.StreamIndex | |||
| } | |||
| func (t *ToDriver) GetRange() exec.Range { | |||
| @@ -157,32 +161,32 @@ func (t *ToDriver) GetRange() exec.Range { | |||
| type ToShardStore struct { | |||
| Hub cdssdk.Hub | |||
| Storage cdssdk.Storage | |||
| StreamType StreamType | |||
| StreamIndex StreamIndex | |||
| Range exec.Range | |||
| FileHashStoreKey string | |||
| } | |||
| func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strType StreamType, fileHashStoreKey string) *ToShardStore { | |||
| func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strIdx StreamIndex, fileHashStoreKey string) *ToShardStore { | |||
| return &ToShardStore{ | |||
| Hub: hub, | |||
| Storage: stg, | |||
| StreamType: strType, | |||
| StreamIndex: strIdx, | |||
| FileHashStoreKey: fileHashStoreKey, | |||
| } | |||
| } | |||
| func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamType StreamType, fileHashStoreKey string, rng exec.Range) *ToShardStore { | |||
| func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamIndex StreamIndex, fileHashStoreKey string, rng exec.Range) *ToShardStore { | |||
| return &ToShardStore{ | |||
| Hub: hub, | |||
| Storage: stg, | |||
| StreamType: streamType, | |||
| StreamIndex: streamIndex, | |||
| FileHashStoreKey: fileHashStoreKey, | |||
| Range: rng, | |||
| } | |||
| } | |||
| func (t *ToShardStore) GetStreamType() StreamType { | |||
| return t.StreamType | |||
| func (t *ToShardStore) GetStreamIndex() StreamIndex { | |||
| return t.StreamIndex | |||
| } | |||
| func (t *ToShardStore) GetRange() exec.Range { | |||
| @@ -207,9 +211,9 @@ func NewLoadToShared(hub cdssdk.Hub, storage cdssdk.Storage, userID cdssdk.UserI | |||
| } | |||
| } | |||
| func (t *LoadToShared) GetStreamType() StreamType { | |||
| return StreamType{ | |||
| Type: StreamTypeRaw, | |||
| func (t *LoadToShared) GetStreamIndex() StreamIndex { | |||
| return StreamIndex{ | |||
| Type: StreamIndexRaw, | |||
| } | |||
| } | |||
| @@ -0,0 +1,81 @@ | |||
| package ops2 | |||
| import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||
| ) | |||
| // TODO 想办法直接使用ops里的Node,而不是重新实现一遍 | |||
| type FromDriverNode struct { | |||
| dag.NodeBase | |||
| From ioswitch2.From | |||
| Handle *exec.DriverWriteStream | |||
| } | |||
| func (b *GraphNodeBuilder) NewFromDriver(fr ioswitch2.From, handle *exec.DriverWriteStream) *FromDriverNode { | |||
| node := &FromDriverNode{ | |||
| From: fr, | |||
| Handle: handle, | |||
| } | |||
| b.AddNode(node) | |||
| node.OutputStreams().SetupNew(node, b.NewVar()) | |||
| return node | |||
| } | |||
| func (f *FromDriverNode) GetFrom() ioswitch2.From { | |||
| return f.From | |||
| } | |||
| func (t *FromDriverNode) Output() dag.Slot { | |||
| return dag.Slot{ | |||
| Var: t.OutputStreams().Get(0), | |||
| Index: 0, | |||
| } | |||
| } | |||
| func (t *FromDriverNode) GenerateOp() (exec.Op, error) { | |||
| t.Handle.ID = t.OutputStreams().Get(0).VarID | |||
| return nil, nil | |||
| } | |||
| type ToDriverNode struct { | |||
| dag.NodeBase | |||
| To ioswitch2.To | |||
| Handle *exec.DriverReadStream | |||
| Range exec.Range | |||
| } | |||
| func (b *GraphNodeBuilder) NewToDriver(to ioswitch2.To, handle *exec.DriverReadStream) *ToDriverNode { | |||
| node := &ToDriverNode{ | |||
| To: to, | |||
| Handle: handle, | |||
| } | |||
| b.AddNode(node) | |||
| return node | |||
| } | |||
| func (t *ToDriverNode) GetTo() ioswitch2.To { | |||
| return t.To | |||
| } | |||
| func (t *ToDriverNode) SetInput(v *dag.Var) { | |||
| t.InputStreams().EnsureSize(1) | |||
| v.StreamTo(t, 0) | |||
| } | |||
| func (t *ToDriverNode) Input() dag.Slot { | |||
| return dag.Slot{ | |||
| Var: t.InputStreams().Get(0), | |||
| Index: 0, | |||
| } | |||
| } | |||
| func (t *ToDriverNode) GenerateOp() (exec.Op, error) { | |||
| t.Handle.ID = t.InputStreams().Get(0).VarID | |||
| return nil, nil | |||
| } | |||
| @@ -3,6 +3,7 @@ package ops2 | |||
| import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||
| ) | |||
| type GraphNodeBuilder struct { | |||
| @@ -15,11 +16,13 @@ func NewGraphNodeBuilder() *GraphNodeBuilder { | |||
| type FromNode interface { | |||
| dag.Node | |||
| GetFrom() ioswitch2.From | |||
| Output() dag.Slot | |||
| } | |||
| type ToNode interface { | |||
| dag.Node | |||
| GetTo() ioswitch2.To | |||
| Input() dag.Slot | |||
| SetInput(input *dag.Var) | |||
| } | |||
| @@ -72,7 +72,11 @@ func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||
| } | |||
| func (o *Range) String() string { | |||
| return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input, o.Output) | |||
| len := "" | |||
| if o.Length != nil { | |||
| len = fmt.Sprintf("%v", *o.Length) | |||
| } | |||
| return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, len, o.Input, o.Output) | |||
| } | |||
| type RangeNode struct { | |||
| @@ -1 +1,215 @@ | |||
| package ops2 | |||
| import ( | |||
| "context" | |||
| "fmt" | |||
| "io" | |||
| "gitlink.org.cn/cloudream/common/pkgs/future" | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/utils" | |||
| "gitlink.org.cn/cloudream/common/utils/io2" | |||
| ) | |||
| func init() { | |||
| exec.UseOp[*SegmentSplit]() | |||
| exec.UseOp[*SegmentJoin]() | |||
| } | |||
| type SegmentSplit struct { | |||
| Input exec.VarID | |||
| Segments []int64 | |||
| Outputs []exec.VarID | |||
| } | |||
| func (o *SegmentSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||
| input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| defer input.Stream.Close() | |||
| for i, outID := range o.Outputs { | |||
| fut := future.NewSetVoid() | |||
| segStr := io.LimitReader(input.Stream, o.Segments[i]) | |||
| segStr2 := io2.DelegateReadCloser(segStr, func() error { | |||
| fut.SetError(context.Canceled) | |||
| return nil | |||
| }) | |||
| segStr2 = io2.AfterEOF(segStr2, func(str io.ReadCloser, err error) { | |||
| fut.SetVoid() | |||
| }) | |||
| e.PutVar(outID, &exec.StreamValue{Stream: segStr2}) | |||
| err = fut.Wait(ctx.Context) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| func (o *SegmentSplit) String() string { | |||
| return fmt.Sprintf("SegmentSplit(%v, %v) -> %v", o.Input, o.Segments, o.Outputs) | |||
| } | |||
| type SegmentJoin struct { | |||
| Inputs []exec.VarID | |||
| Output exec.VarID | |||
| // 这些字段只在执行时使用 | |||
| ctx *exec.ExecContext | |||
| e *exec.Executor | |||
| nextStreamIdx int | |||
| nextStream io.ReadCloser | |||
| fut *future.SetVoidFuture | |||
| } | |||
| func (o *SegmentJoin) Read(buf []byte) (int, error) { | |||
| for { | |||
| if o.nextStream == nil { | |||
| if o.nextStreamIdx >= len(o.Inputs) { | |||
| o.fut.SetVoid() | |||
| return 0, io.EOF | |||
| } | |||
| input, err := exec.BindVar[*exec.StreamValue](o.e, o.ctx.Context, o.Inputs[o.nextStreamIdx]) | |||
| if err != nil { | |||
| return 0, err | |||
| } | |||
| o.nextStream = input.Stream | |||
| o.nextStreamIdx++ | |||
| } | |||
| n, err := o.nextStream.Read(buf) | |||
| if err == io.EOF { | |||
| o.nextStream.Close() | |||
| o.nextStream = nil | |||
| continue | |||
| } | |||
| return n, err | |||
| } | |||
| } | |||
| func (o *SegmentJoin) Close() error { | |||
| if o.nextStream != nil { | |||
| o.nextStream.Close() | |||
| o.nextStream = nil | |||
| o.fut.SetVoid() | |||
| } | |||
| return nil | |||
| } | |||
| func (o *SegmentJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||
| o.ctx = ctx | |||
| o.e = e | |||
| o.nextStreamIdx = 0 | |||
| o.nextStream = nil | |||
| o.fut = future.NewSetVoid() | |||
| e.PutVar(o.Output, &exec.StreamValue{Stream: o}) | |||
| return o.fut.Wait(ctx.Context) | |||
| } | |||
| func (o *SegmentJoin) String() string { | |||
| return fmt.Sprintf("SegmentJoin %v -> %v", utils.FormatVarIDs(o.Inputs), o.Output) | |||
| } | |||
| type SegmentSplitNode struct { | |||
| dag.NodeBase | |||
| segments []int64 | |||
| } | |||
| func (b *GraphNodeBuilder) NewSegmentSplit(segments []int64) *SegmentSplitNode { | |||
| node := &SegmentSplitNode{ | |||
| segments: segments, | |||
| } | |||
| b.AddNode(node) | |||
| node.OutputStreams().Resize(len(segments)) | |||
| return node | |||
| } | |||
| func (n *SegmentSplitNode) SetInput(input *dag.Var) { | |||
| n.InputStreams().EnsureSize(1) | |||
| input.StreamTo(n, 0) | |||
| } | |||
| func (n *SegmentSplitNode) Segment(index int) *dag.Var { | |||
| // 必须连续消耗流 | |||
| for i := 0; i <= index; i++ { | |||
| if n.OutputStreams().Get(i) == nil { | |||
| n.OutputStreams().Setup(n, n.Graph().NewVar(), i) | |||
| } | |||
| } | |||
| return n.OutputStreams().Get(index) | |||
| } | |||
| func (t *SegmentSplitNode) GenerateOp() (exec.Op, error) { | |||
| lastUsedSeg := 0 | |||
| for i := t.OutputStreams().Len() - 1; i >= 0; i-- { | |||
| if t.OutputStreams().Get(i) != nil { | |||
| lastUsedSeg = i | |||
| break | |||
| } | |||
| } | |||
| return &SegmentSplit{ | |||
| Input: t.InputStreams().Get(0).VarID, | |||
| Segments: t.segments[:lastUsedSeg+1], | |||
| Outputs: t.OutputStreams().GetVarIDs(), | |||
| }, nil | |||
| } | |||
| type SegmentJoinNode struct { | |||
| dag.NodeBase | |||
| UsedStart int | |||
| UsedCount int | |||
| } | |||
| func (b *GraphNodeBuilder) NewSegmentJoin(segmentSizes []int64) *SegmentJoinNode { | |||
| node := &SegmentJoinNode{} | |||
| b.AddNode(node) | |||
| node.InputStreams().Resize(len(segmentSizes)) | |||
| node.OutputStreams().SetupNew(node, b.NewVar()) | |||
| return node | |||
| } | |||
| func (n *SegmentJoinNode) SetInput(index int, input *dag.Var) { | |||
| input.StreamTo(n, index) | |||
| } | |||
| // 记录本计划中实际要使用的分段的范围,范围外的分段流都会取消输入 | |||
| func (n *SegmentJoinNode) MarkUsed(start, cnt int) { | |||
| n.UsedStart = start | |||
| n.UsedCount = cnt | |||
| for i := 0; i < start; i++ { | |||
| str := n.InputStreams().Get(i) | |||
| if str != nil { | |||
| str.StreamNotTo(n, i) | |||
| } | |||
| } | |||
| for i := start + cnt; i < n.InputStreams().Len(); i++ { | |||
| str := n.InputStreams().Get(i) | |||
| if str != nil { | |||
| str.StreamNotTo(n, i) | |||
| } | |||
| } | |||
| } | |||
| func (n *SegmentJoinNode) Joined() *dag.Var { | |||
| return n.OutputStreams().Get(0) | |||
| } | |||
| func (t *SegmentJoinNode) GenerateOp() (exec.Op, error) { | |||
| return &SegmentJoin{ | |||
| Inputs: t.InputStreams().GetVarIDsRanged(t.UsedStart, t.UsedStart+t.UsedCount), | |||
| Output: t.OutputStreams().Get(0).VarID, | |||
| }, nil | |||
| } | |||
| @@ -10,6 +10,7 @@ import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | |||
| "gitlink.org.cn/cloudream/common/utils/io2" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | |||
| ) | |||
| @@ -115,12 +116,14 @@ func (o *ShardWrite) String() string { | |||
| type ShardReadNode struct { | |||
| dag.NodeBase | |||
| From ioswitch2.From | |||
| StorageID cdssdk.StorageID | |||
| Open types.OpenOption | |||
| } | |||
| func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode { | |||
| func (b *GraphNodeBuilder) NewShardRead(fr ioswitch2.From, stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode { | |||
| node := &ShardReadNode{ | |||
| From: fr, | |||
| StorageID: stgID, | |||
| Open: open, | |||
| } | |||
| @@ -129,6 +132,10 @@ func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenO | |||
| return node | |||
| } | |||
| func (t *ShardReadNode) GetFrom() ioswitch2.From { | |||
| return t.From | |||
| } | |||
| func (t *ShardReadNode) Output() dag.Slot { | |||
| return dag.Slot{ | |||
| Var: t.OutputStreams().Get(0), | |||
| @@ -150,12 +157,14 @@ func (t *ShardReadNode) GenerateOp() (exec.Op, error) { | |||
| type ShardWriteNode struct { | |||
| dag.NodeBase | |||
| To ioswitch2.To | |||
| StorageID cdssdk.StorageID | |||
| FileHashStoreKey string | |||
| } | |||
| func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode { | |||
| func (b *GraphNodeBuilder) NewShardWrite(to ioswitch2.To, stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode { | |||
| node := &ShardWriteNode{ | |||
| To: to, | |||
| StorageID: stgID, | |||
| FileHashStoreKey: fileHashStoreKey, | |||
| } | |||
| @@ -163,6 +172,10 @@ func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKe | |||
| return node | |||
| } | |||
| func (t *ShardWriteNode) GetTo() ioswitch2.To { | |||
| return t.To | |||
| } | |||
| func (t *ShardWriteNode) SetInput(input *dag.Var) { | |||
| t.InputStreams().EnsureSize(1) | |||
| input.StreamTo(t, 0) | |||
| @@ -7,6 +7,7 @@ import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | |||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | |||
| ) | |||
| @@ -64,14 +65,16 @@ func (o *SharedLoad) String() string { | |||
| type SharedLoadNode struct { | |||
| dag.NodeBase | |||
| To ioswitch2.To | |||
| StorageID cdssdk.StorageID | |||
| UserID cdssdk.UserID | |||
| PackageID cdssdk.PackageID | |||
| Path string | |||
| } | |||
| func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode { | |||
| func (b *GraphNodeBuilder) NewSharedLoad(to ioswitch2.To, stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode { | |||
| node := &SharedLoadNode{ | |||
| To: to, | |||
| StorageID: stgID, | |||
| UserID: userID, | |||
| PackageID: packageID, | |||
| @@ -81,6 +84,10 @@ func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.U | |||
| return node | |||
| } | |||
| func (t *SharedLoadNode) GetTo() ioswitch2.To { | |||
| return t.To | |||
| } | |||
| func (t *SharedLoadNode) SetInput(input *dag.Var) { | |||
| t.InputStreams().EnsureSize(1) | |||
| input.StreamTo(t, 0) | |||
| @@ -15,9 +15,9 @@ import ( | |||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | |||
| ) | |||
| type TypedStream struct { | |||
| Stream *dag.Var | |||
| StreamType ioswitch2.StreamType | |||
| type IndexedStream struct { | |||
| Stream *dag.Var | |||
| StreamIndex ioswitch2.StreamIndex | |||
| } | |||
| type ParseContext struct { | |||
| @@ -25,33 +25,43 @@ type ParseContext struct { | |||
| DAG *ops2.GraphNodeBuilder | |||
| // 为了产生所有To所需的数据范围,而需要From打开的范围。 | |||
| // 这个范围是基于整个文件的,且上下界都取整到条带大小的整数倍,因此上界是有可能超过文件大小的。 | |||
| ToNodes map[ioswitch2.To]ops2.ToNode | |||
| TypedStreams []TypedStream | |||
| StreamRange exec.Range | |||
| EC cdssdk.ECRedundancy | |||
| ToNodes map[ioswitch2.To]ops2.ToNode | |||
| IndexedStreams []IndexedStream | |||
| StreamRange exec.Range | |||
| UseEC bool // 是否使用纠删码 | |||
| UseSegment bool // 是否使用分段 | |||
| } | |||
| func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy) error { | |||
| func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) error { | |||
| ctx := ParseContext{ | |||
| Ft: ft, | |||
| DAG: ops2.NewGraphNodeBuilder(), | |||
| ToNodes: make(map[ioswitch2.To]ops2.ToNode), | |||
| EC: ec, | |||
| } | |||
| // 分成两个阶段: | |||
| // 1. 基于From和To生成更多指令,初步匹配to的需求 | |||
| err := checkEncodingParams(&ctx) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| // 计算一下打开流的范围 | |||
| calcStreamRange(&ctx) | |||
| err := extend(&ctx) | |||
| err = extend(&ctx) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| // 2. 优化上一步生成的指令 | |||
| err = removeUnusedSegment(&ctx) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| // 对于删除指令的优化,需要反复进行,直到没有变化为止。 | |||
| // 从目前实现上来说不会死循环 | |||
| for { | |||
| @@ -79,17 +89,18 @@ func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy) | |||
| } | |||
| // 下面这些只需要执行一次,但需要按顺序 | |||
| removeUnusedFromNode(&ctx) | |||
| dropUnused(&ctx) | |||
| storeIPFSWriteResult(&ctx) | |||
| generateClone(&ctx) | |||
| generateRange(&ctx) | |||
| generateClone(&ctx) | |||
| return plan.Generate(ctx.DAG.Graph, blder) | |||
| } | |||
| func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.Var { | |||
| func findOutputStream(ctx *ParseContext, streamIndex ioswitch2.StreamIndex) *dag.Var { | |||
| var ret *dag.Var | |||
| for _, s := range ctx.TypedStreams { | |||
| if s.StreamType == streamType { | |||
| for _, s := range ctx.IndexedStreams { | |||
| if s.StreamIndex == streamIndex { | |||
| ret = s.Stream | |||
| break | |||
| } | |||
| @@ -97,35 +108,91 @@ func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.V | |||
| return ret | |||
| } | |||
| // 计算输入流的打开范围。会把流的范围按条带大小取整 | |||
| func calcStreamRange(ctx *ParseContext) { | |||
| stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K) | |||
| // 检查使用不同编码时参数是否设置到位 | |||
| func checkEncodingParams(ctx *ParseContext) error { | |||
| for _, f := range ctx.Ft.Froms { | |||
| if f.GetStreamIndex().IsEC() { | |||
| ctx.UseEC = true | |||
| if ctx.Ft.ECParam == nil { | |||
| return fmt.Errorf("EC encoding parameters not set") | |||
| } | |||
| } | |||
| rng := exec.Range{ | |||
| Offset: math.MaxInt64, | |||
| if f.GetStreamIndex().IsSegment() { | |||
| ctx.UseSegment = true | |||
| if ctx.Ft.SegmentParam == nil { | |||
| return fmt.Errorf("segment parameters not set") | |||
| } | |||
| } | |||
| } | |||
| for _, t := range ctx.Ft.Toes { | |||
| if t.GetStreamIndex().IsEC() { | |||
| ctx.UseEC = true | |||
| if ctx.Ft.ECParam == nil { | |||
| return fmt.Errorf("EC encoding parameters not set") | |||
| } | |||
| } | |||
| if t.GetStreamIndex().IsSegment() { | |||
| ctx.UseSegment = true | |||
| if ctx.Ft.SegmentParam == nil { | |||
| return fmt.Errorf("segment parameters not set") | |||
| } | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| // 计算输入流的打开范围。如果From或者To中包含EC的流,则会将打开范围扩大到条带大小的整数倍。 | |||
| func calcStreamRange(ctx *ParseContext) { | |||
| rng := exec.NewRange(math.MaxInt64, 0) | |||
| for _, to := range ctx.Ft.Toes { | |||
| if to.GetStreamType().IsRaw() { | |||
| strIdx := to.GetStreamIndex() | |||
| if strIdx.IsRaw() { | |||
| toRng := to.GetRange() | |||
| rng.ExtendStart(math2.Floor(toRng.Offset, stripSize)) | |||
| rng.ExtendStart(toRng.Offset) | |||
| if toRng.Length != nil { | |||
| rng.ExtendEnd(math2.Ceil(toRng.Offset+*toRng.Length, stripSize)) | |||
| rng.ExtendEnd(toRng.Offset + *toRng.Length) | |||
| } else { | |||
| rng.Length = nil | |||
| } | |||
| } else { | |||
| } else if strIdx.IsEC() { | |||
| toRng := to.GetRange() | |||
| blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.EC.ChunkSize)) | |||
| stripSize := ctx.Ft.ECParam.StripSize() | |||
| blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.Ft.ECParam.ChunkSize)) | |||
| rng.ExtendStart(blkStartIndex * stripSize) | |||
| if toRng.Length != nil { | |||
| blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.EC.ChunkSize)) | |||
| blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.Ft.ECParam.ChunkSize)) | |||
| rng.ExtendEnd(blkEndIndex * stripSize) | |||
| } else { | |||
| rng.Length = nil | |||
| } | |||
| } else if strIdx.IsSegment() { | |||
| // Segment节点的Range是相对于本段的,需要加上本段的起始位置 | |||
| toRng := to.GetRange() | |||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(strIdx.Index) | |||
| offset := toRng.Offset + segStart | |||
| rng.ExtendStart(offset) | |||
| if toRng.Length != nil { | |||
| rng.ExtendEnd(offset + *toRng.Length) | |||
| } else { | |||
| rng.Length = nil | |||
| } | |||
| } | |||
| } | |||
| if ctx.UseEC { | |||
| stripSize := ctx.Ft.ECParam.StripSize() | |||
| rng.ExtendStart(math2.Floor(rng.Offset, stripSize)) | |||
| if rng.Length != nil { | |||
| rng.ExtendEnd(math2.Ceil(rng.Offset+*rng.Length, stripSize)) | |||
| } | |||
| } | |||
| @@ -139,57 +206,112 @@ func extend(ctx *ParseContext) error { | |||
| return err | |||
| } | |||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||
| Stream: frNode.Output().Var, | |||
| StreamType: fr.GetStreamType(), | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: frNode.Output().Var, | |||
| StreamIndex: fr.GetStreamIndex(), | |||
| }) | |||
| // 对于完整文件的From,生成Split指令 | |||
| if fr.GetStreamType().IsRaw() { | |||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.EC.ChunkSize) | |||
| splitNode.Split(frNode.Output().Var, ctx.EC.K) | |||
| for i := 0; i < ctx.EC.K; i++ { | |||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||
| Stream: splitNode.SubStream(i), | |||
| StreamType: ioswitch2.ECSrteam(i), | |||
| }) | |||
| if fr.GetStreamIndex().IsRaw() { | |||
| // 只有输入输出需要EC编码的块时,才生成相关指令 | |||
| if ctx.UseEC { | |||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize) | |||
| splitNode.Split(frNode.Output().Var, ctx.Ft.ECParam.K) | |||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: splitNode.SubStream(i), | |||
| StreamIndex: ioswitch2.ECSrteam(i), | |||
| }) | |||
| } | |||
| } | |||
| // 同上 | |||
| if ctx.UseSegment { | |||
| splitNode := ctx.DAG.NewSegmentSplit(ctx.Ft.SegmentParam.Segments) | |||
| splitNode.SetInput(frNode.Output().Var) | |||
| for i := 0; i < len(ctx.Ft.SegmentParam.Segments); i++ { | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: splitNode.Segment(i), | |||
| StreamIndex: ioswitch2.SegmentStream(i), | |||
| }) | |||
| } | |||
| } | |||
| } | |||
| } | |||
| // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令 | |||
| ecInputStrs := make(map[int]*dag.Var) | |||
| for _, s := range ctx.TypedStreams { | |||
| if s.StreamType.IsEC() && ecInputStrs[s.StreamType.Index] == nil { | |||
| ecInputStrs[s.StreamType.Index] = s.Stream | |||
| if len(ecInputStrs) == ctx.EC.K { | |||
| break | |||
| if ctx.UseEC { | |||
| // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令 | |||
| ecInputStrs := make(map[int]*dag.Var) | |||
| for _, s := range ctx.IndexedStreams { | |||
| if s.StreamIndex.IsEC() && ecInputStrs[s.StreamIndex.Index] == nil { | |||
| ecInputStrs[s.StreamIndex.Index] = s.Stream | |||
| if len(ecInputStrs) == ctx.Ft.ECParam.K { | |||
| break | |||
| } | |||
| } | |||
| } | |||
| } | |||
| if len(ecInputStrs) == ctx.EC.K { | |||
| mulNode := ctx.DAG.NewECMultiply(ctx.EC) | |||
| if len(ecInputStrs) == ctx.Ft.ECParam.K { | |||
| mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam) | |||
| for i, s := range ecInputStrs { | |||
| mulNode.AddInput(s, i) | |||
| } | |||
| for i := 0; i < ctx.EC.N; i++ { | |||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||
| Stream: mulNode.NewOutput(i), | |||
| StreamType: ioswitch2.ECSrteam(i), | |||
| for i, s := range ecInputStrs { | |||
| mulNode.AddInput(s, i) | |||
| } | |||
| for i := 0; i < ctx.Ft.ECParam.N; i++ { | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: mulNode.NewOutput(i), | |||
| StreamIndex: ioswitch2.ECSrteam(i), | |||
| }) | |||
| } | |||
| joinNode := ctx.DAG.NewChunkedJoin(ctx.Ft.ECParam.ChunkSize) | |||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||
| // 不可能找不到流 | |||
| joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i))) | |||
| } | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: joinNode.Joined(), | |||
| StreamIndex: ioswitch2.RawStream(), | |||
| }) | |||
| } | |||
| } | |||
| joinNode := ctx.DAG.NewChunkedJoin(ctx.EC.ChunkSize) | |||
| for i := 0; i < ctx.EC.K; i++ { | |||
| // 不可能找不到流 | |||
| joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i))) | |||
| if ctx.UseSegment { | |||
| // 先假设有所有的顺序分段,生成Join指令,后续根据Range再实际计算是否缺少流 | |||
| joinNode := ctx.DAG.NewSegmentJoin(ctx.Ft.SegmentParam.Segments) | |||
| for i := 0; i < ctx.Ft.SegmentParam.SegmentCount(); i++ { | |||
| str := findOutputStream(ctx, ioswitch2.SegmentStream(i)) | |||
| if str != nil { | |||
| joinNode.SetInput(i, str) | |||
| } | |||
| } | |||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||
| Stream: joinNode.Joined(), | |||
| StreamType: ioswitch2.RawStream(), | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: joinNode.Joined(), | |||
| StreamIndex: ioswitch2.RawStream(), | |||
| }) | |||
| // SegmentJoin生成的Join指令可以用来生成EC块 | |||
| if ctx.UseEC { | |||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize) | |||
| splitNode.Split(joinNode.Joined(), ctx.Ft.ECParam.K) | |||
| mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam) | |||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||
| mulNode.AddInput(splitNode.SubStream(i), i) | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: splitNode.SubStream(i), | |||
| StreamIndex: ioswitch2.ECSrteam(i), | |||
| }) | |||
| } | |||
| for i := 0; i < ctx.Ft.ECParam.N; i++ { | |||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||
| Stream: mulNode.NewOutput(i), | |||
| StreamIndex: ioswitch2.ECSrteam(i), | |||
| }) | |||
| } | |||
| } | |||
| } | |||
| // 为每一个To找到一个输入流 | |||
| @@ -200,9 +322,9 @@ func extend(ctx *ParseContext) error { | |||
| } | |||
| ctx.ToNodes[to] = toNode | |||
| str := findOutputStream(ctx, to.GetStreamType()) | |||
| str := findOutputStream(ctx, to.GetStreamIndex()) | |||
| if str == nil { | |||
| return fmt.Errorf("no output stream found for data index %d", to.GetStreamType()) | |||
| return fmt.Errorf("no output stream found for data index %d", to.GetStreamIndex()) | |||
| } | |||
| toNode.SetInput(str) | |||
| @@ -213,26 +335,48 @@ func extend(ctx *ParseContext) error { | |||
| func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | |||
| var repRange exec.Range | |||
| var blkRange exec.Range | |||
| repRange.Offset = ctx.StreamRange.Offset | |||
| blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize) | |||
| if ctx.StreamRange.Length != nil { | |||
| repRngLen := *ctx.StreamRange.Length | |||
| repRange.Length = &repRngLen | |||
| } | |||
| blkRngLen := *ctx.StreamRange.Length / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize) | |||
| blkRange.Length = &blkRngLen | |||
| var blkRange exec.Range | |||
| if ctx.UseEC { | |||
| blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize) | |||
| if ctx.StreamRange.Length != nil { | |||
| blkRngLen := *ctx.StreamRange.Length / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize) | |||
| blkRange.Length = &blkRngLen | |||
| } | |||
| } | |||
| switch f := f.(type) { | |||
| case *ioswitch2.FromShardstore: | |||
| t := ctx.DAG.NewShardRead(f.Storage.StorageID, types.NewOpen(f.FileHash)) | |||
| t := ctx.DAG.NewShardRead(f, f.Storage.StorageID, types.NewOpen(f.FileHash)) | |||
| if f.StreamType.IsRaw() { | |||
| if f.StreamIndex.IsRaw() { | |||
| t.Open.WithNullableLength(repRange.Offset, repRange.Length) | |||
| } else { | |||
| } else if f.StreamIndex.IsEC() { | |||
| t.Open.WithNullableLength(blkRange.Offset, blkRange.Length) | |||
| } else if f.StreamIndex.IsSegment() { | |||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index) | |||
| segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index] | |||
| segEnd := segStart + segLen | |||
| // 打开的范围不超过本段的范围 | |||
| openOff := ctx.StreamRange.Offset - segStart | |||
| openOff = math2.Clamp(openOff, 0, segLen) | |||
| openLen := segLen | |||
| if ctx.StreamRange.Length != nil { | |||
| strEnd := ctx.StreamRange.Offset + *ctx.StreamRange.Length | |||
| openEnd := math2.Min(strEnd, segEnd) | |||
| openLen = openEnd - segStart - openOff | |||
| } | |||
| t.Open.WithNullableLength(openOff, &openLen) | |||
| } | |||
| switch addr := f.Hub.Address.(type) { | |||
| @@ -251,16 +395,36 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | |||
| return t, nil | |||
| case *ioswitch2.FromDriver: | |||
| n := ctx.DAG.NewFromDriver(f.Handle) | |||
| n := ctx.DAG.NewFromDriver(f, f.Handle) | |||
| n.Env().ToEnvDriver() | |||
| n.Env().Pinned = true | |||
| if f.StreamType.IsRaw() { | |||
| if f.StreamIndex.IsRaw() { | |||
| f.Handle.RangeHint.Offset = repRange.Offset | |||
| f.Handle.RangeHint.Length = repRange.Length | |||
| } else { | |||
| } else if f.StreamIndex.IsEC() { | |||
| f.Handle.RangeHint.Offset = blkRange.Offset | |||
| f.Handle.RangeHint.Length = blkRange.Length | |||
| } else if f.StreamIndex.IsSegment() { | |||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index) | |||
| segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index] | |||
| segEnd := segStart + segLen | |||
| // 打开的范围不超过本段的范围 | |||
| openOff := repRange.Offset - segStart | |||
| openOff = math2.Clamp(openOff, 0, segLen) | |||
| openLen := segLen | |||
| if repRange.Length != nil { | |||
| repEnd := repRange.Offset + *repRange.Length | |||
| openEnd := math2.Min(repEnd, segEnd) | |||
| openLen = openEnd - openOff | |||
| } | |||
| f.Handle.RangeHint.Offset = openOff | |||
| f.Handle.RangeHint.Length = &openLen | |||
| } | |||
| return n, nil | |||
| @@ -273,7 +437,7 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | |||
| func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) { | |||
| switch t := t.(type) { | |||
| case *ioswitch2.ToShardStore: | |||
| n := ctx.DAG.NewShardWrite(t.Storage.StorageID, t.FileHashStoreKey) | |||
| n := ctx.DAG.NewShardWrite(t, t.Storage.StorageID, t.FileHashStoreKey) | |||
| if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | |||
| return nil, err | |||
| @@ -284,14 +448,14 @@ func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) { | |||
| return n, nil | |||
| case *ioswitch2.ToDriver: | |||
| n := ctx.DAG.NewToDriver(t.Handle) | |||
| n := ctx.DAG.NewToDriver(t, t.Handle) | |||
| n.Env().ToEnvDriver() | |||
| n.Env().Pinned = true | |||
| return n, nil | |||
| case *ioswitch2.LoadToShared: | |||
| n := ctx.DAG.NewSharedLoad(t.Storage.StorageID, t.UserID, t.PackageID, t.Path) | |||
| n := ctx.DAG.NewSharedLoad(t, t.Storage.StorageID, t.UserID, t.PackageID, t.Path) | |||
| if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | |||
| return nil, err | |||
| @@ -321,6 +485,34 @@ func setEnvByAddress(n dag.Node, hub cdssdk.Hub, addr cdssdk.HubAddressInfo) err | |||
| return nil | |||
| } | |||
| // 从SegmentJoin中删除未使用的分段 | |||
| func removeUnusedSegment(ctx *ParseContext) error { | |||
| var err error | |||
| dag.WalkOnlyType[*ops2.SegmentJoinNode](ctx.DAG.Graph, func(node *ops2.SegmentJoinNode) bool { | |||
| start := ctx.StreamRange.Offset | |||
| var end *int64 | |||
| if ctx.StreamRange.Length != nil { | |||
| e := ctx.StreamRange.Offset + *ctx.StreamRange.Length | |||
| end = &e | |||
| } | |||
| segStart, segEnd := ctx.Ft.SegmentParam.CalcSegmentRange(start, end) | |||
| node.MarkUsed(segStart, segEnd) | |||
| for i := segStart; i < segEnd; i++ { | |||
| if node.InputStreams().Get(i) == nil { | |||
| err = fmt.Errorf("segment %v missed to join an raw stream", i) | |||
| return false | |||
| } | |||
| } | |||
| return true | |||
| }) | |||
| return err | |||
| } | |||
| // 删除输出流未被使用的Join指令 | |||
| func removeUnusedJoin(ctx *ParseContext) bool { | |||
| changed := false | |||
| @@ -352,6 +544,8 @@ func removeUnusedMultiplyOutput(ctx *ParseContext) bool { | |||
| node.OutputIndexes[i2] = -2 | |||
| changed = true | |||
| } | |||
| // TODO2 没有修改SlotIndex | |||
| node.OutputStreams().SetRawArray(lo2.RemoveAllDefault(outArr)) | |||
| node.OutputIndexes = lo2.RemoveAll(node.OutputIndexes, -2) | |||
| @@ -504,6 +698,20 @@ func pin(ctx *ParseContext) bool { | |||
| return changed | |||
| } | |||
| // 删除未使用的From流,不会删除FromDriver | |||
| func removeUnusedFromNode(ctx *ParseContext) { | |||
| dag.WalkOnlyType[ops2.FromNode](ctx.DAG.Graph, func(node ops2.FromNode) bool { | |||
| if _, ok := node.(*ops2.FromDriverNode); ok { | |||
| return true | |||
| } | |||
| if node.Output().Var == nil { | |||
| ctx.DAG.RemoveNode(node) | |||
| } | |||
| return true | |||
| }) | |||
| } | |||
| // 对于所有未使用的流,增加Drop指令 | |||
| func dropUnused(ctx *ParseContext) { | |||
| ctx.DAG.Walk(func(node dag.Node) bool { | |||
| @@ -539,10 +747,10 @@ func generateRange(ctx *ParseContext) { | |||
| to := ctx.Ft.Toes[i] | |||
| toNode := ctx.ToNodes[to] | |||
| toStrType := to.GetStreamType() | |||
| toStrIdx := to.GetStreamIndex() | |||
| toRng := to.GetRange() | |||
| if toStrType.IsRaw() { | |||
| if toStrIdx.IsRaw() { | |||
| n := ctx.DAG.NewRange() | |||
| toInput := toNode.Input() | |||
| *n.Env() = *toInput.Var.From().Node.Env() | |||
| @@ -553,11 +761,11 @@ func generateRange(ctx *ParseContext) { | |||
| toInput.Var.StreamNotTo(toNode, toInput.Index) | |||
| toNode.SetInput(rnged) | |||
| } else { | |||
| stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K) | |||
| } else if toStrIdx.IsEC() { | |||
| stripSize := int64(ctx.Ft.ECParam.ChunkSize * ctx.Ft.ECParam.K) | |||
| blkStartIdx := ctx.StreamRange.Offset / stripSize | |||
| blkStart := blkStartIdx * int64(ctx.EC.ChunkSize) | |||
| blkStart := blkStartIdx * int64(ctx.Ft.ECParam.ChunkSize) | |||
| n := ctx.DAG.NewRange() | |||
| toInput := toNode.Input() | |||
| @@ -568,6 +776,26 @@ func generateRange(ctx *ParseContext) { | |||
| }) | |||
| toInput.Var.StreamNotTo(toNode, toInput.Index) | |||
| toNode.SetInput(rnged) | |||
| } else if toStrIdx.IsSegment() { | |||
| // if frNode, ok := toNode.Input().Var.From().Node.(ops2.FromNode); ok { | |||
| // // 目前只有To也是分段时,才可能对接一个提供分段的From,此时不需要再生成Range指令 | |||
| // if frNode.GetFrom().GetStreamIndex().IsSegment() { | |||
| // continue | |||
| // } | |||
| // } | |||
| // segStart := ctx.Ft.SegmentParam.CalcSegmentStart(toStrIdx.Index) | |||
| // strStart := segStart + toRng.Offset | |||
| // n := ctx.DAG.NewRange() | |||
| // toInput := toNode.Input() | |||
| // *n.Env() = *toInput.Var.From().Node.Env() | |||
| // rnged := n.RangeStream(toInput.Var, exec.Range{ | |||
| // Offset: strStart - ctx.StreamRange.Offset, | |||
| // Length: toRng.Length, | |||
| // }) | |||
| // toInput.Var.StreamNotTo(toNode, toInput.Index) | |||
| // toNode.SetInput(rnged) | |||
| } | |||
| } | |||
| } | |||
| @@ -39,7 +39,7 @@ type CreateLoadResult struct { | |||
| func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) error { | |||
| uploadTime := time.Now() | |||
| ft := ioswitch2.NewFromTo() | |||
| ft := ioswitch2.FromTo{} | |||
| fromExec, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream()) | |||
| ft.AddFrom(fromExec) | |||
| for _, stg := range u.targetStgs { | |||
| @@ -48,7 +48,7 @@ func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) e | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -47,7 +47,7 @@ func (w *UpdateUploader) Upload(path string, size int64, stream io.Reader) error | |||
| ft.AddFrom(fromExec).AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg.Storage, ioswitch2.RawStream(), "fileHash")) | |||
| plans := exec.NewPlanBuilder() | |||
| err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -142,43 +142,47 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) { | |||
| switch newRed := newRed.(type) { | |||
| case *cdssdk.RepRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep") | |||
| updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs) | |||
| updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs, userAllStorages) | |||
| case *cdssdk.ECRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec") | |||
| updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs) | |||
| updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs, userAllStorages) | |||
| case *cdssdk.LRCRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc") | |||
| updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages) | |||
| updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages, userAllStorages) | |||
| case *cdssdk.SegmentRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment") | |||
| updating, err = t.noneToSeg(execCtx, obj, newRed, selectedStorages, userAllStorages) | |||
| } | |||
| case *cdssdk.RepRedundancy: | |||
| switch newRed := newRed.(type) { | |||
| case *cdssdk.RepRedundancy: | |||
| updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs) | |||
| updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs, userAllStorages) | |||
| case *cdssdk.ECRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec") | |||
| updating, err = t.repToEC(execCtx, obj, newRed, newECStgs) | |||
| updating, err = t.repToEC(execCtx, obj, newRed, newECStgs, userAllStorages) | |||
| } | |||
| case *cdssdk.ECRedundancy: | |||
| switch newRed := newRed.(type) { | |||
| case *cdssdk.RepRedundancy: | |||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep") | |||
| updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs) | |||
| updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs, userAllStorages) | |||
| case *cdssdk.ECRedundancy: | |||
| uploadStorages := t.rechooseStoragesForEC(obj, srcRed, userAllStorages) | |||
| updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages) | |||
| updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages) | |||
| } | |||
| case *cdssdk.LRCRedundancy: | |||
| switch newRed := newRed.(type) { | |||
| case *cdssdk.LRCRedundancy: | |||
| uploadStorages := t.rechooseStoragesForLRC(obj, srcRed, userAllStorages) | |||
| updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages) | |||
| updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages) | |||
| } | |||
| } | |||
| @@ -205,11 +209,23 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) { | |||
| func (t *CheckPackageRedundancy) chooseRedundancy(obj stgmod.ObjectDetail, userAllStgs map[cdssdk.StorageID]*StorageLoadInfo) (cdssdk.Redundancy, []*StorageLoadInfo) { | |||
| switch obj.Object.Redundancy.(type) { | |||
| case *cdssdk.NoneRedundancy: | |||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||
| return &cdssdk.DefaultECRedundancy, newStgs | |||
| if obj.Object.Size > config.Cfg().ECFileSizeThreshold { | |||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||
| return &cdssdk.DefaultECRedundancy, newStgs | |||
| } | |||
| // newLRCStorages := t.chooseNewStoragesForLRC(&cdssdk.DefaultLRCRedundancy, userAllStorages) | |||
| // return &cdssdk.DefaultLRCRedundancy, newLRCStorages | |||
| return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs) | |||
| case *cdssdk.RepRedundancy: | |||
| if obj.Object.Size > config.Cfg().ECFileSizeThreshold { | |||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||
| return &cdssdk.DefaultECRedundancy, newStgs | |||
| } | |||
| case *cdssdk.ECRedundancy: | |||
| if obj.Object.Size <= config.Cfg().ECFileSizeThreshold { | |||
| return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs) | |||
| } | |||
| case *cdssdk.LRCRedundancy: | |||
| newLRCStgs := t.rechooseStoragesForLRC(obj, &cdssdk.DefaultLRCRedundancy, userAllStgs) | |||
| @@ -278,6 +294,14 @@ func (t *CheckPackageRedundancy) chooseNewStoragesForLRC(red *cdssdk.LRCRedundan | |||
| return t.chooseSoManyStorages(red.N, sortedStorages) | |||
| } | |||
| func (t *CheckPackageRedundancy) chooseNewStoragesForSeg(segCount int, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo { | |||
| sortedStorages := sort2.Sort(lo.Values(allStgs), func(left *StorageLoadInfo, right *StorageLoadInfo) int { | |||
| return sort2.Cmp(right.AccessAmount, left.AccessAmount) | |||
| }) | |||
| return t.chooseSoManyStorages(segCount, sortedStorages) | |||
| } | |||
| func (t *CheckPackageRedundancy) rechooseStoragesForRep(mostBlockStgIDs []cdssdk.StorageID, red *cdssdk.RepRedundancy, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo { | |||
| type rechooseStorage struct { | |||
| *StorageLoadInfo | |||
| @@ -421,25 +445,16 @@ func (t *CheckPackageRedundancy) chooseSoManyStorages(count int, stgs []*Storage | |||
| return chosen | |||
| } | |||
| func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| if len(obj.Blocks) == 0 { | |||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | |||
| } | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||
| } | |||
| if getStgs.Storages[0] == nil { | |||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||
| if !ok { | |||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||
| } | |||
| if getStgs.Storages[0].MasterHub == nil { | |||
| if srcStg.Storage.MasterHub == nil { | |||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||
| } | |||
| @@ -447,13 +462,13 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object | |||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||
| for i, stg := range uploadStgs { | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -483,35 +498,27 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| if len(obj.Blocks) == 0 { | |||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | |||
| } | |||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||
| } | |||
| if getStgs.Storages[0] == nil { | |||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||
| if !ok { | |||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||
| } | |||
| if getStgs.Storages[0].MasterHub == nil { | |||
| if srcStg.Storage.MasterHub == nil { | |||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||
| } | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||
| ft.ECParam = red | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||
| for i := 0; i < red.N; i++ { | |||
| ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err = parser.Parse(ft, plans, *red) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -540,25 +547,16 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| if len(obj.Blocks) == 0 { | |||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | |||
| } | |||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||
| } | |||
| if getStgs.Storages[0] == nil { | |||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||
| if !ok { | |||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||
| } | |||
| if getStgs.Storages[0].MasterHub == nil { | |||
| if srcStg.Storage.MasterHub == nil { | |||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||
| } | |||
| @@ -568,7 +566,7 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err = lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, -1), toes, plans) | |||
| err := lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, -1), toes, plans) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -597,25 +595,70 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.SegmentRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| if len(obj.Blocks) == 0 { | |||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | |||
| } | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||
| if !ok { | |||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||
| } | |||
| if srcStg.Storage.MasterHub == nil { | |||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||
| } | |||
| // 如果选择的备份节点都是同一个,那么就只要上传一次 | |||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.SegmentParam = red | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||
| for i, stg := range uploadStgs { | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.SegmentStream(i), fmt.Sprintf("%d", i))) | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||
| // TODO 添加依赖 | |||
| execCtx := exec.NewExecContext() | |||
| exec.SetValueByType(execCtx, ctx.Args.StgMgr) | |||
| ret, err := plans.Execute(execCtx).Wait(context.Background()) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||
| return nil, fmt.Errorf("executing io plan: %w", err) | |||
| } | |||
| var blocks []stgmod.ObjectBlock | |||
| for i, stg := range uploadStgs { | |||
| blocks = append(blocks, stgmod.ObjectBlock{ | |||
| ObjectID: obj.Object.ObjectID, | |||
| Index: i, | |||
| StorageID: stg.Storage.Storage.StorageID, | |||
| FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, | |||
| }) | |||
| } | |||
| return &coormq.UpdatingObjectRedundancy{ | |||
| ObjectID: obj.Object.ObjectID, | |||
| Redundancy: red, | |||
| Blocks: blocks, | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| if len(obj.Blocks) == 0 { | |||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | |||
| } | |||
| if getStgs.Storages[0] == nil { | |||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||
| if !ok { | |||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||
| } | |||
| if getStgs.Storages[0].MasterHub == nil { | |||
| if srcStg.Storage.MasterHub == nil { | |||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||
| } | |||
| @@ -623,13 +666,13 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD | |||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||
| for i, stg := range uploadStgs { | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | |||
| } | |||
| plans := exec.NewPlanBuilder() | |||
| err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||
| err := parser.Parse(ft, plans) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -659,23 +702,28 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| return t.noneToEC(ctx, obj, red, uploadStorages) | |||
| func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| return t.noneToEC(ctx, obj, red, uploadStorages, allStgs) | |||
| } | |||
| func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| var chosenBlocks []stgmod.GrouppedObjectBlock | |||
| var chosenBlockIndexes []int | |||
| var chosenBlockStg []stgmod.StorageDetail | |||
| for _, block := range obj.GroupBlocks() { | |||
| if len(block.StorageIDs) > 0 { | |||
| // TODO 考虑选择最优的节点 | |||
| stg, ok := allStgs[block.StorageIDs[0]] | |||
| if !ok { | |||
| continue | |||
| } | |||
| if stg.Storage.MasterHub == nil { | |||
| continue | |||
| } | |||
| chosenBlocks = append(chosenBlocks, block) | |||
| chosenBlockIndexes = append(chosenBlockIndexes, block.Index) | |||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||
| } | |||
| if len(chosenBlocks) == srcRed.K { | |||
| @@ -694,9 +742,10 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||
| planBlder := exec.NewPlanBuilder() | |||
| for i := range uploadStgs { | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.ECParam = srcRed | |||
| for _, block := range chosenBlocks { | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(block.Index))) | |||
| for i2, block := range chosenBlocks { | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index))) | |||
| } | |||
| len := obj.Object.Size | |||
| @@ -705,7 +754,7 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||
| Length: &len, | |||
| })) | |||
| err := parser.Parse(ft, planBlder, *srcRed) | |||
| err := parser.Parse(ft, planBlder) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -736,19 +785,23 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| grpBlocks := obj.GroupBlocks() | |||
| var chosenBlocks []stgmod.GrouppedObjectBlock | |||
| var chosenBlockStg []stgmod.StorageDetail | |||
| for _, block := range grpBlocks { | |||
| if len(block.StorageIDs) > 0 { | |||
| stg, ok := allStgs[block.StorageIDs[0]] | |||
| if !ok { | |||
| continue | |||
| } | |||
| if stg.Storage.MasterHub == nil { | |||
| continue | |||
| } | |||
| chosenBlocks = append(chosenBlocks, block) | |||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||
| } | |||
| if len(chosenBlocks) == srcRed.K { | |||
| @@ -786,15 +839,15 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet | |||
| // 否则就要重建出这个节点需要的块 | |||
| ft := ioswitch2.NewFromTo() | |||
| for _, block := range chosenBlocks { | |||
| stg := stg.Storage | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(block.Index))) | |||
| ft.ECParam = srcRed | |||
| for i2, block := range chosenBlocks { | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index))) | |||
| } | |||
| // 输出只需要自己要保存的那一块 | |||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | |||
| err := parser.Parse(ft, planBlder, *srcRed) | |||
| err := parser.Parse(ft, planBlder) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||
| } | |||
| @@ -830,12 +883,7 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet | |||
| }, nil | |||
| } | |||
| func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||
| if err != nil { | |||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||
| } | |||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||
| func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| blocksGrpByIndex := obj.GroupBlocks() | |||
| @@ -870,7 +918,7 @@ func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectD | |||
| // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadStorages) | |||
| } | |||
| return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages) | |||
| return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages, allStgs) | |||
| } | |||
| /* | |||
| @@ -939,11 +987,22 @@ TODO2 修复这一块的代码 | |||
| }, nil | |||
| } | |||
| */ | |||
| func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||
| var chosenBlocks []stgmod.GrouppedObjectBlock | |||
| var chosenBlockStg []stgmod.StorageDetail | |||
| for _, block := range grpBlocks { | |||
| if len(block.StorageIDs) > 0 && block.Index < red.M() { | |||
| stg, ok := allStgs[block.StorageIDs[0]] | |||
| if !ok { | |||
| continue | |||
| } | |||
| if stg.Storage.MasterHub == nil { | |||
| continue | |||
| } | |||
| chosenBlocks = append(chosenBlocks, block) | |||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||
| } | |||
| if len(chosenBlocks) == red.K { | |||
| @@ -982,10 +1041,8 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O | |||
| // 否则就要重建出这个节点需要的块 | |||
| for _, block := range chosenBlocks { | |||
| fmt.Printf("b: %v\n", block.Index) | |||
| stg := storage.Storage | |||
| froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *stg.MasterHub, stg.Storage, block.Index)) | |||
| for i2, block := range chosenBlocks { | |||
| froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, block.Index)) | |||
| } | |||
| // 输出只需要自己要保存的那一块 | |||
| @@ -745,7 +745,7 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st | |||
| toStg := allStgInfos[solu.blockList[i].StorageID] | |||
| ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, toStg.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID))) | |||
| err := parser.Parse(ft, planBld, cdssdk.DefaultECRedundancy) | |||
| err := parser.Parse(ft, planBld) | |||
| if err != nil { | |||
| // TODO 错误处理 | |||
| continue | |||
| @@ -798,13 +798,14 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg | |||
| for id, idxs := range reconstrct { | |||
| ft := ioswitch2.NewFromTo() | |||
| ft.ECParam = ecRed | |||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.RawStream())) | |||
| for _, i := range *idxs { | |||
| ft.AddTo(ioswitch2.NewToShardStore(*allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d.%d", obj.Object.ObjectID, i))) | |||
| } | |||
| err := parser.Parse(ft, planBld, *ecRed) | |||
| err := parser.Parse(ft, planBld) | |||
| if err != nil { | |||
| // TODO 错误处理 | |||
| continue | |||