| @@ -1,6 +1,5 @@ | |||||
| package cmdline | package cmdline | ||||
| /* | |||||
| import ( | import ( | ||||
| "context" | "context" | ||||
| "fmt" | "fmt" | ||||
| @@ -13,54 +12,31 @@ import ( | |||||
| stgglb "gitlink.org.cn/cloudream/storage/common/globals" | stgglb "gitlink.org.cn/cloudream/storage/common/globals" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser" | "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc" | |||||
| lrcparser "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc/parser" | |||||
| coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" | coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" | ||||
| ) | ) | ||||
| func init() { | func init() { | ||||
| rootCmd.AddCommand(&cobra.Command{ | rootCmd.AddCommand(&cobra.Command{ | ||||
| Use: "test", | |||||
| Short: "test", | |||||
| // Args: cobra.ExactArgs(1), | |||||
| Use: "test2", | |||||
| Short: "test2", | |||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| // cmdCtx := GetCmdCtx(cmd) | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | coorCli, err := stgglb.CoordinatorMQPool.Acquire() | ||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | defer stgglb.CoordinatorMQPool.Release(coorCli) | ||||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.AddFrom(ioswitch2.NewFromNode("Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", &nodes.Nodes[0], -1)) | |||||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], -1, "asd")) | |||||
| le := int64(3) | |||||
| toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 5, Length: &le}) | |||||
| ft.AddTo(toExec) | |||||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 0, "0")) | |||||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 1, "1")) | |||||
| ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 2, "2")) | |||||
| // ft.AddFrom(ioswitch2.NewFromNode("QmS2s8GRYHEurXL7V1zUtKvf2H1BGcQc5NN1T1hiSnWvbd", &nodes.Nodes[0], 1)) | |||||
| // ft.AddFrom(ioswitch2.NewFromNode("QmUgUEUMzdnjPNx6xu9PDGXpSyXTk8wzPWvyYZ9zasE1WW", &nodes.Nodes[1], 2)) | |||||
| // le := int64(5) | |||||
| // toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 3, Length: &le}) | |||||
| // toExec, hd := plans.NewToExecutorWithRange(1, plans.Range{Offset: 0, Length: nil}) | |||||
| // toExec2, hd2 := plans.NewToExecutorWithRange(2, plans.Range{Offset: 0, Length: nil}) | |||||
| // ft.AddTo(toExec) | |||||
| // ft.AddTo(toExec2) | |||||
| // fromExec, hd := plans.NewFromExecutor(-1) | |||||
| // ft.AddFrom(fromExec) | |||||
| // ft.AddTo(plans.NewToNode(nodes.Nodes[1], -1, "asd")) | |||||
| parser := parser.NewParser(cdssdk.DefaultECRedundancy) | |||||
| ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3) | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.RawStream())) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0), "0")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1), "1")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2), "2")) | |||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = parser.Parse(ft, plans) | err = parser.Parse(ft, plans) | ||||
| @@ -68,7 +44,9 @@ func init() { | |||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| exec := plans.Execute() | |||||
| fmt.Printf("plans: %v\n", plans) | |||||
| exec := plans.Execute(exec.NewExecContext()) | |||||
| fut := future.NewSetVoid() | fut := future.NewSetVoid() | ||||
| go func() { | go func() { | ||||
| @@ -77,176 +55,130 @@ func init() { | |||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| fmt.Printf("mp: %+v\n", mp) | |||||
| fmt.Printf("0: %v, 1: %v, 2: %v\n", mp["0"], mp["1"], mp["2"]) | |||||
| fut.SetVoid() | fut.SetVoid() | ||||
| }() | }() | ||||
| go func() { | |||||
| // exec.BeginWrite(io.NopCloser(bytes.NewBuffer([]byte("hello world"))), hd) | |||||
| // if err != nil { | |||||
| // panic(err) | |||||
| // } | |||||
| str, err := exec.BeginRead(hd) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| defer str.Close() | |||||
| data, err := io.ReadAll(str) | |||||
| if err != nil && err != io.EOF { | |||||
| panic(err) | |||||
| } | |||||
| fmt.Printf("data: %v(%v)\n", string(data), len(data)) | |||||
| }() | |||||
| fut.Wait(context.TODO()) | fut.Wait(context.TODO()) | ||||
| }, | }, | ||||
| }) | }) | ||||
| // rootCmd.AddCommand(&cobra.Command{ | |||||
| // Use: "test", | |||||
| // Short: "test", | |||||
| // // Args: cobra.ExactArgs(1), | |||||
| // Run: func(cmd *cobra.Command, args []string) { | |||||
| // cmdCtx := GetCmdCtx(cmd) | |||||
| // file, _ := cmdCtx.Cmdline.Svc.ObjectSvc().Download(1, downloader.DownloadReqeust{ | |||||
| // ObjectID: 27379, | |||||
| // Length: -1, | |||||
| // }) | |||||
| // data, _ := io.ReadAll(file.File) | |||||
| // fmt.Printf("data: %v(%v)\n", string(data), len(data)) | |||||
| // }, | |||||
| // }) | |||||
| rootCmd.AddCommand(&cobra.Command{ | rootCmd.AddCommand(&cobra.Command{ | ||||
| Use: "test3", | |||||
| Short: "test3", | |||||
| // Args: cobra.ExactArgs(1), | |||||
| Use: "test", | |||||
| Short: "test", | |||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| // cmdCtx := GetCmdCtx(cmd) | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | coorCli, err := stgglb.CoordinatorMQPool.Acquire() | ||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | defer stgglb.CoordinatorMQPool.Release(coorCli) | ||||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| red := cdssdk.DefaultLRCRedundancy | |||||
| var toes []ioswitchlrc.To | |||||
| for i := 0; i < red.N; i++ { | |||||
| toes = append(toes, ioswitchlrc.NewToNode(nodes.Nodes[i%2], i, fmt.Sprintf("%d", i))) | |||||
| } | |||||
| ft := ioswitch2.NewFromTo() | |||||
| ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3) | |||||
| ft.ECParam = &cdssdk.DefaultECRedundancy | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore("22CC59CE3297F78F2D20DC1E33181B77F21E6782097C94E1664F99F129834069", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0))) | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore("5EAC20EB3EBC7B5FA176C5BD1C01041FB2A6D14C35D6A232CA83D7F1E4B01ADE", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1))) | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore("A9BC1802F37100C80C72A1D6E8F53C0E0B73F85F99153D8C78FB01CEC9D8D903", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2))) | |||||
| toDrv, drvStr := ioswitch2.NewToDriverWithRange(ioswitch2.RawStream(), exec.NewRange(500, 500)) | |||||
| ft.AddTo(toDrv) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2")) | |||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = lrcparser.Encode(ioswitchlrc.NewFromNode("QmNspjDLxQbAsuh37jRXKvLWHE2f7JpqY4HEJ8x7Jgbzqa", &nodes.Nodes[0], -1), toes, plans) | |||||
| err = parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||||
| } | } | ||||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||||
| } | |||||
| fmt.Printf("plans: %v\n", plans) | |||||
| fmt.Printf("ioRet: %v\n", ioRet) | |||||
| }, | |||||
| }) | |||||
| exec := plans.Execute(exec.NewExecContext()) | |||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test4", | |||||
| Short: "test4", | |||||
| // Args: cobra.ExactArgs(1), | |||||
| Run: func(cmd *cobra.Command, args []string) { | |||||
| // cmdCtx := GetCmdCtx(cmd) | |||||
| fut := future.NewSetVoid() | |||||
| go func() { | |||||
| mp, err := exec.Wait(context.Background()) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| for k, v := range mp { | |||||
| fmt.Printf("%s: %v\n", k, v) | |||||
| } | |||||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| fut.SetVoid() | |||||
| }() | |||||
| // red := cdssdk.DefaultLRCRedundancy | |||||
| go func() { | |||||
| str, err := exec.BeginRead(drvStr) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| plans := exec.NewPlanBuilder() | |||||
| err = lrcparser.ReconstructGroup([]ioswitchlrc.From{ | |||||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 0), | |||||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 1), | |||||
| }, []ioswitchlrc.To{ | |||||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"), | |||||
| }, plans) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||||
| } | |||||
| data, err := io.ReadAll(str) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||||
| } | |||||
| fmt.Printf("read(%v): %s\n", len(data), string(data)) | |||||
| }() | |||||
| fmt.Printf("ioRet: %v\n", ioRet) | |||||
| fut.Wait(context.TODO()) | |||||
| }, | }, | ||||
| }) | }) | ||||
| rootCmd.AddCommand(&cobra.Command{ | rootCmd.AddCommand(&cobra.Command{ | ||||
| Use: "test3", | Use: "test3", | ||||
| Short: "test3", | Short: "test3", | ||||
| // Args: cobra.ExactArgs(1), | |||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| // cmdCtx := GetCmdCtx(cmd) | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | coorCli, err := stgglb.CoordinatorMQPool.Acquire() | ||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | defer stgglb.CoordinatorMQPool.Release(coorCli) | ||||
| nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2})) | |||||
| stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2})) | |||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| } | } | ||||
| // red := cdssdk.DefaultLRCRedundancy | |||||
| ft := ioswitch2.NewFromTo() | |||||
| ft.ECParam = &cdssdk.DefaultECRedundancy | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.RawStream())) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1")) | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2")) | |||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| le := int64(1293) | |||||
| err = lrcparser.ReconstructAny([]ioswitchlrc.From{ | |||||
| ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[0], 0), | |||||
| ioswitchlrc.NewFromNode("QmQBKncEDqxw3BrGr3th3gS3jUC2fizGz1w29ZxxrrKfNv", &nodes.Nodes[0], 2), | |||||
| }, []ioswitchlrc.To{ | |||||
| ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], -1, "-1", exec.Range{0, &le}), | |||||
| ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], 0, "0", exec.Range{10, &le}), | |||||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 1, "1"), | |||||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 2, "2"), | |||||
| ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"), | |||||
| }, plans) | |||||
| err = parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| panic(err) | panic(err) | ||||
| // return nil, fmt.Errorf("parsing plan: %w", err) | |||||
| } | } | ||||
| ioRet, err := plans.Execute().Wait(context.TODO()) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| // return nil, fmt.Errorf("executing io plan: %w", err) | |||||
| } | |||||
| fmt.Printf("plans: %v\n", plans) | |||||
| exec := plans.Execute(exec.NewExecContext()) | |||||
| fmt.Printf("ioRet: %v\n", ioRet) | |||||
| fut := future.NewSetVoid() | |||||
| go func() { | |||||
| mp, err := exec.Wait(context.Background()) | |||||
| if err != nil { | |||||
| panic(err) | |||||
| } | |||||
| for k, v := range mp { | |||||
| fmt.Printf("%s: %v\n", k, v) | |||||
| } | |||||
| fut.SetVoid() | |||||
| }() | |||||
| fut.Wait(context.TODO()) | |||||
| }, | }, | ||||
| }) | }) | ||||
| } | } | ||||
| */ | |||||
| @@ -400,7 +400,7 @@ func (iter *DownloadObjectIterator) downloadFromStorage(stg *stgmod.StorageDetai | |||||
| strHandle = handle | strHandle = handle | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| if err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy); err != nil { | |||||
| if err := parser.Parse(ft, plans); err != nil { | |||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -199,6 +199,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||||
| } | } | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.ECParam = s.red | |||||
| for _, b := range s.blocks { | for _, b := range s.blocks { | ||||
| stg := b.Storage | stg := b.Storage | ||||
| ft.AddFrom(ioswitch2.NewFromShardstore(b.Block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(b.Block.Index))) | ft.AddFrom(ioswitch2.NewFromShardstore(b.Block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(b.Block.Index))) | ||||
| @@ -210,7 +211,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||||
| ft.AddTo(toExec) | ft.AddTo(toExec) | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err := parser.Parse(ft, plans, *s.red) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return 0, err | return 0, err | ||||
| } | } | ||||
| @@ -6,7 +6,7 @@ import ( | |||||
| ) | ) | ||||
| type From interface { | type From interface { | ||||
| GetStreamType() StreamType | |||||
| GetStreamIndex() StreamIndex | |||||
| } | } | ||||
| type To interface { | type To interface { | ||||
| @@ -14,60 +14,64 @@ type To interface { | |||||
| // 如果DataIndex == -1,则表示在整个文件的范围。 | // 如果DataIndex == -1,则表示在整个文件的范围。 | ||||
| // 如果DataIndex >= 0,则表示在文件的某个分片的范围。 | // 如果DataIndex >= 0,则表示在文件的某个分片的范围。 | ||||
| GetRange() exec.Range | GetRange() exec.Range | ||||
| GetStreamType() StreamType | |||||
| GetStreamIndex() StreamIndex | |||||
| } | } | ||||
| const ( | const ( | ||||
| // 未处理的完整文件流 | // 未处理的完整文件流 | ||||
| StreamTypeRaw = iota | |||||
| StreamIndexRaw = iota | |||||
| // EC编码的某一块的流 | // EC编码的某一块的流 | ||||
| StreamTypeEC | |||||
| StreamIndexEC | |||||
| // 分段编码的某一段的流 | // 分段编码的某一段的流 | ||||
| StreamTypeSegment | |||||
| StreamIndexSegment | |||||
| ) | ) | ||||
| type StreamType struct { | |||||
| type StreamIndex struct { | |||||
| Type int | Type int | ||||
| Index int | Index int | ||||
| } | } | ||||
| func RawStream() StreamType { | |||||
| return StreamType{ | |||||
| Type: StreamTypeRaw, | |||||
| func RawStream() StreamIndex { | |||||
| return StreamIndex{ | |||||
| Type: StreamIndexRaw, | |||||
| } | } | ||||
| } | } | ||||
| func ECSrteam(index int) StreamType { | |||||
| return StreamType{ | |||||
| Type: StreamTypeEC, | |||||
| func ECSrteam(index int) StreamIndex { | |||||
| return StreamIndex{ | |||||
| Type: StreamIndexEC, | |||||
| Index: index, | Index: index, | ||||
| } | } | ||||
| } | } | ||||
| func SegmentStream(index int) StreamType { | |||||
| return StreamType{ | |||||
| Type: StreamTypeSegment, | |||||
| func SegmentStream(index int) StreamIndex { | |||||
| return StreamIndex{ | |||||
| Type: StreamIndexSegment, | |||||
| Index: index, | Index: index, | ||||
| } | } | ||||
| } | } | ||||
| func (s StreamType) IsRaw() bool { | |||||
| return s.Type == StreamTypeRaw | |||||
| func (s StreamIndex) IsRaw() bool { | |||||
| return s.Type == StreamIndexRaw | |||||
| } | } | ||||
| func (s StreamType) IsEC() bool { | |||||
| return s.Type == StreamTypeEC | |||||
| func (s StreamIndex) IsEC() bool { | |||||
| return s.Type == StreamIndexEC | |||||
| } | } | ||||
| func (s StreamType) IsSegment() bool { | |||||
| return s.Type == StreamTypeSegment | |||||
| func (s StreamIndex) IsSegment() bool { | |||||
| return s.Type == StreamIndexSegment | |||||
| } | } | ||||
| type FromTos []FromTo | type FromTos []FromTo | ||||
| type FromTo struct { | type FromTo struct { | ||||
| Froms []From | |||||
| Toes []To | |||||
| // 如果输入或者输出用到了EC编码的流,则需要提供EC参数。 | |||||
| ECParam *cdssdk.ECRedundancy | |||||
| // 同上 | |||||
| SegmentParam *cdssdk.SegmentRedundancy | |||||
| Froms []From | |||||
| Toes []To | |||||
| } | } | ||||
| func NewFromTo() FromTo { | func NewFromTo() FromTo { | ||||
| @@ -85,69 +89,69 @@ func (ft *FromTo) AddTo(to To) *FromTo { | |||||
| } | } | ||||
| type FromDriver struct { | type FromDriver struct { | ||||
| Handle *exec.DriverWriteStream | |||||
| StreamType StreamType | |||||
| Handle *exec.DriverWriteStream | |||||
| StreamIndex StreamIndex | |||||
| } | } | ||||
| func NewFromDriver(strType StreamType) (*FromDriver, *exec.DriverWriteStream) { | |||||
| func NewFromDriver(strIdx StreamIndex) (*FromDriver, *exec.DriverWriteStream) { | |||||
| handle := &exec.DriverWriteStream{ | handle := &exec.DriverWriteStream{ | ||||
| RangeHint: &exec.Range{}, | RangeHint: &exec.Range{}, | ||||
| } | } | ||||
| return &FromDriver{ | return &FromDriver{ | ||||
| Handle: handle, | |||||
| StreamType: strType, | |||||
| Handle: handle, | |||||
| StreamIndex: strIdx, | |||||
| }, handle | }, handle | ||||
| } | } | ||||
| func (f *FromDriver) GetStreamType() StreamType { | |||||
| return f.StreamType | |||||
| func (f *FromDriver) GetStreamIndex() StreamIndex { | |||||
| return f.StreamIndex | |||||
| } | } | ||||
| type FromShardstore struct { | type FromShardstore struct { | ||||
| FileHash cdssdk.FileHash | |||||
| Hub cdssdk.Hub | |||||
| Storage cdssdk.Storage | |||||
| StreamType StreamType | |||||
| FileHash cdssdk.FileHash | |||||
| Hub cdssdk.Hub | |||||
| Storage cdssdk.Storage | |||||
| StreamIndex StreamIndex | |||||
| } | } | ||||
| func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strType StreamType) *FromShardstore { | |||||
| func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strIdx StreamIndex) *FromShardstore { | |||||
| return &FromShardstore{ | return &FromShardstore{ | ||||
| FileHash: fileHash, | |||||
| Hub: hub, | |||||
| Storage: storage, | |||||
| StreamType: strType, | |||||
| FileHash: fileHash, | |||||
| Hub: hub, | |||||
| Storage: storage, | |||||
| StreamIndex: strIdx, | |||||
| } | } | ||||
| } | } | ||||
| func (f *FromShardstore) GetStreamType() StreamType { | |||||
| return f.StreamType | |||||
| func (f *FromShardstore) GetStreamIndex() StreamIndex { | |||||
| return f.StreamIndex | |||||
| } | } | ||||
| type ToDriver struct { | type ToDriver struct { | ||||
| Handle *exec.DriverReadStream | |||||
| StreamType StreamType | |||||
| Range exec.Range | |||||
| Handle *exec.DriverReadStream | |||||
| StreamIndex StreamIndex | |||||
| Range exec.Range | |||||
| } | } | ||||
| func NewToDriver(strType StreamType) (*ToDriver, *exec.DriverReadStream) { | |||||
| func NewToDriver(strIdx StreamIndex) (*ToDriver, *exec.DriverReadStream) { | |||||
| str := exec.DriverReadStream{} | str := exec.DriverReadStream{} | ||||
| return &ToDriver{ | return &ToDriver{ | ||||
| Handle: &str, | |||||
| StreamType: strType, | |||||
| Handle: &str, | |||||
| StreamIndex: strIdx, | |||||
| }, &str | }, &str | ||||
| } | } | ||||
| func NewToDriverWithRange(strType StreamType, rng exec.Range) (*ToDriver, *exec.DriverReadStream) { | |||||
| func NewToDriverWithRange(strIdx StreamIndex, rng exec.Range) (*ToDriver, *exec.DriverReadStream) { | |||||
| str := exec.DriverReadStream{} | str := exec.DriverReadStream{} | ||||
| return &ToDriver{ | return &ToDriver{ | ||||
| Handle: &str, | |||||
| StreamType: strType, | |||||
| Range: rng, | |||||
| Handle: &str, | |||||
| StreamIndex: strIdx, | |||||
| Range: rng, | |||||
| }, &str | }, &str | ||||
| } | } | ||||
| func (t *ToDriver) GetStreamType() StreamType { | |||||
| return t.StreamType | |||||
| func (t *ToDriver) GetStreamIndex() StreamIndex { | |||||
| return t.StreamIndex | |||||
| } | } | ||||
| func (t *ToDriver) GetRange() exec.Range { | func (t *ToDriver) GetRange() exec.Range { | ||||
| @@ -157,32 +161,32 @@ func (t *ToDriver) GetRange() exec.Range { | |||||
| type ToShardStore struct { | type ToShardStore struct { | ||||
| Hub cdssdk.Hub | Hub cdssdk.Hub | ||||
| Storage cdssdk.Storage | Storage cdssdk.Storage | ||||
| StreamType StreamType | |||||
| StreamIndex StreamIndex | |||||
| Range exec.Range | Range exec.Range | ||||
| FileHashStoreKey string | FileHashStoreKey string | ||||
| } | } | ||||
| func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strType StreamType, fileHashStoreKey string) *ToShardStore { | |||||
| func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strIdx StreamIndex, fileHashStoreKey string) *ToShardStore { | |||||
| return &ToShardStore{ | return &ToShardStore{ | ||||
| Hub: hub, | Hub: hub, | ||||
| Storage: stg, | Storage: stg, | ||||
| StreamType: strType, | |||||
| StreamIndex: strIdx, | |||||
| FileHashStoreKey: fileHashStoreKey, | FileHashStoreKey: fileHashStoreKey, | ||||
| } | } | ||||
| } | } | ||||
| func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamType StreamType, fileHashStoreKey string, rng exec.Range) *ToShardStore { | |||||
| func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamIndex StreamIndex, fileHashStoreKey string, rng exec.Range) *ToShardStore { | |||||
| return &ToShardStore{ | return &ToShardStore{ | ||||
| Hub: hub, | Hub: hub, | ||||
| Storage: stg, | Storage: stg, | ||||
| StreamType: streamType, | |||||
| StreamIndex: streamIndex, | |||||
| FileHashStoreKey: fileHashStoreKey, | FileHashStoreKey: fileHashStoreKey, | ||||
| Range: rng, | Range: rng, | ||||
| } | } | ||||
| } | } | ||||
| func (t *ToShardStore) GetStreamType() StreamType { | |||||
| return t.StreamType | |||||
| func (t *ToShardStore) GetStreamIndex() StreamIndex { | |||||
| return t.StreamIndex | |||||
| } | } | ||||
| func (t *ToShardStore) GetRange() exec.Range { | func (t *ToShardStore) GetRange() exec.Range { | ||||
| @@ -207,9 +211,9 @@ func NewLoadToShared(hub cdssdk.Hub, storage cdssdk.Storage, userID cdssdk.UserI | |||||
| } | } | ||||
| } | } | ||||
| func (t *LoadToShared) GetStreamType() StreamType { | |||||
| return StreamType{ | |||||
| Type: StreamTypeRaw, | |||||
| func (t *LoadToShared) GetStreamIndex() StreamIndex { | |||||
| return StreamIndex{ | |||||
| Type: StreamIndexRaw, | |||||
| } | } | ||||
| } | } | ||||
| @@ -0,0 +1,81 @@ | |||||
| package ops2 | |||||
| import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | |||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||||
| ) | |||||
| // TODO 想办法直接使用ops里的Node,而不是重新实现一遍 | |||||
| type FromDriverNode struct { | |||||
| dag.NodeBase | |||||
| From ioswitch2.From | |||||
| Handle *exec.DriverWriteStream | |||||
| } | |||||
| func (b *GraphNodeBuilder) NewFromDriver(fr ioswitch2.From, handle *exec.DriverWriteStream) *FromDriverNode { | |||||
| node := &FromDriverNode{ | |||||
| From: fr, | |||||
| Handle: handle, | |||||
| } | |||||
| b.AddNode(node) | |||||
| node.OutputStreams().SetupNew(node, b.NewVar()) | |||||
| return node | |||||
| } | |||||
| func (f *FromDriverNode) GetFrom() ioswitch2.From { | |||||
| return f.From | |||||
| } | |||||
| func (t *FromDriverNode) Output() dag.Slot { | |||||
| return dag.Slot{ | |||||
| Var: t.OutputStreams().Get(0), | |||||
| Index: 0, | |||||
| } | |||||
| } | |||||
| func (t *FromDriverNode) GenerateOp() (exec.Op, error) { | |||||
| t.Handle.ID = t.OutputStreams().Get(0).VarID | |||||
| return nil, nil | |||||
| } | |||||
| type ToDriverNode struct { | |||||
| dag.NodeBase | |||||
| To ioswitch2.To | |||||
| Handle *exec.DriverReadStream | |||||
| Range exec.Range | |||||
| } | |||||
| func (b *GraphNodeBuilder) NewToDriver(to ioswitch2.To, handle *exec.DriverReadStream) *ToDriverNode { | |||||
| node := &ToDriverNode{ | |||||
| To: to, | |||||
| Handle: handle, | |||||
| } | |||||
| b.AddNode(node) | |||||
| return node | |||||
| } | |||||
| func (t *ToDriverNode) GetTo() ioswitch2.To { | |||||
| return t.To | |||||
| } | |||||
| func (t *ToDriverNode) SetInput(v *dag.Var) { | |||||
| t.InputStreams().EnsureSize(1) | |||||
| v.StreamTo(t, 0) | |||||
| } | |||||
| func (t *ToDriverNode) Input() dag.Slot { | |||||
| return dag.Slot{ | |||||
| Var: t.InputStreams().Get(0), | |||||
| Index: 0, | |||||
| } | |||||
| } | |||||
| func (t *ToDriverNode) GenerateOp() (exec.Op, error) { | |||||
| t.Handle.ID = t.InputStreams().Get(0).VarID | |||||
| return nil, nil | |||||
| } | |||||
| @@ -3,6 +3,7 @@ package ops2 | |||||
| import ( | import ( | ||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | ||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops" | "gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||||
| ) | ) | ||||
| type GraphNodeBuilder struct { | type GraphNodeBuilder struct { | ||||
| @@ -15,11 +16,13 @@ func NewGraphNodeBuilder() *GraphNodeBuilder { | |||||
| type FromNode interface { | type FromNode interface { | ||||
| dag.Node | dag.Node | ||||
| GetFrom() ioswitch2.From | |||||
| Output() dag.Slot | Output() dag.Slot | ||||
| } | } | ||||
| type ToNode interface { | type ToNode interface { | ||||
| dag.Node | dag.Node | ||||
| GetTo() ioswitch2.To | |||||
| Input() dag.Slot | Input() dag.Slot | ||||
| SetInput(input *dag.Var) | SetInput(input *dag.Var) | ||||
| } | } | ||||
| @@ -72,7 +72,11 @@ func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| func (o *Range) String() string { | func (o *Range) String() string { | ||||
| return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input, o.Output) | |||||
| len := "" | |||||
| if o.Length != nil { | |||||
| len = fmt.Sprintf("%v", *o.Length) | |||||
| } | |||||
| return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, len, o.Input, o.Output) | |||||
| } | } | ||||
| type RangeNode struct { | type RangeNode struct { | ||||
| @@ -1 +1,215 @@ | |||||
| package ops2 | package ops2 | ||||
| import ( | |||||
| "context" | |||||
| "fmt" | |||||
| "io" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/future" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/utils" | |||||
| "gitlink.org.cn/cloudream/common/utils/io2" | |||||
| ) | |||||
| func init() { | |||||
| exec.UseOp[*SegmentSplit]() | |||||
| exec.UseOp[*SegmentJoin]() | |||||
| } | |||||
| type SegmentSplit struct { | |||||
| Input exec.VarID | |||||
| Segments []int64 | |||||
| Outputs []exec.VarID | |||||
| } | |||||
| func (o *SegmentSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| defer input.Stream.Close() | |||||
| for i, outID := range o.Outputs { | |||||
| fut := future.NewSetVoid() | |||||
| segStr := io.LimitReader(input.Stream, o.Segments[i]) | |||||
| segStr2 := io2.DelegateReadCloser(segStr, func() error { | |||||
| fut.SetError(context.Canceled) | |||||
| return nil | |||||
| }) | |||||
| segStr2 = io2.AfterEOF(segStr2, func(str io.ReadCloser, err error) { | |||||
| fut.SetVoid() | |||||
| }) | |||||
| e.PutVar(outID, &exec.StreamValue{Stream: segStr2}) | |||||
| err = fut.Wait(ctx.Context) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func (o *SegmentSplit) String() string { | |||||
| return fmt.Sprintf("SegmentSplit(%v, %v) -> %v", o.Input, o.Segments, o.Outputs) | |||||
| } | |||||
| type SegmentJoin struct { | |||||
| Inputs []exec.VarID | |||||
| Output exec.VarID | |||||
| // 这些字段只在执行时使用 | |||||
| ctx *exec.ExecContext | |||||
| e *exec.Executor | |||||
| nextStreamIdx int | |||||
| nextStream io.ReadCloser | |||||
| fut *future.SetVoidFuture | |||||
| } | |||||
| func (o *SegmentJoin) Read(buf []byte) (int, error) { | |||||
| for { | |||||
| if o.nextStream == nil { | |||||
| if o.nextStreamIdx >= len(o.Inputs) { | |||||
| o.fut.SetVoid() | |||||
| return 0, io.EOF | |||||
| } | |||||
| input, err := exec.BindVar[*exec.StreamValue](o.e, o.ctx.Context, o.Inputs[o.nextStreamIdx]) | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| o.nextStream = input.Stream | |||||
| o.nextStreamIdx++ | |||||
| } | |||||
| n, err := o.nextStream.Read(buf) | |||||
| if err == io.EOF { | |||||
| o.nextStream.Close() | |||||
| o.nextStream = nil | |||||
| continue | |||||
| } | |||||
| return n, err | |||||
| } | |||||
| } | |||||
| func (o *SegmentJoin) Close() error { | |||||
| if o.nextStream != nil { | |||||
| o.nextStream.Close() | |||||
| o.nextStream = nil | |||||
| o.fut.SetVoid() | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func (o *SegmentJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| o.ctx = ctx | |||||
| o.e = e | |||||
| o.nextStreamIdx = 0 | |||||
| o.nextStream = nil | |||||
| o.fut = future.NewSetVoid() | |||||
| e.PutVar(o.Output, &exec.StreamValue{Stream: o}) | |||||
| return o.fut.Wait(ctx.Context) | |||||
| } | |||||
| func (o *SegmentJoin) String() string { | |||||
| return fmt.Sprintf("SegmentJoin %v -> %v", utils.FormatVarIDs(o.Inputs), o.Output) | |||||
| } | |||||
| type SegmentSplitNode struct { | |||||
| dag.NodeBase | |||||
| segments []int64 | |||||
| } | |||||
| func (b *GraphNodeBuilder) NewSegmentSplit(segments []int64) *SegmentSplitNode { | |||||
| node := &SegmentSplitNode{ | |||||
| segments: segments, | |||||
| } | |||||
| b.AddNode(node) | |||||
| node.OutputStreams().Resize(len(segments)) | |||||
| return node | |||||
| } | |||||
| func (n *SegmentSplitNode) SetInput(input *dag.Var) { | |||||
| n.InputStreams().EnsureSize(1) | |||||
| input.StreamTo(n, 0) | |||||
| } | |||||
| func (n *SegmentSplitNode) Segment(index int) *dag.Var { | |||||
| // 必须连续消耗流 | |||||
| for i := 0; i <= index; i++ { | |||||
| if n.OutputStreams().Get(i) == nil { | |||||
| n.OutputStreams().Setup(n, n.Graph().NewVar(), i) | |||||
| } | |||||
| } | |||||
| return n.OutputStreams().Get(index) | |||||
| } | |||||
| func (t *SegmentSplitNode) GenerateOp() (exec.Op, error) { | |||||
| lastUsedSeg := 0 | |||||
| for i := t.OutputStreams().Len() - 1; i >= 0; i-- { | |||||
| if t.OutputStreams().Get(i) != nil { | |||||
| lastUsedSeg = i | |||||
| break | |||||
| } | |||||
| } | |||||
| return &SegmentSplit{ | |||||
| Input: t.InputStreams().Get(0).VarID, | |||||
| Segments: t.segments[:lastUsedSeg+1], | |||||
| Outputs: t.OutputStreams().GetVarIDs(), | |||||
| }, nil | |||||
| } | |||||
| type SegmentJoinNode struct { | |||||
| dag.NodeBase | |||||
| UsedStart int | |||||
| UsedCount int | |||||
| } | |||||
| func (b *GraphNodeBuilder) NewSegmentJoin(segmentSizes []int64) *SegmentJoinNode { | |||||
| node := &SegmentJoinNode{} | |||||
| b.AddNode(node) | |||||
| node.InputStreams().Resize(len(segmentSizes)) | |||||
| node.OutputStreams().SetupNew(node, b.NewVar()) | |||||
| return node | |||||
| } | |||||
| func (n *SegmentJoinNode) SetInput(index int, input *dag.Var) { | |||||
| input.StreamTo(n, index) | |||||
| } | |||||
| // 记录本计划中实际要使用的分段的范围,范围外的分段流都会取消输入 | |||||
| func (n *SegmentJoinNode) MarkUsed(start, cnt int) { | |||||
| n.UsedStart = start | |||||
| n.UsedCount = cnt | |||||
| for i := 0; i < start; i++ { | |||||
| str := n.InputStreams().Get(i) | |||||
| if str != nil { | |||||
| str.StreamNotTo(n, i) | |||||
| } | |||||
| } | |||||
| for i := start + cnt; i < n.InputStreams().Len(); i++ { | |||||
| str := n.InputStreams().Get(i) | |||||
| if str != nil { | |||||
| str.StreamNotTo(n, i) | |||||
| } | |||||
| } | |||||
| } | |||||
| func (n *SegmentJoinNode) Joined() *dag.Var { | |||||
| return n.OutputStreams().Get(0) | |||||
| } | |||||
| func (t *SegmentJoinNode) GenerateOp() (exec.Op, error) { | |||||
| return &SegmentJoin{ | |||||
| Inputs: t.InputStreams().GetVarIDsRanged(t.UsedStart, t.UsedStart+t.UsedCount), | |||||
| Output: t.OutputStreams().Get(0).VarID, | |||||
| }, nil | |||||
| } | |||||
| @@ -10,6 +10,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/io2" | "gitlink.org.cn/cloudream/common/utils/io2" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | ||||
| ) | ) | ||||
| @@ -115,12 +116,14 @@ func (o *ShardWrite) String() string { | |||||
| type ShardReadNode struct { | type ShardReadNode struct { | ||||
| dag.NodeBase | dag.NodeBase | ||||
| From ioswitch2.From | |||||
| StorageID cdssdk.StorageID | StorageID cdssdk.StorageID | ||||
| Open types.OpenOption | Open types.OpenOption | ||||
| } | } | ||||
| func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode { | |||||
| func (b *GraphNodeBuilder) NewShardRead(fr ioswitch2.From, stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode { | |||||
| node := &ShardReadNode{ | node := &ShardReadNode{ | ||||
| From: fr, | |||||
| StorageID: stgID, | StorageID: stgID, | ||||
| Open: open, | Open: open, | ||||
| } | } | ||||
| @@ -129,6 +132,10 @@ func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenO | |||||
| return node | return node | ||||
| } | } | ||||
| func (t *ShardReadNode) GetFrom() ioswitch2.From { | |||||
| return t.From | |||||
| } | |||||
| func (t *ShardReadNode) Output() dag.Slot { | func (t *ShardReadNode) Output() dag.Slot { | ||||
| return dag.Slot{ | return dag.Slot{ | ||||
| Var: t.OutputStreams().Get(0), | Var: t.OutputStreams().Get(0), | ||||
| @@ -150,12 +157,14 @@ func (t *ShardReadNode) GenerateOp() (exec.Op, error) { | |||||
| type ShardWriteNode struct { | type ShardWriteNode struct { | ||||
| dag.NodeBase | dag.NodeBase | ||||
| To ioswitch2.To | |||||
| StorageID cdssdk.StorageID | StorageID cdssdk.StorageID | ||||
| FileHashStoreKey string | FileHashStoreKey string | ||||
| } | } | ||||
| func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode { | |||||
| func (b *GraphNodeBuilder) NewShardWrite(to ioswitch2.To, stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode { | |||||
| node := &ShardWriteNode{ | node := &ShardWriteNode{ | ||||
| To: to, | |||||
| StorageID: stgID, | StorageID: stgID, | ||||
| FileHashStoreKey: fileHashStoreKey, | FileHashStoreKey: fileHashStoreKey, | ||||
| } | } | ||||
| @@ -163,6 +172,10 @@ func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKe | |||||
| return node | return node | ||||
| } | } | ||||
| func (t *ShardWriteNode) GetTo() ioswitch2.To { | |||||
| return t.To | |||||
| } | |||||
| func (t *ShardWriteNode) SetInput(input *dag.Var) { | func (t *ShardWriteNode) SetInput(input *dag.Var) { | ||||
| t.InputStreams().EnsureSize(1) | t.InputStreams().EnsureSize(1) | ||||
| input.StreamTo(t, 0) | input.StreamTo(t, 0) | ||||
| @@ -7,6 +7,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" | ||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2" | |||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | "gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr" | ||||
| ) | ) | ||||
| @@ -64,14 +65,16 @@ func (o *SharedLoad) String() string { | |||||
| type SharedLoadNode struct { | type SharedLoadNode struct { | ||||
| dag.NodeBase | dag.NodeBase | ||||
| To ioswitch2.To | |||||
| StorageID cdssdk.StorageID | StorageID cdssdk.StorageID | ||||
| UserID cdssdk.UserID | UserID cdssdk.UserID | ||||
| PackageID cdssdk.PackageID | PackageID cdssdk.PackageID | ||||
| Path string | Path string | ||||
| } | } | ||||
| func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode { | |||||
| func (b *GraphNodeBuilder) NewSharedLoad(to ioswitch2.To, stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode { | |||||
| node := &SharedLoadNode{ | node := &SharedLoadNode{ | ||||
| To: to, | |||||
| StorageID: stgID, | StorageID: stgID, | ||||
| UserID: userID, | UserID: userID, | ||||
| PackageID: packageID, | PackageID: packageID, | ||||
| @@ -81,6 +84,10 @@ func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.U | |||||
| return node | return node | ||||
| } | } | ||||
| func (t *SharedLoadNode) GetTo() ioswitch2.To { | |||||
| return t.To | |||||
| } | |||||
| func (t *SharedLoadNode) SetInput(input *dag.Var) { | func (t *SharedLoadNode) SetInput(input *dag.Var) { | ||||
| t.InputStreams().EnsureSize(1) | t.InputStreams().EnsureSize(1) | ||||
| input.StreamTo(t, 0) | input.StreamTo(t, 0) | ||||
| @@ -15,9 +15,9 @@ import ( | |||||
| "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" | ||||
| ) | ) | ||||
| type TypedStream struct { | |||||
| Stream *dag.Var | |||||
| StreamType ioswitch2.StreamType | |||||
| type IndexedStream struct { | |||||
| Stream *dag.Var | |||||
| StreamIndex ioswitch2.StreamIndex | |||||
| } | } | ||||
| type ParseContext struct { | type ParseContext struct { | ||||
| @@ -25,33 +25,43 @@ type ParseContext struct { | |||||
| DAG *ops2.GraphNodeBuilder | DAG *ops2.GraphNodeBuilder | ||||
| // 为了产生所有To所需的数据范围,而需要From打开的范围。 | // 为了产生所有To所需的数据范围,而需要From打开的范围。 | ||||
| // 这个范围是基于整个文件的,且上下界都取整到条带大小的整数倍,因此上界是有可能超过文件大小的。 | // 这个范围是基于整个文件的,且上下界都取整到条带大小的整数倍,因此上界是有可能超过文件大小的。 | ||||
| ToNodes map[ioswitch2.To]ops2.ToNode | |||||
| TypedStreams []TypedStream | |||||
| StreamRange exec.Range | |||||
| EC cdssdk.ECRedundancy | |||||
| ToNodes map[ioswitch2.To]ops2.ToNode | |||||
| IndexedStreams []IndexedStream | |||||
| StreamRange exec.Range | |||||
| UseEC bool // 是否使用纠删码 | |||||
| UseSegment bool // 是否使用分段 | |||||
| } | } | ||||
| func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy) error { | |||||
| func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) error { | |||||
| ctx := ParseContext{ | ctx := ParseContext{ | ||||
| Ft: ft, | Ft: ft, | ||||
| DAG: ops2.NewGraphNodeBuilder(), | DAG: ops2.NewGraphNodeBuilder(), | ||||
| ToNodes: make(map[ioswitch2.To]ops2.ToNode), | ToNodes: make(map[ioswitch2.To]ops2.ToNode), | ||||
| EC: ec, | |||||
| } | } | ||||
| // 分成两个阶段: | // 分成两个阶段: | ||||
| // 1. 基于From和To生成更多指令,初步匹配to的需求 | // 1. 基于From和To生成更多指令,初步匹配to的需求 | ||||
| err := checkEncodingParams(&ctx) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| // 计算一下打开流的范围 | // 计算一下打开流的范围 | ||||
| calcStreamRange(&ctx) | calcStreamRange(&ctx) | ||||
| err := extend(&ctx) | |||||
| err = extend(&ctx) | |||||
| if err != nil { | if err != nil { | ||||
| return err | return err | ||||
| } | } | ||||
| // 2. 优化上一步生成的指令 | // 2. 优化上一步生成的指令 | ||||
| err = removeUnusedSegment(&ctx) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| // 对于删除指令的优化,需要反复进行,直到没有变化为止。 | // 对于删除指令的优化,需要反复进行,直到没有变化为止。 | ||||
| // 从目前实现上来说不会死循环 | // 从目前实现上来说不会死循环 | ||||
| for { | for { | ||||
| @@ -79,17 +89,18 @@ func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy) | |||||
| } | } | ||||
| // 下面这些只需要执行一次,但需要按顺序 | // 下面这些只需要执行一次,但需要按顺序 | ||||
| removeUnusedFromNode(&ctx) | |||||
| dropUnused(&ctx) | dropUnused(&ctx) | ||||
| storeIPFSWriteResult(&ctx) | storeIPFSWriteResult(&ctx) | ||||
| generateClone(&ctx) | |||||
| generateRange(&ctx) | generateRange(&ctx) | ||||
| generateClone(&ctx) | |||||
| return plan.Generate(ctx.DAG.Graph, blder) | return plan.Generate(ctx.DAG.Graph, blder) | ||||
| } | } | ||||
| func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.Var { | |||||
| func findOutputStream(ctx *ParseContext, streamIndex ioswitch2.StreamIndex) *dag.Var { | |||||
| var ret *dag.Var | var ret *dag.Var | ||||
| for _, s := range ctx.TypedStreams { | |||||
| if s.StreamType == streamType { | |||||
| for _, s := range ctx.IndexedStreams { | |||||
| if s.StreamIndex == streamIndex { | |||||
| ret = s.Stream | ret = s.Stream | ||||
| break | break | ||||
| } | } | ||||
| @@ -97,35 +108,91 @@ func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.V | |||||
| return ret | return ret | ||||
| } | } | ||||
| // 计算输入流的打开范围。会把流的范围按条带大小取整 | |||||
| func calcStreamRange(ctx *ParseContext) { | |||||
| stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K) | |||||
| // 检查使用不同编码时参数是否设置到位 | |||||
| func checkEncodingParams(ctx *ParseContext) error { | |||||
| for _, f := range ctx.Ft.Froms { | |||||
| if f.GetStreamIndex().IsEC() { | |||||
| ctx.UseEC = true | |||||
| if ctx.Ft.ECParam == nil { | |||||
| return fmt.Errorf("EC encoding parameters not set") | |||||
| } | |||||
| } | |||||
| rng := exec.Range{ | |||||
| Offset: math.MaxInt64, | |||||
| if f.GetStreamIndex().IsSegment() { | |||||
| ctx.UseSegment = true | |||||
| if ctx.Ft.SegmentParam == nil { | |||||
| return fmt.Errorf("segment parameters not set") | |||||
| } | |||||
| } | |||||
| } | } | ||||
| for _, t := range ctx.Ft.Toes { | |||||
| if t.GetStreamIndex().IsEC() { | |||||
| ctx.UseEC = true | |||||
| if ctx.Ft.ECParam == nil { | |||||
| return fmt.Errorf("EC encoding parameters not set") | |||||
| } | |||||
| } | |||||
| if t.GetStreamIndex().IsSegment() { | |||||
| ctx.UseSegment = true | |||||
| if ctx.Ft.SegmentParam == nil { | |||||
| return fmt.Errorf("segment parameters not set") | |||||
| } | |||||
| } | |||||
| } | |||||
| return nil | |||||
| } | |||||
| // 计算输入流的打开范围。如果From或者To中包含EC的流,则会将打开范围扩大到条带大小的整数倍。 | |||||
| func calcStreamRange(ctx *ParseContext) { | |||||
| rng := exec.NewRange(math.MaxInt64, 0) | |||||
| for _, to := range ctx.Ft.Toes { | for _, to := range ctx.Ft.Toes { | ||||
| if to.GetStreamType().IsRaw() { | |||||
| strIdx := to.GetStreamIndex() | |||||
| if strIdx.IsRaw() { | |||||
| toRng := to.GetRange() | toRng := to.GetRange() | ||||
| rng.ExtendStart(math2.Floor(toRng.Offset, stripSize)) | |||||
| rng.ExtendStart(toRng.Offset) | |||||
| if toRng.Length != nil { | if toRng.Length != nil { | ||||
| rng.ExtendEnd(math2.Ceil(toRng.Offset+*toRng.Length, stripSize)) | |||||
| rng.ExtendEnd(toRng.Offset + *toRng.Length) | |||||
| } else { | } else { | ||||
| rng.Length = nil | rng.Length = nil | ||||
| } | } | ||||
| } else { | |||||
| } else if strIdx.IsEC() { | |||||
| toRng := to.GetRange() | toRng := to.GetRange() | ||||
| blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.EC.ChunkSize)) | |||||
| stripSize := ctx.Ft.ECParam.StripSize() | |||||
| blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.Ft.ECParam.ChunkSize)) | |||||
| rng.ExtendStart(blkStartIndex * stripSize) | rng.ExtendStart(blkStartIndex * stripSize) | ||||
| if toRng.Length != nil { | if toRng.Length != nil { | ||||
| blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.EC.ChunkSize)) | |||||
| blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.Ft.ECParam.ChunkSize)) | |||||
| rng.ExtendEnd(blkEndIndex * stripSize) | rng.ExtendEnd(blkEndIndex * stripSize) | ||||
| } else { | } else { | ||||
| rng.Length = nil | rng.Length = nil | ||||
| } | } | ||||
| } else if strIdx.IsSegment() { | |||||
| // Segment节点的Range是相对于本段的,需要加上本段的起始位置 | |||||
| toRng := to.GetRange() | |||||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(strIdx.Index) | |||||
| offset := toRng.Offset + segStart | |||||
| rng.ExtendStart(offset) | |||||
| if toRng.Length != nil { | |||||
| rng.ExtendEnd(offset + *toRng.Length) | |||||
| } else { | |||||
| rng.Length = nil | |||||
| } | |||||
| } | |||||
| } | |||||
| if ctx.UseEC { | |||||
| stripSize := ctx.Ft.ECParam.StripSize() | |||||
| rng.ExtendStart(math2.Floor(rng.Offset, stripSize)) | |||||
| if rng.Length != nil { | |||||
| rng.ExtendEnd(math2.Ceil(rng.Offset+*rng.Length, stripSize)) | |||||
| } | } | ||||
| } | } | ||||
| @@ -139,57 +206,112 @@ func extend(ctx *ParseContext) error { | |||||
| return err | return err | ||||
| } | } | ||||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||||
| Stream: frNode.Output().Var, | |||||
| StreamType: fr.GetStreamType(), | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: frNode.Output().Var, | |||||
| StreamIndex: fr.GetStreamIndex(), | |||||
| }) | }) | ||||
| // 对于完整文件的From,生成Split指令 | // 对于完整文件的From,生成Split指令 | ||||
| if fr.GetStreamType().IsRaw() { | |||||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.EC.ChunkSize) | |||||
| splitNode.Split(frNode.Output().Var, ctx.EC.K) | |||||
| for i := 0; i < ctx.EC.K; i++ { | |||||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||||
| Stream: splitNode.SubStream(i), | |||||
| StreamType: ioswitch2.ECSrteam(i), | |||||
| }) | |||||
| if fr.GetStreamIndex().IsRaw() { | |||||
| // 只有输入输出需要EC编码的块时,才生成相关指令 | |||||
| if ctx.UseEC { | |||||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize) | |||||
| splitNode.Split(frNode.Output().Var, ctx.Ft.ECParam.K) | |||||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: splitNode.SubStream(i), | |||||
| StreamIndex: ioswitch2.ECSrteam(i), | |||||
| }) | |||||
| } | |||||
| } | |||||
| // 同上 | |||||
| if ctx.UseSegment { | |||||
| splitNode := ctx.DAG.NewSegmentSplit(ctx.Ft.SegmentParam.Segments) | |||||
| splitNode.SetInput(frNode.Output().Var) | |||||
| for i := 0; i < len(ctx.Ft.SegmentParam.Segments); i++ { | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: splitNode.Segment(i), | |||||
| StreamIndex: ioswitch2.SegmentStream(i), | |||||
| }) | |||||
| } | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令 | |||||
| ecInputStrs := make(map[int]*dag.Var) | |||||
| for _, s := range ctx.TypedStreams { | |||||
| if s.StreamType.IsEC() && ecInputStrs[s.StreamType.Index] == nil { | |||||
| ecInputStrs[s.StreamType.Index] = s.Stream | |||||
| if len(ecInputStrs) == ctx.EC.K { | |||||
| break | |||||
| if ctx.UseEC { | |||||
| // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令 | |||||
| ecInputStrs := make(map[int]*dag.Var) | |||||
| for _, s := range ctx.IndexedStreams { | |||||
| if s.StreamIndex.IsEC() && ecInputStrs[s.StreamIndex.Index] == nil { | |||||
| ecInputStrs[s.StreamIndex.Index] = s.Stream | |||||
| if len(ecInputStrs) == ctx.Ft.ECParam.K { | |||||
| break | |||||
| } | |||||
| } | } | ||||
| } | } | ||||
| } | |||||
| if len(ecInputStrs) == ctx.EC.K { | |||||
| mulNode := ctx.DAG.NewECMultiply(ctx.EC) | |||||
| if len(ecInputStrs) == ctx.Ft.ECParam.K { | |||||
| mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam) | |||||
| for i, s := range ecInputStrs { | |||||
| mulNode.AddInput(s, i) | |||||
| } | |||||
| for i := 0; i < ctx.EC.N; i++ { | |||||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||||
| Stream: mulNode.NewOutput(i), | |||||
| StreamType: ioswitch2.ECSrteam(i), | |||||
| for i, s := range ecInputStrs { | |||||
| mulNode.AddInput(s, i) | |||||
| } | |||||
| for i := 0; i < ctx.Ft.ECParam.N; i++ { | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: mulNode.NewOutput(i), | |||||
| StreamIndex: ioswitch2.ECSrteam(i), | |||||
| }) | |||||
| } | |||||
| joinNode := ctx.DAG.NewChunkedJoin(ctx.Ft.ECParam.ChunkSize) | |||||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||||
| // 不可能找不到流 | |||||
| joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i))) | |||||
| } | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: joinNode.Joined(), | |||||
| StreamIndex: ioswitch2.RawStream(), | |||||
| }) | }) | ||||
| } | } | ||||
| } | |||||
| joinNode := ctx.DAG.NewChunkedJoin(ctx.EC.ChunkSize) | |||||
| for i := 0; i < ctx.EC.K; i++ { | |||||
| // 不可能找不到流 | |||||
| joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i))) | |||||
| if ctx.UseSegment { | |||||
| // 先假设有所有的顺序分段,生成Join指令,后续根据Range再实际计算是否缺少流 | |||||
| joinNode := ctx.DAG.NewSegmentJoin(ctx.Ft.SegmentParam.Segments) | |||||
| for i := 0; i < ctx.Ft.SegmentParam.SegmentCount(); i++ { | |||||
| str := findOutputStream(ctx, ioswitch2.SegmentStream(i)) | |||||
| if str != nil { | |||||
| joinNode.SetInput(i, str) | |||||
| } | |||||
| } | } | ||||
| ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{ | |||||
| Stream: joinNode.Joined(), | |||||
| StreamType: ioswitch2.RawStream(), | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: joinNode.Joined(), | |||||
| StreamIndex: ioswitch2.RawStream(), | |||||
| }) | }) | ||||
| // SegmentJoin生成的Join指令可以用来生成EC块 | |||||
| if ctx.UseEC { | |||||
| splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize) | |||||
| splitNode.Split(joinNode.Joined(), ctx.Ft.ECParam.K) | |||||
| mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam) | |||||
| for i := 0; i < ctx.Ft.ECParam.K; i++ { | |||||
| mulNode.AddInput(splitNode.SubStream(i), i) | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: splitNode.SubStream(i), | |||||
| StreamIndex: ioswitch2.ECSrteam(i), | |||||
| }) | |||||
| } | |||||
| for i := 0; i < ctx.Ft.ECParam.N; i++ { | |||||
| ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{ | |||||
| Stream: mulNode.NewOutput(i), | |||||
| StreamIndex: ioswitch2.ECSrteam(i), | |||||
| }) | |||||
| } | |||||
| } | |||||
| } | } | ||||
| // 为每一个To找到一个输入流 | // 为每一个To找到一个输入流 | ||||
| @@ -200,9 +322,9 @@ func extend(ctx *ParseContext) error { | |||||
| } | } | ||||
| ctx.ToNodes[to] = toNode | ctx.ToNodes[to] = toNode | ||||
| str := findOutputStream(ctx, to.GetStreamType()) | |||||
| str := findOutputStream(ctx, to.GetStreamIndex()) | |||||
| if str == nil { | if str == nil { | ||||
| return fmt.Errorf("no output stream found for data index %d", to.GetStreamType()) | |||||
| return fmt.Errorf("no output stream found for data index %d", to.GetStreamIndex()) | |||||
| } | } | ||||
| toNode.SetInput(str) | toNode.SetInput(str) | ||||
| @@ -213,26 +335,48 @@ func extend(ctx *ParseContext) error { | |||||
| func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | ||||
| var repRange exec.Range | var repRange exec.Range | ||||
| var blkRange exec.Range | |||||
| repRange.Offset = ctx.StreamRange.Offset | repRange.Offset = ctx.StreamRange.Offset | ||||
| blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize) | |||||
| if ctx.StreamRange.Length != nil { | if ctx.StreamRange.Length != nil { | ||||
| repRngLen := *ctx.StreamRange.Length | repRngLen := *ctx.StreamRange.Length | ||||
| repRange.Length = &repRngLen | repRange.Length = &repRngLen | ||||
| } | |||||
| blkRngLen := *ctx.StreamRange.Length / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize) | |||||
| blkRange.Length = &blkRngLen | |||||
| var blkRange exec.Range | |||||
| if ctx.UseEC { | |||||
| blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize) | |||||
| if ctx.StreamRange.Length != nil { | |||||
| blkRngLen := *ctx.StreamRange.Length / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize) | |||||
| blkRange.Length = &blkRngLen | |||||
| } | |||||
| } | } | ||||
| switch f := f.(type) { | switch f := f.(type) { | ||||
| case *ioswitch2.FromShardstore: | case *ioswitch2.FromShardstore: | ||||
| t := ctx.DAG.NewShardRead(f.Storage.StorageID, types.NewOpen(f.FileHash)) | |||||
| t := ctx.DAG.NewShardRead(f, f.Storage.StorageID, types.NewOpen(f.FileHash)) | |||||
| if f.StreamType.IsRaw() { | |||||
| if f.StreamIndex.IsRaw() { | |||||
| t.Open.WithNullableLength(repRange.Offset, repRange.Length) | t.Open.WithNullableLength(repRange.Offset, repRange.Length) | ||||
| } else { | |||||
| } else if f.StreamIndex.IsEC() { | |||||
| t.Open.WithNullableLength(blkRange.Offset, blkRange.Length) | t.Open.WithNullableLength(blkRange.Offset, blkRange.Length) | ||||
| } else if f.StreamIndex.IsSegment() { | |||||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index) | |||||
| segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index] | |||||
| segEnd := segStart + segLen | |||||
| // 打开的范围不超过本段的范围 | |||||
| openOff := ctx.StreamRange.Offset - segStart | |||||
| openOff = math2.Clamp(openOff, 0, segLen) | |||||
| openLen := segLen | |||||
| if ctx.StreamRange.Length != nil { | |||||
| strEnd := ctx.StreamRange.Offset + *ctx.StreamRange.Length | |||||
| openEnd := math2.Min(strEnd, segEnd) | |||||
| openLen = openEnd - segStart - openOff | |||||
| } | |||||
| t.Open.WithNullableLength(openOff, &openLen) | |||||
| } | } | ||||
| switch addr := f.Hub.Address.(type) { | switch addr := f.Hub.Address.(type) { | ||||
| @@ -251,16 +395,36 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | |||||
| return t, nil | return t, nil | ||||
| case *ioswitch2.FromDriver: | case *ioswitch2.FromDriver: | ||||
| n := ctx.DAG.NewFromDriver(f.Handle) | |||||
| n := ctx.DAG.NewFromDriver(f, f.Handle) | |||||
| n.Env().ToEnvDriver() | n.Env().ToEnvDriver() | ||||
| n.Env().Pinned = true | n.Env().Pinned = true | ||||
| if f.StreamType.IsRaw() { | |||||
| if f.StreamIndex.IsRaw() { | |||||
| f.Handle.RangeHint.Offset = repRange.Offset | f.Handle.RangeHint.Offset = repRange.Offset | ||||
| f.Handle.RangeHint.Length = repRange.Length | f.Handle.RangeHint.Length = repRange.Length | ||||
| } else { | |||||
| } else if f.StreamIndex.IsEC() { | |||||
| f.Handle.RangeHint.Offset = blkRange.Offset | f.Handle.RangeHint.Offset = blkRange.Offset | ||||
| f.Handle.RangeHint.Length = blkRange.Length | f.Handle.RangeHint.Length = blkRange.Length | ||||
| } else if f.StreamIndex.IsSegment() { | |||||
| segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index) | |||||
| segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index] | |||||
| segEnd := segStart + segLen | |||||
| // 打开的范围不超过本段的范围 | |||||
| openOff := repRange.Offset - segStart | |||||
| openOff = math2.Clamp(openOff, 0, segLen) | |||||
| openLen := segLen | |||||
| if repRange.Length != nil { | |||||
| repEnd := repRange.Offset + *repRange.Length | |||||
| openEnd := math2.Min(repEnd, segEnd) | |||||
| openLen = openEnd - openOff | |||||
| } | |||||
| f.Handle.RangeHint.Offset = openOff | |||||
| f.Handle.RangeHint.Length = &openLen | |||||
| } | } | ||||
| return n, nil | return n, nil | ||||
| @@ -273,7 +437,7 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) { | |||||
| func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) { | func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) { | ||||
| switch t := t.(type) { | switch t := t.(type) { | ||||
| case *ioswitch2.ToShardStore: | case *ioswitch2.ToShardStore: | ||||
| n := ctx.DAG.NewShardWrite(t.Storage.StorageID, t.FileHashStoreKey) | |||||
| n := ctx.DAG.NewShardWrite(t, t.Storage.StorageID, t.FileHashStoreKey) | |||||
| if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | ||||
| return nil, err | return nil, err | ||||
| @@ -284,14 +448,14 @@ func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) { | |||||
| return n, nil | return n, nil | ||||
| case *ioswitch2.ToDriver: | case *ioswitch2.ToDriver: | ||||
| n := ctx.DAG.NewToDriver(t.Handle) | |||||
| n := ctx.DAG.NewToDriver(t, t.Handle) | |||||
| n.Env().ToEnvDriver() | n.Env().ToEnvDriver() | ||||
| n.Env().Pinned = true | n.Env().Pinned = true | ||||
| return n, nil | return n, nil | ||||
| case *ioswitch2.LoadToShared: | case *ioswitch2.LoadToShared: | ||||
| n := ctx.DAG.NewSharedLoad(t.Storage.StorageID, t.UserID, t.PackageID, t.Path) | |||||
| n := ctx.DAG.NewSharedLoad(t, t.Storage.StorageID, t.UserID, t.PackageID, t.Path) | |||||
| if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil { | ||||
| return nil, err | return nil, err | ||||
| @@ -321,6 +485,34 @@ func setEnvByAddress(n dag.Node, hub cdssdk.Hub, addr cdssdk.HubAddressInfo) err | |||||
| return nil | return nil | ||||
| } | } | ||||
| // 从SegmentJoin中删除未使用的分段 | |||||
| func removeUnusedSegment(ctx *ParseContext) error { | |||||
| var err error | |||||
| dag.WalkOnlyType[*ops2.SegmentJoinNode](ctx.DAG.Graph, func(node *ops2.SegmentJoinNode) bool { | |||||
| start := ctx.StreamRange.Offset | |||||
| var end *int64 | |||||
| if ctx.StreamRange.Length != nil { | |||||
| e := ctx.StreamRange.Offset + *ctx.StreamRange.Length | |||||
| end = &e | |||||
| } | |||||
| segStart, segEnd := ctx.Ft.SegmentParam.CalcSegmentRange(start, end) | |||||
| node.MarkUsed(segStart, segEnd) | |||||
| for i := segStart; i < segEnd; i++ { | |||||
| if node.InputStreams().Get(i) == nil { | |||||
| err = fmt.Errorf("segment %v missed to join an raw stream", i) | |||||
| return false | |||||
| } | |||||
| } | |||||
| return true | |||||
| }) | |||||
| return err | |||||
| } | |||||
| // 删除输出流未被使用的Join指令 | // 删除输出流未被使用的Join指令 | ||||
| func removeUnusedJoin(ctx *ParseContext) bool { | func removeUnusedJoin(ctx *ParseContext) bool { | ||||
| changed := false | changed := false | ||||
| @@ -352,6 +544,8 @@ func removeUnusedMultiplyOutput(ctx *ParseContext) bool { | |||||
| node.OutputIndexes[i2] = -2 | node.OutputIndexes[i2] = -2 | ||||
| changed = true | changed = true | ||||
| } | } | ||||
| // TODO2 没有修改SlotIndex | |||||
| node.OutputStreams().SetRawArray(lo2.RemoveAllDefault(outArr)) | node.OutputStreams().SetRawArray(lo2.RemoveAllDefault(outArr)) | ||||
| node.OutputIndexes = lo2.RemoveAll(node.OutputIndexes, -2) | node.OutputIndexes = lo2.RemoveAll(node.OutputIndexes, -2) | ||||
| @@ -504,6 +698,20 @@ func pin(ctx *ParseContext) bool { | |||||
| return changed | return changed | ||||
| } | } | ||||
| // 删除未使用的From流,不会删除FromDriver | |||||
| func removeUnusedFromNode(ctx *ParseContext) { | |||||
| dag.WalkOnlyType[ops2.FromNode](ctx.DAG.Graph, func(node ops2.FromNode) bool { | |||||
| if _, ok := node.(*ops2.FromDriverNode); ok { | |||||
| return true | |||||
| } | |||||
| if node.Output().Var == nil { | |||||
| ctx.DAG.RemoveNode(node) | |||||
| } | |||||
| return true | |||||
| }) | |||||
| } | |||||
| // 对于所有未使用的流,增加Drop指令 | // 对于所有未使用的流,增加Drop指令 | ||||
| func dropUnused(ctx *ParseContext) { | func dropUnused(ctx *ParseContext) { | ||||
| ctx.DAG.Walk(func(node dag.Node) bool { | ctx.DAG.Walk(func(node dag.Node) bool { | ||||
| @@ -539,10 +747,10 @@ func generateRange(ctx *ParseContext) { | |||||
| to := ctx.Ft.Toes[i] | to := ctx.Ft.Toes[i] | ||||
| toNode := ctx.ToNodes[to] | toNode := ctx.ToNodes[to] | ||||
| toStrType := to.GetStreamType() | |||||
| toStrIdx := to.GetStreamIndex() | |||||
| toRng := to.GetRange() | toRng := to.GetRange() | ||||
| if toStrType.IsRaw() { | |||||
| if toStrIdx.IsRaw() { | |||||
| n := ctx.DAG.NewRange() | n := ctx.DAG.NewRange() | ||||
| toInput := toNode.Input() | toInput := toNode.Input() | ||||
| *n.Env() = *toInput.Var.From().Node.Env() | *n.Env() = *toInput.Var.From().Node.Env() | ||||
| @@ -553,11 +761,11 @@ func generateRange(ctx *ParseContext) { | |||||
| toInput.Var.StreamNotTo(toNode, toInput.Index) | toInput.Var.StreamNotTo(toNode, toInput.Index) | ||||
| toNode.SetInput(rnged) | toNode.SetInput(rnged) | ||||
| } else { | |||||
| stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K) | |||||
| } else if toStrIdx.IsEC() { | |||||
| stripSize := int64(ctx.Ft.ECParam.ChunkSize * ctx.Ft.ECParam.K) | |||||
| blkStartIdx := ctx.StreamRange.Offset / stripSize | blkStartIdx := ctx.StreamRange.Offset / stripSize | ||||
| blkStart := blkStartIdx * int64(ctx.EC.ChunkSize) | |||||
| blkStart := blkStartIdx * int64(ctx.Ft.ECParam.ChunkSize) | |||||
| n := ctx.DAG.NewRange() | n := ctx.DAG.NewRange() | ||||
| toInput := toNode.Input() | toInput := toNode.Input() | ||||
| @@ -568,6 +776,26 @@ func generateRange(ctx *ParseContext) { | |||||
| }) | }) | ||||
| toInput.Var.StreamNotTo(toNode, toInput.Index) | toInput.Var.StreamNotTo(toNode, toInput.Index) | ||||
| toNode.SetInput(rnged) | toNode.SetInput(rnged) | ||||
| } else if toStrIdx.IsSegment() { | |||||
| // if frNode, ok := toNode.Input().Var.From().Node.(ops2.FromNode); ok { | |||||
| // // 目前只有To也是分段时,才可能对接一个提供分段的From,此时不需要再生成Range指令 | |||||
| // if frNode.GetFrom().GetStreamIndex().IsSegment() { | |||||
| // continue | |||||
| // } | |||||
| // } | |||||
| // segStart := ctx.Ft.SegmentParam.CalcSegmentStart(toStrIdx.Index) | |||||
| // strStart := segStart + toRng.Offset | |||||
| // n := ctx.DAG.NewRange() | |||||
| // toInput := toNode.Input() | |||||
| // *n.Env() = *toInput.Var.From().Node.Env() | |||||
| // rnged := n.RangeStream(toInput.Var, exec.Range{ | |||||
| // Offset: strStart - ctx.StreamRange.Offset, | |||||
| // Length: toRng.Length, | |||||
| // }) | |||||
| // toInput.Var.StreamNotTo(toNode, toInput.Index) | |||||
| // toNode.SetInput(rnged) | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| @@ -39,7 +39,7 @@ type CreateLoadResult struct { | |||||
| func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) error { | func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) error { | ||||
| uploadTime := time.Now() | uploadTime := time.Now() | ||||
| ft := ioswitch2.NewFromTo() | |||||
| ft := ioswitch2.FromTo{} | |||||
| fromExec, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream()) | fromExec, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream()) | ||||
| ft.AddFrom(fromExec) | ft.AddFrom(fromExec) | ||||
| for _, stg := range u.targetStgs { | for _, stg := range u.targetStgs { | ||||
| @@ -48,7 +48,7 @@ func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) e | |||||
| } | } | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("parsing plan: %w", err) | return fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -47,7 +47,7 @@ func (w *UpdateUploader) Upload(path string, size int64, stream io.Reader) error | |||||
| ft.AddFrom(fromExec).AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg.Storage, ioswitch2.RawStream(), "fileHash")) | ft.AddFrom(fromExec).AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg.Storage, ioswitch2.RawStream(), "fileHash")) | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("parsing plan: %w", err) | return fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -142,43 +142,47 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) { | |||||
| switch newRed := newRed.(type) { | switch newRed := newRed.(type) { | ||||
| case *cdssdk.RepRedundancy: | case *cdssdk.RepRedundancy: | ||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep") | log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep") | ||||
| updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs) | |||||
| updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs, userAllStorages) | |||||
| case *cdssdk.ECRedundancy: | case *cdssdk.ECRedundancy: | ||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec") | log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec") | ||||
| updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs) | |||||
| updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs, userAllStorages) | |||||
| case *cdssdk.LRCRedundancy: | case *cdssdk.LRCRedundancy: | ||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc") | log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc") | ||||
| updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages) | |||||
| updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages, userAllStorages) | |||||
| case *cdssdk.SegmentRedundancy: | |||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment") | |||||
| updating, err = t.noneToSeg(execCtx, obj, newRed, selectedStorages, userAllStorages) | |||||
| } | } | ||||
| case *cdssdk.RepRedundancy: | case *cdssdk.RepRedundancy: | ||||
| switch newRed := newRed.(type) { | switch newRed := newRed.(type) { | ||||
| case *cdssdk.RepRedundancy: | case *cdssdk.RepRedundancy: | ||||
| updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs) | |||||
| updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs, userAllStorages) | |||||
| case *cdssdk.ECRedundancy: | case *cdssdk.ECRedundancy: | ||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec") | log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec") | ||||
| updating, err = t.repToEC(execCtx, obj, newRed, newECStgs) | |||||
| updating, err = t.repToEC(execCtx, obj, newRed, newECStgs, userAllStorages) | |||||
| } | } | ||||
| case *cdssdk.ECRedundancy: | case *cdssdk.ECRedundancy: | ||||
| switch newRed := newRed.(type) { | switch newRed := newRed.(type) { | ||||
| case *cdssdk.RepRedundancy: | case *cdssdk.RepRedundancy: | ||||
| log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep") | log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep") | ||||
| updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs) | |||||
| updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs, userAllStorages) | |||||
| case *cdssdk.ECRedundancy: | case *cdssdk.ECRedundancy: | ||||
| uploadStorages := t.rechooseStoragesForEC(obj, srcRed, userAllStorages) | uploadStorages := t.rechooseStoragesForEC(obj, srcRed, userAllStorages) | ||||
| updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages) | |||||
| updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages) | |||||
| } | } | ||||
| case *cdssdk.LRCRedundancy: | case *cdssdk.LRCRedundancy: | ||||
| switch newRed := newRed.(type) { | switch newRed := newRed.(type) { | ||||
| case *cdssdk.LRCRedundancy: | case *cdssdk.LRCRedundancy: | ||||
| uploadStorages := t.rechooseStoragesForLRC(obj, srcRed, userAllStorages) | uploadStorages := t.rechooseStoragesForLRC(obj, srcRed, userAllStorages) | ||||
| updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages) | |||||
| updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages) | |||||
| } | } | ||||
| } | } | ||||
| @@ -205,11 +209,23 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) { | |||||
| func (t *CheckPackageRedundancy) chooseRedundancy(obj stgmod.ObjectDetail, userAllStgs map[cdssdk.StorageID]*StorageLoadInfo) (cdssdk.Redundancy, []*StorageLoadInfo) { | func (t *CheckPackageRedundancy) chooseRedundancy(obj stgmod.ObjectDetail, userAllStgs map[cdssdk.StorageID]*StorageLoadInfo) (cdssdk.Redundancy, []*StorageLoadInfo) { | ||||
| switch obj.Object.Redundancy.(type) { | switch obj.Object.Redundancy.(type) { | ||||
| case *cdssdk.NoneRedundancy: | case *cdssdk.NoneRedundancy: | ||||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||||
| return &cdssdk.DefaultECRedundancy, newStgs | |||||
| if obj.Object.Size > config.Cfg().ECFileSizeThreshold { | |||||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||||
| return &cdssdk.DefaultECRedundancy, newStgs | |||||
| } | |||||
| // newLRCStorages := t.chooseNewStoragesForLRC(&cdssdk.DefaultLRCRedundancy, userAllStorages) | |||||
| // return &cdssdk.DefaultLRCRedundancy, newLRCStorages | |||||
| return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs) | |||||
| case *cdssdk.RepRedundancy: | |||||
| if obj.Object.Size > config.Cfg().ECFileSizeThreshold { | |||||
| newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs) | |||||
| return &cdssdk.DefaultECRedundancy, newStgs | |||||
| } | |||||
| case *cdssdk.ECRedundancy: | |||||
| if obj.Object.Size <= config.Cfg().ECFileSizeThreshold { | |||||
| return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs) | |||||
| } | |||||
| case *cdssdk.LRCRedundancy: | case *cdssdk.LRCRedundancy: | ||||
| newLRCStgs := t.rechooseStoragesForLRC(obj, &cdssdk.DefaultLRCRedundancy, userAllStgs) | newLRCStgs := t.rechooseStoragesForLRC(obj, &cdssdk.DefaultLRCRedundancy, userAllStgs) | ||||
| @@ -278,6 +294,14 @@ func (t *CheckPackageRedundancy) chooseNewStoragesForLRC(red *cdssdk.LRCRedundan | |||||
| return t.chooseSoManyStorages(red.N, sortedStorages) | return t.chooseSoManyStorages(red.N, sortedStorages) | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) chooseNewStoragesForSeg(segCount int, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo { | |||||
| sortedStorages := sort2.Sort(lo.Values(allStgs), func(left *StorageLoadInfo, right *StorageLoadInfo) int { | |||||
| return sort2.Cmp(right.AccessAmount, left.AccessAmount) | |||||
| }) | |||||
| return t.chooseSoManyStorages(segCount, sortedStorages) | |||||
| } | |||||
| func (t *CheckPackageRedundancy) rechooseStoragesForRep(mostBlockStgIDs []cdssdk.StorageID, red *cdssdk.RepRedundancy, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo { | func (t *CheckPackageRedundancy) rechooseStoragesForRep(mostBlockStgIDs []cdssdk.StorageID, red *cdssdk.RepRedundancy, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo { | ||||
| type rechooseStorage struct { | type rechooseStorage struct { | ||||
| *StorageLoadInfo | *StorageLoadInfo | ||||
| @@ -421,25 +445,16 @@ func (t *CheckPackageRedundancy) chooseSoManyStorages(count int, stgs []*Storage | |||||
| return chosen | return chosen | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| if len(obj.Blocks) == 0 { | if len(obj.Blocks) == 0 { | ||||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | ||||
| } | } | ||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||||
| } | |||||
| if getStgs.Storages[0] == nil { | |||||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| if getStgs.Storages[0].MasterHub == nil { | |||||
| if srcStg.Storage.MasterHub == nil { | |||||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| @@ -447,13 +462,13 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object | |||||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||||
| for i, stg := range uploadStgs { | for i, stg := range uploadStgs { | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | ||||
| } | } | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -483,35 +498,27 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| if len(obj.Blocks) == 0 { | if len(obj.Blocks) == 0 { | ||||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | ||||
| } | } | ||||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||||
| } | |||||
| if getStgs.Storages[0] == nil { | |||||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| if getStgs.Storages[0].MasterHub == nil { | |||||
| if srcStg.Storage.MasterHub == nil { | |||||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||||
| ft.ECParam = red | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||||
| for i := 0; i < red.N; i++ { | for i := 0; i < red.N; i++ { | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | ||||
| } | } | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = parser.Parse(ft, plans, *red) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -540,25 +547,16 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| if len(obj.Blocks) == 0 { | if len(obj.Blocks) == 0 { | ||||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") | ||||
| } | } | ||||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||||
| } | |||||
| if getStgs.Storages[0] == nil { | |||||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| if getStgs.Storages[0].MasterHub == nil { | |||||
| if srcStg.Storage.MasterHub == nil { | |||||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| @@ -568,7 +566,7 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object | |||||
| } | } | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, -1), toes, plans) | |||||
| err := lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, -1), toes, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -597,25 +595,70 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.SegmentRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| if len(obj.Blocks) == 0 { | if len(obj.Blocks) == 0 { | ||||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | ||||
| } | } | ||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | |||||
| } | |||||
| if srcStg.Storage.MasterHub == nil { | |||||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | |||||
| } | |||||
| // 如果选择的备份节点都是同一个,那么就只要上传一次 | |||||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | |||||
| ft := ioswitch2.NewFromTo() | |||||
| ft.SegmentParam = red | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||||
| for i, stg := range uploadStgs { | |||||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.SegmentStream(i), fmt.Sprintf("%d", i))) | |||||
| } | |||||
| plans := exec.NewPlanBuilder() | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| return nil, fmt.Errorf("parsing plan: %w", err) | |||||
| } | } | ||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID})) | |||||
| // TODO 添加依赖 | |||||
| execCtx := exec.NewExecContext() | |||||
| exec.SetValueByType(execCtx, ctx.Args.StgMgr) | |||||
| ret, err := plans.Execute(execCtx).Wait(context.Background()) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("requesting to get storages: %w", err) | |||||
| return nil, fmt.Errorf("executing io plan: %w", err) | |||||
| } | |||||
| var blocks []stgmod.ObjectBlock | |||||
| for i, stg := range uploadStgs { | |||||
| blocks = append(blocks, stgmod.ObjectBlock{ | |||||
| ObjectID: obj.Object.ObjectID, | |||||
| Index: i, | |||||
| StorageID: stg.Storage.Storage.StorageID, | |||||
| FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, | |||||
| }) | |||||
| } | |||||
| return &coormq.UpdatingObjectRedundancy{ | |||||
| ObjectID: obj.Object.ObjectID, | |||||
| Redundancy: red, | |||||
| Blocks: blocks, | |||||
| }, nil | |||||
| } | |||||
| func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| if len(obj.Blocks) == 0 { | |||||
| return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep") | |||||
| } | } | ||||
| if getStgs.Storages[0] == nil { | |||||
| srcStg, ok := allStgs[obj.Blocks[0].StorageID] | |||||
| if !ok { | |||||
| return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| if getStgs.Storages[0].MasterHub == nil { | |||||
| if srcStg.Storage.MasterHub == nil { | |||||
| return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID) | ||||
| } | } | ||||
| @@ -623,13 +666,13 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD | |||||
| uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream())) | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream())) | |||||
| for i, stg := range uploadStgs { | for i, stg := range uploadStgs { | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i))) | ||||
| } | } | ||||
| plans := exec.NewPlanBuilder() | plans := exec.NewPlanBuilder() | ||||
| err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy) | |||||
| err := parser.Parse(ft, plans) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -659,23 +702,28 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| return t.noneToEC(ctx, obj, red, uploadStorages) | |||||
| func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| return t.noneToEC(ctx, obj, red, uploadStorages, allStgs) | |||||
| } | } | ||||
| func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| var chosenBlocks []stgmod.GrouppedObjectBlock | var chosenBlocks []stgmod.GrouppedObjectBlock | ||||
| var chosenBlockIndexes []int | var chosenBlockIndexes []int | ||||
| var chosenBlockStg []stgmod.StorageDetail | |||||
| for _, block := range obj.GroupBlocks() { | for _, block := range obj.GroupBlocks() { | ||||
| if len(block.StorageIDs) > 0 { | if len(block.StorageIDs) > 0 { | ||||
| // TODO 考虑选择最优的节点 | |||||
| stg, ok := allStgs[block.StorageIDs[0]] | |||||
| if !ok { | |||||
| continue | |||||
| } | |||||
| if stg.Storage.MasterHub == nil { | |||||
| continue | |||||
| } | |||||
| chosenBlocks = append(chosenBlocks, block) | chosenBlocks = append(chosenBlocks, block) | ||||
| chosenBlockIndexes = append(chosenBlockIndexes, block.Index) | chosenBlockIndexes = append(chosenBlockIndexes, block.Index) | ||||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||||
| } | } | ||||
| if len(chosenBlocks) == srcRed.K { | if len(chosenBlocks) == srcRed.K { | ||||
| @@ -694,9 +742,10 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||||
| planBlder := exec.NewPlanBuilder() | planBlder := exec.NewPlanBuilder() | ||||
| for i := range uploadStgs { | for i := range uploadStgs { | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.ECParam = srcRed | |||||
| for _, block := range chosenBlocks { | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(block.Index))) | |||||
| for i2, block := range chosenBlocks { | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index))) | |||||
| } | } | ||||
| len := obj.Object.Size | len := obj.Object.Size | ||||
| @@ -705,7 +754,7 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||||
| Length: &len, | Length: &len, | ||||
| })) | })) | ||||
| err := parser.Parse(ft, planBlder, *srcRed) | |||||
| err := parser.Parse(ft, planBlder) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -736,19 +785,23 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| grpBlocks := obj.GroupBlocks() | grpBlocks := obj.GroupBlocks() | ||||
| var chosenBlocks []stgmod.GrouppedObjectBlock | var chosenBlocks []stgmod.GrouppedObjectBlock | ||||
| var chosenBlockStg []stgmod.StorageDetail | |||||
| for _, block := range grpBlocks { | for _, block := range grpBlocks { | ||||
| if len(block.StorageIDs) > 0 { | if len(block.StorageIDs) > 0 { | ||||
| stg, ok := allStgs[block.StorageIDs[0]] | |||||
| if !ok { | |||||
| continue | |||||
| } | |||||
| if stg.Storage.MasterHub == nil { | |||||
| continue | |||||
| } | |||||
| chosenBlocks = append(chosenBlocks, block) | chosenBlocks = append(chosenBlocks, block) | ||||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||||
| } | } | ||||
| if len(chosenBlocks) == srcRed.K { | if len(chosenBlocks) == srcRed.K { | ||||
| @@ -786,15 +839,15 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet | |||||
| // 否则就要重建出这个节点需要的块 | // 否则就要重建出这个节点需要的块 | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| for _, block := range chosenBlocks { | |||||
| stg := stg.Storage | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(block.Index))) | |||||
| ft.ECParam = srcRed | |||||
| for i2, block := range chosenBlocks { | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index))) | |||||
| } | } | ||||
| // 输出只需要自己要保存的那一块 | // 输出只需要自己要保存的那一块 | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i))) | ||||
| err := parser.Parse(ft, planBlder, *srcRed) | |||||
| err := parser.Parse(ft, planBlder) | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("parsing plan: %w", err) | return nil, fmt.Errorf("parsing plan: %w", err) | ||||
| } | } | ||||
| @@ -830,12 +883,7 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| return nil, fmt.Errorf("new coordinator client: %w", err) | |||||
| } | |||||
| defer stgglb.CoordinatorMQPool.Release(coorCli) | |||||
| func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| blocksGrpByIndex := obj.GroupBlocks() | blocksGrpByIndex := obj.GroupBlocks() | ||||
| @@ -870,7 +918,7 @@ func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectD | |||||
| // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadStorages) | // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadStorages) | ||||
| } | } | ||||
| return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages) | |||||
| return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages, allStgs) | |||||
| } | } | ||||
| /* | /* | ||||
| @@ -939,11 +987,22 @@ TODO2 修复这一块的代码 | |||||
| }, nil | }, nil | ||||
| } | } | ||||
| */ | */ | ||||
| func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { | |||||
| var chosenBlocks []stgmod.GrouppedObjectBlock | var chosenBlocks []stgmod.GrouppedObjectBlock | ||||
| var chosenBlockStg []stgmod.StorageDetail | |||||
| for _, block := range grpBlocks { | for _, block := range grpBlocks { | ||||
| if len(block.StorageIDs) > 0 && block.Index < red.M() { | if len(block.StorageIDs) > 0 && block.Index < red.M() { | ||||
| stg, ok := allStgs[block.StorageIDs[0]] | |||||
| if !ok { | |||||
| continue | |||||
| } | |||||
| if stg.Storage.MasterHub == nil { | |||||
| continue | |||||
| } | |||||
| chosenBlocks = append(chosenBlocks, block) | chosenBlocks = append(chosenBlocks, block) | ||||
| chosenBlockStg = append(chosenBlockStg, stg.Storage) | |||||
| } | } | ||||
| if len(chosenBlocks) == red.K { | if len(chosenBlocks) == red.K { | ||||
| @@ -982,10 +1041,8 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O | |||||
| // 否则就要重建出这个节点需要的块 | // 否则就要重建出这个节点需要的块 | ||||
| for _, block := range chosenBlocks { | |||||
| fmt.Printf("b: %v\n", block.Index) | |||||
| stg := storage.Storage | |||||
| froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *stg.MasterHub, stg.Storage, block.Index)) | |||||
| for i2, block := range chosenBlocks { | |||||
| froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, block.Index)) | |||||
| } | } | ||||
| // 输出只需要自己要保存的那一块 | // 输出只需要自己要保存的那一块 | ||||
| @@ -745,7 +745,7 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st | |||||
| toStg := allStgInfos[solu.blockList[i].StorageID] | toStg := allStgInfos[solu.blockList[i].StorageID] | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, toStg.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID))) | ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, toStg.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID))) | ||||
| err := parser.Parse(ft, planBld, cdssdk.DefaultECRedundancy) | |||||
| err := parser.Parse(ft, planBld) | |||||
| if err != nil { | if err != nil { | ||||
| // TODO 错误处理 | // TODO 错误处理 | ||||
| continue | continue | ||||
| @@ -798,13 +798,14 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg | |||||
| for id, idxs := range reconstrct { | for id, idxs := range reconstrct { | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| ft.ECParam = ecRed | |||||
| ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.RawStream())) | ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.RawStream())) | ||||
| for _, i := range *idxs { | for _, i := range *idxs { | ||||
| ft.AddTo(ioswitch2.NewToShardStore(*allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d.%d", obj.Object.ObjectID, i))) | ft.AddTo(ioswitch2.NewToShardStore(*allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d.%d", obj.Object.ObjectID, i))) | ||||
| } | } | ||||
| err := parser.Parse(ft, planBld, *ecRed) | |||||
| err := parser.Parse(ft, planBld) | |||||
| if err != nil { | if err != nil { | ||||
| // TODO 错误处理 | // TODO 错误处理 | ||||
| continue | continue | ||||