Browse Source

FromTo支持分段编码

gitlink
Sydonian 11 months ago
parent
commit
4a9a73b850
15 changed files with 950 additions and 405 deletions
  1. +79
    -147
      client/internal/cmdline/test.go
  2. +1
    -1
      common/pkgs/downloader/iterator.go
  3. +2
    -1
      common/pkgs/downloader/strip_iterator.go
  4. +67
    -63
      common/pkgs/ioswitch2/fromto.go
  5. +81
    -0
      common/pkgs/ioswitch2/ops2/driver.go
  6. +3
    -0
      common/pkgs/ioswitch2/ops2/ops.go
  7. +5
    -1
      common/pkgs/ioswitch2/ops2/range.go
  8. +214
    -0
      common/pkgs/ioswitch2/ops2/segment.go
  9. +15
    -2
      common/pkgs/ioswitch2/ops2/shard_store.go
  10. +8
    -1
      common/pkgs/ioswitch2/ops2/shared_store.go
  11. +311
    -83
      common/pkgs/ioswitch2/parser/parser.go
  12. +2
    -2
      common/pkgs/uploader/create_load.go
  13. +1
    -1
      common/pkgs/uploader/update.go
  14. +158
    -101
      scanner/internal/event/check_package_redundancy.go
  15. +3
    -2
      scanner/internal/event/clean_pinned.go

+ 79
- 147
client/internal/cmdline/test.go View File

@@ -1,6 +1,5 @@
package cmdline

/*
import (
"context"
"fmt"
@@ -13,54 +12,31 @@ import (
stgglb "gitlink.org.cn/cloudream/storage/common/globals"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc"
lrcparser "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc/parser"
coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
)

func init() {
rootCmd.AddCommand(&cobra.Command{
Use: "test",
Short: "test",
// Args: cobra.ExactArgs(1),
Use: "test2",
Short: "test2",
Run: func(cmd *cobra.Command, args []string) {
// cmdCtx := GetCmdCtx(cmd)
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
panic(err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2}))
stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2}))
if err != nil {
panic(err)
}

ft := ioswitch2.NewFromTo()

ft.AddFrom(ioswitch2.NewFromNode("Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", &nodes.Nodes[0], -1))
ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], -1, "asd"))
le := int64(3)
toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 5, Length: &le})
ft.AddTo(toExec)
ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 0, "0"))
ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 1, "1"))
ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 2, "2"))

// ft.AddFrom(ioswitch2.NewFromNode("QmS2s8GRYHEurXL7V1zUtKvf2H1BGcQc5NN1T1hiSnWvbd", &nodes.Nodes[0], 1))
// ft.AddFrom(ioswitch2.NewFromNode("QmUgUEUMzdnjPNx6xu9PDGXpSyXTk8wzPWvyYZ9zasE1WW", &nodes.Nodes[1], 2))
// le := int64(5)
// toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 3, Length: &le})
// toExec, hd := plans.NewToExecutorWithRange(1, plans.Range{Offset: 0, Length: nil})
// toExec2, hd2 := plans.NewToExecutorWithRange(2, plans.Range{Offset: 0, Length: nil})
// ft.AddTo(toExec)
// ft.AddTo(toExec2)

// fromExec, hd := plans.NewFromExecutor(-1)
// ft.AddFrom(fromExec)
// ft.AddTo(plans.NewToNode(nodes.Nodes[1], -1, "asd"))

parser := parser.NewParser(cdssdk.DefaultECRedundancy)
ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3)
ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0), "0"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1), "1"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2), "2"))

plans := exec.NewPlanBuilder()
err = parser.Parse(ft, plans)
@@ -68,7 +44,9 @@ func init() {
panic(err)
}

exec := plans.Execute()
fmt.Printf("plans: %v\n", plans)

exec := plans.Execute(exec.NewExecContext())

fut := future.NewSetVoid()
go func() {
@@ -77,176 +55,130 @@ func init() {
panic(err)
}

fmt.Printf("mp: %+v\n", mp)
fmt.Printf("0: %v, 1: %v, 2: %v\n", mp["0"], mp["1"], mp["2"])
fut.SetVoid()
}()

go func() {
// exec.BeginWrite(io.NopCloser(bytes.NewBuffer([]byte("hello world"))), hd)
// if err != nil {
// panic(err)
// }

str, err := exec.BeginRead(hd)
if err != nil {
panic(err)
}
defer str.Close()
data, err := io.ReadAll(str)
if err != nil && err != io.EOF {
panic(err)
}
fmt.Printf("data: %v(%v)\n", string(data), len(data))
}()

fut.Wait(context.TODO())
},
})

// rootCmd.AddCommand(&cobra.Command{
// Use: "test",
// Short: "test",
// // Args: cobra.ExactArgs(1),
// Run: func(cmd *cobra.Command, args []string) {
// cmdCtx := GetCmdCtx(cmd)
// file, _ := cmdCtx.Cmdline.Svc.ObjectSvc().Download(1, downloader.DownloadReqeust{
// ObjectID: 27379,
// Length: -1,
// })
// data, _ := io.ReadAll(file.File)
// fmt.Printf("data: %v(%v)\n", string(data), len(data))
// },
// })

rootCmd.AddCommand(&cobra.Command{
Use: "test3",
Short: "test3",
// Args: cobra.ExactArgs(1),
Use: "test",
Short: "test",
Run: func(cmd *cobra.Command, args []string) {
// cmdCtx := GetCmdCtx(cmd)

coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
panic(err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2}))
stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2}))
if err != nil {
panic(err)
}

red := cdssdk.DefaultLRCRedundancy

var toes []ioswitchlrc.To
for i := 0; i < red.N; i++ {
toes = append(toes, ioswitchlrc.NewToNode(nodes.Nodes[i%2], i, fmt.Sprintf("%d", i)))
}
ft := ioswitch2.NewFromTo()
ft.SegmentParam = cdssdk.NewSegmentRedundancy(1293, 3)
ft.ECParam = &cdssdk.DefaultECRedundancy
ft.AddFrom(ioswitch2.NewFromShardstore("22CC59CE3297F78F2D20DC1E33181B77F21E6782097C94E1664F99F129834069", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(0)))
ft.AddFrom(ioswitch2.NewFromShardstore("5EAC20EB3EBC7B5FA176C5BD1C01041FB2A6D14C35D6A232CA83D7F1E4B01ADE", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(1)))
ft.AddFrom(ioswitch2.NewFromShardstore("A9BC1802F37100C80C72A1D6E8F53C0E0B73F85F99153D8C78FB01CEC9D8D903", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.SegmentStream(2)))

toDrv, drvStr := ioswitch2.NewToDriverWithRange(ioswitch2.RawStream(), exec.NewRange(500, 500))
ft.AddTo(toDrv)
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2"))

plans := exec.NewPlanBuilder()
err = lrcparser.Encode(ioswitchlrc.NewFromNode("QmNspjDLxQbAsuh37jRXKvLWHE2f7JpqY4HEJ8x7Jgbzqa", &nodes.Nodes[0], -1), toes, plans)
err = parser.Parse(ft, plans)
if err != nil {
panic(err)
// return nil, fmt.Errorf("parsing plan: %w", err)
}

ioRet, err := plans.Execute().Wait(context.TODO())
if err != nil {
panic(err)
// return nil, fmt.Errorf("executing io plan: %w", err)
}
fmt.Printf("plans: %v\n", plans)

fmt.Printf("ioRet: %v\n", ioRet)
},
})
exec := plans.Execute(exec.NewExecContext())

rootCmd.AddCommand(&cobra.Command{
Use: "test4",
Short: "test4",
// Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
// cmdCtx := GetCmdCtx(cmd)
fut := future.NewSetVoid()
go func() {
mp, err := exec.Wait(context.Background())
if err != nil {
panic(err)
}

coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
panic(err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)
for k, v := range mp {
fmt.Printf("%s: %v\n", k, v)
}

nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2}))
if err != nil {
panic(err)
}
fut.SetVoid()
}()

// red := cdssdk.DefaultLRCRedundancy
go func() {
str, err := exec.BeginRead(drvStr)
if err != nil {
panic(err)
}

plans := exec.NewPlanBuilder()
err = lrcparser.ReconstructGroup([]ioswitchlrc.From{
ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 0),
ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 1),
}, []ioswitchlrc.To{
ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"),
}, plans)
if err != nil {
panic(err)
// return nil, fmt.Errorf("parsing plan: %w", err)
}
data, err := io.ReadAll(str)
if err != nil {
panic(err)
}

ioRet, err := plans.Execute().Wait(context.TODO())
if err != nil {
panic(err)
// return nil, fmt.Errorf("executing io plan: %w", err)
}
fmt.Printf("read(%v): %s\n", len(data), string(data))
}()

fmt.Printf("ioRet: %v\n", ioRet)
fut.Wait(context.TODO())
},
})

rootCmd.AddCommand(&cobra.Command{
Use: "test3",
Short: "test3",
// Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
// cmdCtx := GetCmdCtx(cmd)

coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
panic(err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.HubID{1, 2}))
stgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{1, 2}))
if err != nil {
panic(err)
}

// red := cdssdk.DefaultLRCRedundancy
ft := ioswitch2.NewFromTo()
ft.ECParam = &cdssdk.DefaultECRedundancy
ft.AddFrom(ioswitch2.NewFromShardstore("4E69A8B8CD9F42EDE371DA94458BADFB2308AFCA736AA393784A3D81F4746377", *stgs.Storages[1].MasterHub, stgs.Storages[1].Storage, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(0), "EC0"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(1), "EC1"))
ft.AddTo(ioswitch2.NewToShardStore(*stgs.Storages[0].MasterHub, stgs.Storages[0].Storage, ioswitch2.ECSrteam(2), "EC2"))

plans := exec.NewPlanBuilder()
le := int64(1293)
err = lrcparser.ReconstructAny([]ioswitchlrc.From{
ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[0], 0),
ioswitchlrc.NewFromNode("QmQBKncEDqxw3BrGr3th3gS3jUC2fizGz1w29ZxxrrKfNv", &nodes.Nodes[0], 2),
}, []ioswitchlrc.To{
ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], -1, "-1", exec.Range{0, &le}),
ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], 0, "0", exec.Range{10, &le}),
ioswitchlrc.NewToNode(nodes.Nodes[1], 1, "1"),
ioswitchlrc.NewToNode(nodes.Nodes[1], 2, "2"),
ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"),
}, plans)
err = parser.Parse(ft, plans)
if err != nil {
panic(err)
// return nil, fmt.Errorf("parsing plan: %w", err)
}

ioRet, err := plans.Execute().Wait(context.TODO())
if err != nil {
panic(err)
// return nil, fmt.Errorf("executing io plan: %w", err)
}
fmt.Printf("plans: %v\n", plans)

exec := plans.Execute(exec.NewExecContext())

fmt.Printf("ioRet: %v\n", ioRet)
fut := future.NewSetVoid()
go func() {
mp, err := exec.Wait(context.Background())
if err != nil {
panic(err)
}

for k, v := range mp {
fmt.Printf("%s: %v\n", k, v)
}

fut.SetVoid()
}()

fut.Wait(context.TODO())
},
})
}
*/

+ 1
- 1
common/pkgs/downloader/iterator.go View File

@@ -400,7 +400,7 @@ func (iter *DownloadObjectIterator) downloadFromStorage(stg *stgmod.StorageDetai
strHandle = handle

plans := exec.NewPlanBuilder()
if err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy); err != nil {
if err := parser.Parse(ft, plans); err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}



+ 2
- 1
common/pkgs/downloader/strip_iterator.go View File

@@ -199,6 +199,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) {
}

ft := ioswitch2.NewFromTo()
ft.ECParam = s.red
for _, b := range s.blocks {
stg := b.Storage
ft.AddFrom(ioswitch2.NewFromShardstore(b.Block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(b.Block.Index)))
@@ -210,7 +211,7 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) {
ft.AddTo(toExec)

plans := exec.NewPlanBuilder()
err := parser.Parse(ft, plans, *s.red)
err := parser.Parse(ft, plans)
if err != nil {
return 0, err
}


+ 67
- 63
common/pkgs/ioswitch2/fromto.go View File

@@ -6,7 +6,7 @@ import (
)

type From interface {
GetStreamType() StreamType
GetStreamIndex() StreamIndex
}

type To interface {
@@ -14,60 +14,64 @@ type To interface {
// 如果DataIndex == -1,则表示在整个文件的范围。
// 如果DataIndex >= 0,则表示在文件的某个分片的范围。
GetRange() exec.Range
GetStreamType() StreamType
GetStreamIndex() StreamIndex
}

const (
// 未处理的完整文件流
StreamTypeRaw = iota
StreamIndexRaw = iota
// EC编码的某一块的流
StreamTypeEC
StreamIndexEC
// 分段编码的某一段的流
StreamTypeSegment
StreamIndexSegment
)

type StreamType struct {
type StreamIndex struct {
Type int
Index int
}

func RawStream() StreamType {
return StreamType{
Type: StreamTypeRaw,
func RawStream() StreamIndex {
return StreamIndex{
Type: StreamIndexRaw,
}
}

func ECSrteam(index int) StreamType {
return StreamType{
Type: StreamTypeEC,
func ECSrteam(index int) StreamIndex {
return StreamIndex{
Type: StreamIndexEC,
Index: index,
}
}

func SegmentStream(index int) StreamType {
return StreamType{
Type: StreamTypeSegment,
func SegmentStream(index int) StreamIndex {
return StreamIndex{
Type: StreamIndexSegment,
Index: index,
}
}

func (s StreamType) IsRaw() bool {
return s.Type == StreamTypeRaw
func (s StreamIndex) IsRaw() bool {
return s.Type == StreamIndexRaw
}

func (s StreamType) IsEC() bool {
return s.Type == StreamTypeEC
func (s StreamIndex) IsEC() bool {
return s.Type == StreamIndexEC
}

func (s StreamType) IsSegment() bool {
return s.Type == StreamTypeSegment
func (s StreamIndex) IsSegment() bool {
return s.Type == StreamIndexSegment
}

type FromTos []FromTo

type FromTo struct {
Froms []From
Toes []To
// 如果输入或者输出用到了EC编码的流,则需要提供EC参数。
ECParam *cdssdk.ECRedundancy
// 同上
SegmentParam *cdssdk.SegmentRedundancy
Froms []From
Toes []To
}

func NewFromTo() FromTo {
@@ -85,69 +89,69 @@ func (ft *FromTo) AddTo(to To) *FromTo {
}

type FromDriver struct {
Handle *exec.DriverWriteStream
StreamType StreamType
Handle *exec.DriverWriteStream
StreamIndex StreamIndex
}

func NewFromDriver(strType StreamType) (*FromDriver, *exec.DriverWriteStream) {
func NewFromDriver(strIdx StreamIndex) (*FromDriver, *exec.DriverWriteStream) {
handle := &exec.DriverWriteStream{
RangeHint: &exec.Range{},
}
return &FromDriver{
Handle: handle,
StreamType: strType,
Handle: handle,
StreamIndex: strIdx,
}, handle
}

func (f *FromDriver) GetStreamType() StreamType {
return f.StreamType
func (f *FromDriver) GetStreamIndex() StreamIndex {
return f.StreamIndex
}

type FromShardstore struct {
FileHash cdssdk.FileHash
Hub cdssdk.Hub
Storage cdssdk.Storage
StreamType StreamType
FileHash cdssdk.FileHash
Hub cdssdk.Hub
Storage cdssdk.Storage
StreamIndex StreamIndex
}

func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strType StreamType) *FromShardstore {
func NewFromShardstore(fileHash cdssdk.FileHash, hub cdssdk.Hub, storage cdssdk.Storage, strIdx StreamIndex) *FromShardstore {
return &FromShardstore{
FileHash: fileHash,
Hub: hub,
Storage: storage,
StreamType: strType,
FileHash: fileHash,
Hub: hub,
Storage: storage,
StreamIndex: strIdx,
}
}

func (f *FromShardstore) GetStreamType() StreamType {
return f.StreamType
func (f *FromShardstore) GetStreamIndex() StreamIndex {
return f.StreamIndex
}

type ToDriver struct {
Handle *exec.DriverReadStream
StreamType StreamType
Range exec.Range
Handle *exec.DriverReadStream
StreamIndex StreamIndex
Range exec.Range
}

func NewToDriver(strType StreamType) (*ToDriver, *exec.DriverReadStream) {
func NewToDriver(strIdx StreamIndex) (*ToDriver, *exec.DriverReadStream) {
str := exec.DriverReadStream{}
return &ToDriver{
Handle: &str,
StreamType: strType,
Handle: &str,
StreamIndex: strIdx,
}, &str
}

func NewToDriverWithRange(strType StreamType, rng exec.Range) (*ToDriver, *exec.DriverReadStream) {
func NewToDriverWithRange(strIdx StreamIndex, rng exec.Range) (*ToDriver, *exec.DriverReadStream) {
str := exec.DriverReadStream{}
return &ToDriver{
Handle: &str,
StreamType: strType,
Range: rng,
Handle: &str,
StreamIndex: strIdx,
Range: rng,
}, &str
}

func (t *ToDriver) GetStreamType() StreamType {
return t.StreamType
func (t *ToDriver) GetStreamIndex() StreamIndex {
return t.StreamIndex
}

func (t *ToDriver) GetRange() exec.Range {
@@ -157,32 +161,32 @@ func (t *ToDriver) GetRange() exec.Range {
type ToShardStore struct {
Hub cdssdk.Hub
Storage cdssdk.Storage
StreamType StreamType
StreamIndex StreamIndex
Range exec.Range
FileHashStoreKey string
}

func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strType StreamType, fileHashStoreKey string) *ToShardStore {
func NewToShardStore(hub cdssdk.Hub, stg cdssdk.Storage, strIdx StreamIndex, fileHashStoreKey string) *ToShardStore {
return &ToShardStore{
Hub: hub,
Storage: stg,
StreamType: strType,
StreamIndex: strIdx,
FileHashStoreKey: fileHashStoreKey,
}
}

func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamType StreamType, fileHashStoreKey string, rng exec.Range) *ToShardStore {
func NewToShardStoreWithRange(hub cdssdk.Hub, stg cdssdk.Storage, streamIndex StreamIndex, fileHashStoreKey string, rng exec.Range) *ToShardStore {
return &ToShardStore{
Hub: hub,
Storage: stg,
StreamType: streamType,
StreamIndex: streamIndex,
FileHashStoreKey: fileHashStoreKey,
Range: rng,
}
}

func (t *ToShardStore) GetStreamType() StreamType {
return t.StreamType
func (t *ToShardStore) GetStreamIndex() StreamIndex {
return t.StreamIndex
}

func (t *ToShardStore) GetRange() exec.Range {
@@ -207,9 +211,9 @@ func NewLoadToShared(hub cdssdk.Hub, storage cdssdk.Storage, userID cdssdk.UserI
}
}

func (t *LoadToShared) GetStreamType() StreamType {
return StreamType{
Type: StreamTypeRaw,
func (t *LoadToShared) GetStreamIndex() StreamIndex {
return StreamIndex{
Type: StreamIndexRaw,
}
}



+ 81
- 0
common/pkgs/ioswitch2/ops2/driver.go View File

@@ -0,0 +1,81 @@
package ops2

import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
)

// TODO 想办法直接使用ops里的Node,而不是重新实现一遍

type FromDriverNode struct {
dag.NodeBase
From ioswitch2.From
Handle *exec.DriverWriteStream
}

func (b *GraphNodeBuilder) NewFromDriver(fr ioswitch2.From, handle *exec.DriverWriteStream) *FromDriverNode {
node := &FromDriverNode{
From: fr,
Handle: handle,
}
b.AddNode(node)

node.OutputStreams().SetupNew(node, b.NewVar())

return node
}

func (f *FromDriverNode) GetFrom() ioswitch2.From {
return f.From
}

func (t *FromDriverNode) Output() dag.Slot {
return dag.Slot{
Var: t.OutputStreams().Get(0),
Index: 0,
}
}

func (t *FromDriverNode) GenerateOp() (exec.Op, error) {
t.Handle.ID = t.OutputStreams().Get(0).VarID
return nil, nil
}

type ToDriverNode struct {
dag.NodeBase
To ioswitch2.To
Handle *exec.DriverReadStream
Range exec.Range
}

func (b *GraphNodeBuilder) NewToDriver(to ioswitch2.To, handle *exec.DriverReadStream) *ToDriverNode {
node := &ToDriverNode{
To: to,
Handle: handle,
}
b.AddNode(node)

return node
}

func (t *ToDriverNode) GetTo() ioswitch2.To {
return t.To
}

func (t *ToDriverNode) SetInput(v *dag.Var) {
t.InputStreams().EnsureSize(1)
v.StreamTo(t, 0)
}

func (t *ToDriverNode) Input() dag.Slot {
return dag.Slot{
Var: t.InputStreams().Get(0),
Index: 0,
}
}

func (t *ToDriverNode) GenerateOp() (exec.Op, error) {
t.Handle.ID = t.InputStreams().Get(0).VarID
return nil, nil
}

+ 3
- 0
common/pkgs/ioswitch2/ops2/ops.go View File

@@ -3,6 +3,7 @@ package ops2
import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
)

type GraphNodeBuilder struct {
@@ -15,11 +16,13 @@ func NewGraphNodeBuilder() *GraphNodeBuilder {

type FromNode interface {
dag.Node
GetFrom() ioswitch2.From
Output() dag.Slot
}

type ToNode interface {
dag.Node
GetTo() ioswitch2.To
Input() dag.Slot
SetInput(input *dag.Var)
}


+ 5
- 1
common/pkgs/ioswitch2/ops2/range.go View File

@@ -72,7 +72,11 @@ func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
}

func (o *Range) String() string {
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input, o.Output)
len := ""
if o.Length != nil {
len = fmt.Sprintf("%v", *o.Length)
}
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, len, o.Input, o.Output)
}

type RangeNode struct {


+ 214
- 0
common/pkgs/ioswitch2/ops2/segment.go View File

@@ -1 +1,215 @@
package ops2

import (
"context"
"fmt"
"io"

"gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/common/utils/io2"
)

func init() {
exec.UseOp[*SegmentSplit]()
exec.UseOp[*SegmentJoin]()
}

type SegmentSplit struct {
Input exec.VarID
Segments []int64
Outputs []exec.VarID
}

func (o *SegmentSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil {
return err
}
defer input.Stream.Close()

for i, outID := range o.Outputs {
fut := future.NewSetVoid()

segStr := io.LimitReader(input.Stream, o.Segments[i])
segStr2 := io2.DelegateReadCloser(segStr, func() error {
fut.SetError(context.Canceled)
return nil
})

segStr2 = io2.AfterEOF(segStr2, func(str io.ReadCloser, err error) {
fut.SetVoid()
})

e.PutVar(outID, &exec.StreamValue{Stream: segStr2})
err = fut.Wait(ctx.Context)
if err != nil {
return err
}
}

return nil
}

func (o *SegmentSplit) String() string {
return fmt.Sprintf("SegmentSplit(%v, %v) -> %v", o.Input, o.Segments, o.Outputs)
}

type SegmentJoin struct {
Inputs []exec.VarID
Output exec.VarID
// 这些字段只在执行时使用
ctx *exec.ExecContext
e *exec.Executor
nextStreamIdx int
nextStream io.ReadCloser
fut *future.SetVoidFuture
}

func (o *SegmentJoin) Read(buf []byte) (int, error) {
for {
if o.nextStream == nil {
if o.nextStreamIdx >= len(o.Inputs) {
o.fut.SetVoid()
return 0, io.EOF
}

input, err := exec.BindVar[*exec.StreamValue](o.e, o.ctx.Context, o.Inputs[o.nextStreamIdx])
if err != nil {
return 0, err
}

o.nextStream = input.Stream
o.nextStreamIdx++
}

n, err := o.nextStream.Read(buf)
if err == io.EOF {
o.nextStream.Close()
o.nextStream = nil
continue
}
return n, err
}
}

func (o *SegmentJoin) Close() error {
if o.nextStream != nil {
o.nextStream.Close()
o.nextStream = nil
o.fut.SetVoid()
}

return nil
}

func (o *SegmentJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
o.ctx = ctx
o.e = e
o.nextStreamIdx = 0
o.nextStream = nil
o.fut = future.NewSetVoid()

e.PutVar(o.Output, &exec.StreamValue{Stream: o})
return o.fut.Wait(ctx.Context)
}

func (o *SegmentJoin) String() string {
return fmt.Sprintf("SegmentJoin %v -> %v", utils.FormatVarIDs(o.Inputs), o.Output)
}

type SegmentSplitNode struct {
dag.NodeBase
segments []int64
}

func (b *GraphNodeBuilder) NewSegmentSplit(segments []int64) *SegmentSplitNode {
node := &SegmentSplitNode{
segments: segments,
}
b.AddNode(node)
node.OutputStreams().Resize(len(segments))
return node
}

func (n *SegmentSplitNode) SetInput(input *dag.Var) {
n.InputStreams().EnsureSize(1)
input.StreamTo(n, 0)
}

func (n *SegmentSplitNode) Segment(index int) *dag.Var {
// 必须连续消耗流
for i := 0; i <= index; i++ {
if n.OutputStreams().Get(i) == nil {
n.OutputStreams().Setup(n, n.Graph().NewVar(), i)
}
}
return n.OutputStreams().Get(index)
}

func (t *SegmentSplitNode) GenerateOp() (exec.Op, error) {
lastUsedSeg := 0
for i := t.OutputStreams().Len() - 1; i >= 0; i-- {
if t.OutputStreams().Get(i) != nil {
lastUsedSeg = i
break
}
}

return &SegmentSplit{
Input: t.InputStreams().Get(0).VarID,
Segments: t.segments[:lastUsedSeg+1],
Outputs: t.OutputStreams().GetVarIDs(),
}, nil
}

type SegmentJoinNode struct {
dag.NodeBase
UsedStart int
UsedCount int
}

func (b *GraphNodeBuilder) NewSegmentJoin(segmentSizes []int64) *SegmentJoinNode {
node := &SegmentJoinNode{}
b.AddNode(node)
node.InputStreams().Resize(len(segmentSizes))
node.OutputStreams().SetupNew(node, b.NewVar())
return node
}

func (n *SegmentJoinNode) SetInput(index int, input *dag.Var) {
input.StreamTo(n, index)
}

// 记录本计划中实际要使用的分段的范围,范围外的分段流都会取消输入
func (n *SegmentJoinNode) MarkUsed(start, cnt int) {
n.UsedStart = start
n.UsedCount = cnt

for i := 0; i < start; i++ {
str := n.InputStreams().Get(i)
if str != nil {
str.StreamNotTo(n, i)
}
}

for i := start + cnt; i < n.InputStreams().Len(); i++ {
str := n.InputStreams().Get(i)
if str != nil {
str.StreamNotTo(n, i)
}
}
}

func (n *SegmentJoinNode) Joined() *dag.Var {
return n.OutputStreams().Get(0)
}

func (t *SegmentJoinNode) GenerateOp() (exec.Op, error) {
return &SegmentJoin{
Inputs: t.InputStreams().GetVarIDsRanged(t.UsedStart, t.UsedStart+t.UsedCount),
Output: t.OutputStreams().Get(0).VarID,
}, nil
}

+ 15
- 2
common/pkgs/ioswitch2/ops2/shard_store.go View File

@@ -10,6 +10,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr"
"gitlink.org.cn/cloudream/storage/common/pkgs/storage/types"
)
@@ -115,12 +116,14 @@ func (o *ShardWrite) String() string {

type ShardReadNode struct {
dag.NodeBase
From ioswitch2.From
StorageID cdssdk.StorageID
Open types.OpenOption
}

func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode {
func (b *GraphNodeBuilder) NewShardRead(fr ioswitch2.From, stgID cdssdk.StorageID, open types.OpenOption) *ShardReadNode {
node := &ShardReadNode{
From: fr,
StorageID: stgID,
Open: open,
}
@@ -129,6 +132,10 @@ func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenO
return node
}

func (t *ShardReadNode) GetFrom() ioswitch2.From {
return t.From
}

func (t *ShardReadNode) Output() dag.Slot {
return dag.Slot{
Var: t.OutputStreams().Get(0),
@@ -150,12 +157,14 @@ func (t *ShardReadNode) GenerateOp() (exec.Op, error) {

type ShardWriteNode struct {
dag.NodeBase
To ioswitch2.To
StorageID cdssdk.StorageID
FileHashStoreKey string
}

func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode {
func (b *GraphNodeBuilder) NewShardWrite(to ioswitch2.To, stgID cdssdk.StorageID, fileHashStoreKey string) *ShardWriteNode {
node := &ShardWriteNode{
To: to,
StorageID: stgID,
FileHashStoreKey: fileHashStoreKey,
}
@@ -163,6 +172,10 @@ func (b *GraphNodeBuilder) NewShardWrite(stgID cdssdk.StorageID, fileHashStoreKe
return node
}

func (t *ShardWriteNode) GetTo() ioswitch2.To {
return t.To
}

func (t *ShardWriteNode) SetInput(input *dag.Var) {
t.InputStreams().EnsureSize(1)
input.StreamTo(t, 0)


+ 8
- 1
common/pkgs/ioswitch2/ops2/shared_store.go View File

@@ -7,6 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/storage/common/pkgs/storage/mgr"
)

@@ -64,14 +65,16 @@ func (o *SharedLoad) String() string {

type SharedLoadNode struct {
dag.NodeBase
To ioswitch2.To
StorageID cdssdk.StorageID
UserID cdssdk.UserID
PackageID cdssdk.PackageID
Path string
}

func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode {
func (b *GraphNodeBuilder) NewSharedLoad(to ioswitch2.To, stgID cdssdk.StorageID, userID cdssdk.UserID, packageID cdssdk.PackageID, path string) *SharedLoadNode {
node := &SharedLoadNode{
To: to,
StorageID: stgID,
UserID: userID,
PackageID: packageID,
@@ -81,6 +84,10 @@ func (b *GraphNodeBuilder) NewSharedLoad(stgID cdssdk.StorageID, userID cdssdk.U
return node
}

func (t *SharedLoadNode) GetTo() ioswitch2.To {
return t.To
}

func (t *SharedLoadNode) SetInput(input *dag.Var) {
t.InputStreams().EnsureSize(1)
input.StreamTo(t, 0)


+ 311
- 83
common/pkgs/ioswitch2/parser/parser.go View File

@@ -15,9 +15,9 @@ import (
"gitlink.org.cn/cloudream/storage/common/pkgs/storage/types"
)

type TypedStream struct {
Stream *dag.Var
StreamType ioswitch2.StreamType
type IndexedStream struct {
Stream *dag.Var
StreamIndex ioswitch2.StreamIndex
}

type ParseContext struct {
@@ -25,33 +25,43 @@ type ParseContext struct {
DAG *ops2.GraphNodeBuilder
// 为了产生所有To所需的数据范围,而需要From打开的范围。
// 这个范围是基于整个文件的,且上下界都取整到条带大小的整数倍,因此上界是有可能超过文件大小的。
ToNodes map[ioswitch2.To]ops2.ToNode
TypedStreams []TypedStream
StreamRange exec.Range
EC cdssdk.ECRedundancy
ToNodes map[ioswitch2.To]ops2.ToNode
IndexedStreams []IndexedStream
StreamRange exec.Range
UseEC bool // 是否使用纠删码
UseSegment bool // 是否使用分段
}

func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy) error {
func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) error {
ctx := ParseContext{
Ft: ft,
DAG: ops2.NewGraphNodeBuilder(),
ToNodes: make(map[ioswitch2.To]ops2.ToNode),
EC: ec,
}

// 分成两个阶段:
// 1. 基于From和To生成更多指令,初步匹配to的需求

err := checkEncodingParams(&ctx)
if err != nil {
return err
}

// 计算一下打开流的范围
calcStreamRange(&ctx)

err := extend(&ctx)
err = extend(&ctx)
if err != nil {
return err
}

// 2. 优化上一步生成的指令

err = removeUnusedSegment(&ctx)
if err != nil {
return err
}

// 对于删除指令的优化,需要反复进行,直到没有变化为止。
// 从目前实现上来说不会死循环
for {
@@ -79,17 +89,18 @@ func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder, ec cdssdk.ECRedundancy)
}

// 下面这些只需要执行一次,但需要按顺序
removeUnusedFromNode(&ctx)
dropUnused(&ctx)
storeIPFSWriteResult(&ctx)
generateClone(&ctx)
generateRange(&ctx)
generateClone(&ctx)

return plan.Generate(ctx.DAG.Graph, blder)
}
func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.Var {
func findOutputStream(ctx *ParseContext, streamIndex ioswitch2.StreamIndex) *dag.Var {
var ret *dag.Var
for _, s := range ctx.TypedStreams {
if s.StreamType == streamType {
for _, s := range ctx.IndexedStreams {
if s.StreamIndex == streamIndex {
ret = s.Stream
break
}
@@ -97,35 +108,91 @@ func findOutputStream(ctx *ParseContext, streamType ioswitch2.StreamType) *dag.V
return ret
}

// 计算输入流的打开范围。会把流的范围按条带大小取整
func calcStreamRange(ctx *ParseContext) {
stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K)
// 检查使用不同编码时参数是否设置到位
func checkEncodingParams(ctx *ParseContext) error {
for _, f := range ctx.Ft.Froms {
if f.GetStreamIndex().IsEC() {
ctx.UseEC = true
if ctx.Ft.ECParam == nil {
return fmt.Errorf("EC encoding parameters not set")
}
}

rng := exec.Range{
Offset: math.MaxInt64,
if f.GetStreamIndex().IsSegment() {
ctx.UseSegment = true
if ctx.Ft.SegmentParam == nil {
return fmt.Errorf("segment parameters not set")
}
}
}

for _, t := range ctx.Ft.Toes {
if t.GetStreamIndex().IsEC() {
ctx.UseEC = true
if ctx.Ft.ECParam == nil {
return fmt.Errorf("EC encoding parameters not set")
}
}

if t.GetStreamIndex().IsSegment() {
ctx.UseSegment = true
if ctx.Ft.SegmentParam == nil {
return fmt.Errorf("segment parameters not set")
}
}
}

return nil
}

// 计算输入流的打开范围。如果From或者To中包含EC的流,则会将打开范围扩大到条带大小的整数倍。
func calcStreamRange(ctx *ParseContext) {
rng := exec.NewRange(math.MaxInt64, 0)

for _, to := range ctx.Ft.Toes {
if to.GetStreamType().IsRaw() {
strIdx := to.GetStreamIndex()
if strIdx.IsRaw() {
toRng := to.GetRange()
rng.ExtendStart(math2.Floor(toRng.Offset, stripSize))
rng.ExtendStart(toRng.Offset)
if toRng.Length != nil {
rng.ExtendEnd(math2.Ceil(toRng.Offset+*toRng.Length, stripSize))
rng.ExtendEnd(toRng.Offset + *toRng.Length)
} else {
rng.Length = nil
}

} else {
} else if strIdx.IsEC() {
toRng := to.GetRange()

blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.EC.ChunkSize))
stripSize := ctx.Ft.ECParam.StripSize()
blkStartIndex := math2.FloorDiv(toRng.Offset, int64(ctx.Ft.ECParam.ChunkSize))
rng.ExtendStart(blkStartIndex * stripSize)
if toRng.Length != nil {
blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.EC.ChunkSize))
blkEndIndex := math2.CeilDiv(toRng.Offset+*toRng.Length, int64(ctx.Ft.ECParam.ChunkSize))
rng.ExtendEnd(blkEndIndex * stripSize)
} else {
rng.Length = nil
}

} else if strIdx.IsSegment() {
// Segment节点的Range是相对于本段的,需要加上本段的起始位置
toRng := to.GetRange()

segStart := ctx.Ft.SegmentParam.CalcSegmentStart(strIdx.Index)

offset := toRng.Offset + segStart

rng.ExtendStart(offset)
if toRng.Length != nil {
rng.ExtendEnd(offset + *toRng.Length)
} else {
rng.Length = nil
}
}
}

if ctx.UseEC {
stripSize := ctx.Ft.ECParam.StripSize()
rng.ExtendStart(math2.Floor(rng.Offset, stripSize))
if rng.Length != nil {
rng.ExtendEnd(math2.Ceil(rng.Offset+*rng.Length, stripSize))
}
}

@@ -139,57 +206,112 @@ func extend(ctx *ParseContext) error {
return err
}

ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{
Stream: frNode.Output().Var,
StreamType: fr.GetStreamType(),
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: frNode.Output().Var,
StreamIndex: fr.GetStreamIndex(),
})

// 对于完整文件的From,生成Split指令
if fr.GetStreamType().IsRaw() {
splitNode := ctx.DAG.NewChunkedSplit(ctx.EC.ChunkSize)
splitNode.Split(frNode.Output().Var, ctx.EC.K)
for i := 0; i < ctx.EC.K; i++ {
ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{
Stream: splitNode.SubStream(i),
StreamType: ioswitch2.ECSrteam(i),
})
if fr.GetStreamIndex().IsRaw() {
// 只有输入输出需要EC编码的块时,才生成相关指令
if ctx.UseEC {
splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize)
splitNode.Split(frNode.Output().Var, ctx.Ft.ECParam.K)
for i := 0; i < ctx.Ft.ECParam.K; i++ {
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: splitNode.SubStream(i),
StreamIndex: ioswitch2.ECSrteam(i),
})
}
}

// 同上
if ctx.UseSegment {
splitNode := ctx.DAG.NewSegmentSplit(ctx.Ft.SegmentParam.Segments)
splitNode.SetInput(frNode.Output().Var)
for i := 0; i < len(ctx.Ft.SegmentParam.Segments); i++ {
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: splitNode.Segment(i),
StreamIndex: ioswitch2.SegmentStream(i),
})
}
}
}
}

// 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令
ecInputStrs := make(map[int]*dag.Var)
for _, s := range ctx.TypedStreams {
if s.StreamType.IsEC() && ecInputStrs[s.StreamType.Index] == nil {
ecInputStrs[s.StreamType.Index] = s.Stream
if len(ecInputStrs) == ctx.EC.K {
break
if ctx.UseEC {
// 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令
ecInputStrs := make(map[int]*dag.Var)
for _, s := range ctx.IndexedStreams {
if s.StreamIndex.IsEC() && ecInputStrs[s.StreamIndex.Index] == nil {
ecInputStrs[s.StreamIndex.Index] = s.Stream
if len(ecInputStrs) == ctx.Ft.ECParam.K {
break
}
}
}
}

if len(ecInputStrs) == ctx.EC.K {
mulNode := ctx.DAG.NewECMultiply(ctx.EC)
if len(ecInputStrs) == ctx.Ft.ECParam.K {
mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam)

for i, s := range ecInputStrs {
mulNode.AddInput(s, i)
}
for i := 0; i < ctx.EC.N; i++ {
ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{
Stream: mulNode.NewOutput(i),
StreamType: ioswitch2.ECSrteam(i),
for i, s := range ecInputStrs {
mulNode.AddInput(s, i)
}
for i := 0; i < ctx.Ft.ECParam.N; i++ {
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: mulNode.NewOutput(i),
StreamIndex: ioswitch2.ECSrteam(i),
})
}

joinNode := ctx.DAG.NewChunkedJoin(ctx.Ft.ECParam.ChunkSize)
for i := 0; i < ctx.Ft.ECParam.K; i++ {
// 不可能找不到流
joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i)))
}
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: joinNode.Joined(),
StreamIndex: ioswitch2.RawStream(),
})
}
}

joinNode := ctx.DAG.NewChunkedJoin(ctx.EC.ChunkSize)
for i := 0; i < ctx.EC.K; i++ {
// 不可能找不到流
joinNode.AddInput(findOutputStream(ctx, ioswitch2.ECSrteam(i)))
if ctx.UseSegment {
// 先假设有所有的顺序分段,生成Join指令,后续根据Range再实际计算是否缺少流
joinNode := ctx.DAG.NewSegmentJoin(ctx.Ft.SegmentParam.Segments)
for i := 0; i < ctx.Ft.SegmentParam.SegmentCount(); i++ {
str := findOutputStream(ctx, ioswitch2.SegmentStream(i))
if str != nil {
joinNode.SetInput(i, str)
}
}
ctx.TypedStreams = append(ctx.TypedStreams, TypedStream{
Stream: joinNode.Joined(),
StreamType: ioswitch2.RawStream(),
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: joinNode.Joined(),
StreamIndex: ioswitch2.RawStream(),
})

// SegmentJoin生成的Join指令可以用来生成EC块
if ctx.UseEC {
splitNode := ctx.DAG.NewChunkedSplit(ctx.Ft.ECParam.ChunkSize)
splitNode.Split(joinNode.Joined(), ctx.Ft.ECParam.K)

mulNode := ctx.DAG.NewECMultiply(*ctx.Ft.ECParam)

for i := 0; i < ctx.Ft.ECParam.K; i++ {
mulNode.AddInput(splitNode.SubStream(i), i)
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: splitNode.SubStream(i),
StreamIndex: ioswitch2.ECSrteam(i),
})
}

for i := 0; i < ctx.Ft.ECParam.N; i++ {
ctx.IndexedStreams = append(ctx.IndexedStreams, IndexedStream{
Stream: mulNode.NewOutput(i),
StreamIndex: ioswitch2.ECSrteam(i),
})
}
}
}

// 为每一个To找到一个输入流
@@ -200,9 +322,9 @@ func extend(ctx *ParseContext) error {
}
ctx.ToNodes[to] = toNode

str := findOutputStream(ctx, to.GetStreamType())
str := findOutputStream(ctx, to.GetStreamIndex())
if str == nil {
return fmt.Errorf("no output stream found for data index %d", to.GetStreamType())
return fmt.Errorf("no output stream found for data index %d", to.GetStreamIndex())
}

toNode.SetInput(str)
@@ -213,26 +335,48 @@ func extend(ctx *ParseContext) error {

func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) {
var repRange exec.Range
var blkRange exec.Range

repRange.Offset = ctx.StreamRange.Offset
blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize)
if ctx.StreamRange.Length != nil {
repRngLen := *ctx.StreamRange.Length
repRange.Length = &repRngLen
}

blkRngLen := *ctx.StreamRange.Length / int64(ctx.EC.ChunkSize*ctx.EC.K) * int64(ctx.EC.ChunkSize)
blkRange.Length = &blkRngLen
var blkRange exec.Range
if ctx.UseEC {
blkRange.Offset = ctx.StreamRange.Offset / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize)
if ctx.StreamRange.Length != nil {
blkRngLen := *ctx.StreamRange.Length / int64(ctx.Ft.ECParam.ChunkSize*ctx.Ft.ECParam.K) * int64(ctx.Ft.ECParam.ChunkSize)
blkRange.Length = &blkRngLen
}
}

switch f := f.(type) {
case *ioswitch2.FromShardstore:
t := ctx.DAG.NewShardRead(f.Storage.StorageID, types.NewOpen(f.FileHash))
t := ctx.DAG.NewShardRead(f, f.Storage.StorageID, types.NewOpen(f.FileHash))

if f.StreamType.IsRaw() {
if f.StreamIndex.IsRaw() {
t.Open.WithNullableLength(repRange.Offset, repRange.Length)
} else {
} else if f.StreamIndex.IsEC() {
t.Open.WithNullableLength(blkRange.Offset, blkRange.Length)
} else if f.StreamIndex.IsSegment() {
segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index)
segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index]
segEnd := segStart + segLen

// 打开的范围不超过本段的范围

openOff := ctx.StreamRange.Offset - segStart
openOff = math2.Clamp(openOff, 0, segLen)

openLen := segLen

if ctx.StreamRange.Length != nil {
strEnd := ctx.StreamRange.Offset + *ctx.StreamRange.Length
openEnd := math2.Min(strEnd, segEnd)
openLen = openEnd - segStart - openOff
}

t.Open.WithNullableLength(openOff, &openLen)
}

switch addr := f.Hub.Address.(type) {
@@ -251,16 +395,36 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) {
return t, nil

case *ioswitch2.FromDriver:
n := ctx.DAG.NewFromDriver(f.Handle)
n := ctx.DAG.NewFromDriver(f, f.Handle)
n.Env().ToEnvDriver()
n.Env().Pinned = true

if f.StreamType.IsRaw() {
if f.StreamIndex.IsRaw() {
f.Handle.RangeHint.Offset = repRange.Offset
f.Handle.RangeHint.Length = repRange.Length
} else {
} else if f.StreamIndex.IsEC() {
f.Handle.RangeHint.Offset = blkRange.Offset
f.Handle.RangeHint.Length = blkRange.Length
} else if f.StreamIndex.IsSegment() {
segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index)
segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index]
segEnd := segStart + segLen

// 打开的范围不超过本段的范围

openOff := repRange.Offset - segStart
openOff = math2.Clamp(openOff, 0, segLen)

openLen := segLen

if repRange.Length != nil {
repEnd := repRange.Offset + *repRange.Length
openEnd := math2.Min(repEnd, segEnd)
openLen = openEnd - openOff
}

f.Handle.RangeHint.Offset = openOff
f.Handle.RangeHint.Length = &openLen
}

return n, nil
@@ -273,7 +437,7 @@ func buildFromNode(ctx *ParseContext, f ioswitch2.From) (ops2.FromNode, error) {
func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) {
switch t := t.(type) {
case *ioswitch2.ToShardStore:
n := ctx.DAG.NewShardWrite(t.Storage.StorageID, t.FileHashStoreKey)
n := ctx.DAG.NewShardWrite(t, t.Storage.StorageID, t.FileHashStoreKey)

if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil {
return nil, err
@@ -284,14 +448,14 @@ func buildToNode(ctx *ParseContext, t ioswitch2.To) (ops2.ToNode, error) {
return n, nil

case *ioswitch2.ToDriver:
n := ctx.DAG.NewToDriver(t.Handle)
n := ctx.DAG.NewToDriver(t, t.Handle)
n.Env().ToEnvDriver()
n.Env().Pinned = true

return n, nil

case *ioswitch2.LoadToShared:
n := ctx.DAG.NewSharedLoad(t.Storage.StorageID, t.UserID, t.PackageID, t.Path)
n := ctx.DAG.NewSharedLoad(t, t.Storage.StorageID, t.UserID, t.PackageID, t.Path)

if err := setEnvByAddress(n, t.Hub, t.Hub.Address); err != nil {
return nil, err
@@ -321,6 +485,34 @@ func setEnvByAddress(n dag.Node, hub cdssdk.Hub, addr cdssdk.HubAddressInfo) err
return nil
}

// 从SegmentJoin中删除未使用的分段
func removeUnusedSegment(ctx *ParseContext) error {
var err error
dag.WalkOnlyType[*ops2.SegmentJoinNode](ctx.DAG.Graph, func(node *ops2.SegmentJoinNode) bool {
start := ctx.StreamRange.Offset
var end *int64
if ctx.StreamRange.Length != nil {
e := ctx.StreamRange.Offset + *ctx.StreamRange.Length
end = &e
}

segStart, segEnd := ctx.Ft.SegmentParam.CalcSegmentRange(start, end)

node.MarkUsed(segStart, segEnd)

for i := segStart; i < segEnd; i++ {
if node.InputStreams().Get(i) == nil {
err = fmt.Errorf("segment %v missed to join an raw stream", i)
return false
}
}

return true
})

return err
}

// 删除输出流未被使用的Join指令
func removeUnusedJoin(ctx *ParseContext) bool {
changed := false
@@ -352,6 +544,8 @@ func removeUnusedMultiplyOutput(ctx *ParseContext) bool {
node.OutputIndexes[i2] = -2
changed = true
}

// TODO2 没有修改SlotIndex
node.OutputStreams().SetRawArray(lo2.RemoveAllDefault(outArr))
node.OutputIndexes = lo2.RemoveAll(node.OutputIndexes, -2)

@@ -504,6 +698,20 @@ func pin(ctx *ParseContext) bool {
return changed
}

// 删除未使用的From流,不会删除FromDriver
func removeUnusedFromNode(ctx *ParseContext) {
dag.WalkOnlyType[ops2.FromNode](ctx.DAG.Graph, func(node ops2.FromNode) bool {
if _, ok := node.(*ops2.FromDriverNode); ok {
return true
}

if node.Output().Var == nil {
ctx.DAG.RemoveNode(node)
}
return true
})
}

// 对于所有未使用的流,增加Drop指令
func dropUnused(ctx *ParseContext) {
ctx.DAG.Walk(func(node dag.Node) bool {
@@ -539,10 +747,10 @@ func generateRange(ctx *ParseContext) {
to := ctx.Ft.Toes[i]
toNode := ctx.ToNodes[to]

toStrType := to.GetStreamType()
toStrIdx := to.GetStreamIndex()
toRng := to.GetRange()

if toStrType.IsRaw() {
if toStrIdx.IsRaw() {
n := ctx.DAG.NewRange()
toInput := toNode.Input()
*n.Env() = *toInput.Var.From().Node.Env()
@@ -553,11 +761,11 @@ func generateRange(ctx *ParseContext) {
toInput.Var.StreamNotTo(toNode, toInput.Index)
toNode.SetInput(rnged)

} else {
stripSize := int64(ctx.EC.ChunkSize * ctx.EC.K)
} else if toStrIdx.IsEC() {
stripSize := int64(ctx.Ft.ECParam.ChunkSize * ctx.Ft.ECParam.K)
blkStartIdx := ctx.StreamRange.Offset / stripSize

blkStart := blkStartIdx * int64(ctx.EC.ChunkSize)
blkStart := blkStartIdx * int64(ctx.Ft.ECParam.ChunkSize)

n := ctx.DAG.NewRange()
toInput := toNode.Input()
@@ -568,6 +776,26 @@ func generateRange(ctx *ParseContext) {
})
toInput.Var.StreamNotTo(toNode, toInput.Index)
toNode.SetInput(rnged)
} else if toStrIdx.IsSegment() {
// if frNode, ok := toNode.Input().Var.From().Node.(ops2.FromNode); ok {
// // 目前只有To也是分段时,才可能对接一个提供分段的From,此时不需要再生成Range指令
// if frNode.GetFrom().GetStreamIndex().IsSegment() {
// continue
// }
// }

// segStart := ctx.Ft.SegmentParam.CalcSegmentStart(toStrIdx.Index)
// strStart := segStart + toRng.Offset

// n := ctx.DAG.NewRange()
// toInput := toNode.Input()
// *n.Env() = *toInput.Var.From().Node.Env()
// rnged := n.RangeStream(toInput.Var, exec.Range{
// Offset: strStart - ctx.StreamRange.Offset,
// Length: toRng.Length,
// })
// toInput.Var.StreamNotTo(toNode, toInput.Index)
// toNode.SetInput(rnged)
}
}
}


+ 2
- 2
common/pkgs/uploader/create_load.go View File

@@ -39,7 +39,7 @@ type CreateLoadResult struct {
func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) error {
uploadTime := time.Now()

ft := ioswitch2.NewFromTo()
ft := ioswitch2.FromTo{}
fromExec, hd := ioswitch2.NewFromDriver(ioswitch2.RawStream())
ft.AddFrom(fromExec)
for _, stg := range u.targetStgs {
@@ -48,7 +48,7 @@ func (u *CreateLoadUploader) Upload(path string, size int64, stream io.Reader) e
}

plans := exec.NewPlanBuilder()
err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy)
err := parser.Parse(ft, plans)
if err != nil {
return fmt.Errorf("parsing plan: %w", err)
}


+ 1
- 1
common/pkgs/uploader/update.go View File

@@ -47,7 +47,7 @@ func (w *UpdateUploader) Upload(path string, size int64, stream io.Reader) error
ft.AddFrom(fromExec).AddTo(ioswitch2.NewToShardStore(*w.targetStg.MasterHub, w.targetStg.Storage, ioswitch2.RawStream(), "fileHash"))

plans := exec.NewPlanBuilder()
err := parser.Parse(ft, plans, cdssdk.DefaultECRedundancy)
err := parser.Parse(ft, plans)
if err != nil {
return fmt.Errorf("parsing plan: %w", err)
}


+ 158
- 101
scanner/internal/event/check_package_redundancy.go View File

@@ -142,43 +142,47 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
switch newRed := newRed.(type) {
case *cdssdk.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs)
updating, err = t.noneToRep(execCtx, obj, newRed, newRepStgs, userAllStorages)

case *cdssdk.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs)
updating, err = t.noneToEC(execCtx, obj, newRed, newECStgs, userAllStorages)

case *cdssdk.LRCRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages)
updating, err = t.noneToLRC(execCtx, obj, newRed, selectedStorages, userAllStorages)

case *cdssdk.SegmentRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
updating, err = t.noneToSeg(execCtx, obj, newRed, selectedStorages, userAllStorages)
}

case *cdssdk.RepRedundancy:
switch newRed := newRed.(type) {
case *cdssdk.RepRedundancy:
updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs)
updating, err = t.repToRep(execCtx, obj, srcRed, rechoosedRepStgs, userAllStorages)

case *cdssdk.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
updating, err = t.repToEC(execCtx, obj, newRed, newECStgs)
updating, err = t.repToEC(execCtx, obj, newRed, newECStgs, userAllStorages)
}

case *cdssdk.ECRedundancy:
switch newRed := newRed.(type) {
case *cdssdk.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs)
updating, err = t.ecToRep(execCtx, obj, srcRed, newRed, newRepStgs, userAllStorages)

case *cdssdk.ECRedundancy:
uploadStorages := t.rechooseStoragesForEC(obj, srcRed, userAllStorages)
updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages)
updating, err = t.ecToEC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages)
}

case *cdssdk.LRCRedundancy:
switch newRed := newRed.(type) {
case *cdssdk.LRCRedundancy:
uploadStorages := t.rechooseStoragesForLRC(obj, srcRed, userAllStorages)
updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages)
updating, err = t.lrcToLRC(execCtx, obj, srcRed, newRed, uploadStorages, userAllStorages)
}
}

@@ -205,11 +209,23 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
func (t *CheckPackageRedundancy) chooseRedundancy(obj stgmod.ObjectDetail, userAllStgs map[cdssdk.StorageID]*StorageLoadInfo) (cdssdk.Redundancy, []*StorageLoadInfo) {
switch obj.Object.Redundancy.(type) {
case *cdssdk.NoneRedundancy:
newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs)
return &cdssdk.DefaultECRedundancy, newStgs
if obj.Object.Size > config.Cfg().ECFileSizeThreshold {
newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs)
return &cdssdk.DefaultECRedundancy, newStgs
}

// newLRCStorages := t.chooseNewStoragesForLRC(&cdssdk.DefaultLRCRedundancy, userAllStorages)
// return &cdssdk.DefaultLRCRedundancy, newLRCStorages
return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs)

case *cdssdk.RepRedundancy:
if obj.Object.Size > config.Cfg().ECFileSizeThreshold {
newStgs := t.chooseNewStoragesForEC(&cdssdk.DefaultECRedundancy, userAllStgs)
return &cdssdk.DefaultECRedundancy, newStgs
}

case *cdssdk.ECRedundancy:
if obj.Object.Size <= config.Cfg().ECFileSizeThreshold {
return &cdssdk.DefaultRepRedundancy, t.chooseNewStoragesForRep(&cdssdk.DefaultRepRedundancy, userAllStgs)
}

case *cdssdk.LRCRedundancy:
newLRCStgs := t.rechooseStoragesForLRC(obj, &cdssdk.DefaultLRCRedundancy, userAllStgs)
@@ -278,6 +294,14 @@ func (t *CheckPackageRedundancy) chooseNewStoragesForLRC(red *cdssdk.LRCRedundan
return t.chooseSoManyStorages(red.N, sortedStorages)
}

func (t *CheckPackageRedundancy) chooseNewStoragesForSeg(segCount int, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo {
sortedStorages := sort2.Sort(lo.Values(allStgs), func(left *StorageLoadInfo, right *StorageLoadInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})

return t.chooseSoManyStorages(segCount, sortedStorages)
}

func (t *CheckPackageRedundancy) rechooseStoragesForRep(mostBlockStgIDs []cdssdk.StorageID, red *cdssdk.RepRedundancy, allStgs map[cdssdk.StorageID]*StorageLoadInfo) []*StorageLoadInfo {
type rechooseStorage struct {
*StorageLoadInfo
@@ -421,25 +445,16 @@ func (t *CheckPackageRedundancy) chooseSoManyStorages(count int, stgs []*Storage
return chosen
}

func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep")
}

coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID}))
if err != nil {
return nil, fmt.Errorf("requesting to get storages: %w", err)
}
if getStgs.Storages[0] == nil {
srcStg, ok := allStgs[obj.Blocks[0].StorageID]
if !ok {
return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID)
}
if getStgs.Storages[0].MasterHub == nil {
if srcStg.Storage.MasterHub == nil {
return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID)
}

@@ -447,13 +462,13 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object
uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID })

ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream()))
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream()))
for i, stg := range uploadStgs {
ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
}

plans := exec.NewPlanBuilder()
err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy)
err := parser.Parse(ft, plans)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -483,35 +498,27 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object
}, nil
}

func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec")
}

getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID}))
if err != nil {
return nil, fmt.Errorf("requesting to get storages: %w", err)
}
if getStgs.Storages[0] == nil {
srcStg, ok := allStgs[obj.Blocks[0].StorageID]
if !ok {
return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID)
}
if getStgs.Storages[0].MasterHub == nil {
if srcStg.Storage.MasterHub == nil {
return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID)
}

ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream()))
ft.ECParam = red
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream()))
for i := 0; i < red.N; i++ {
ft.AddTo(ioswitch2.NewToShardStore(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i)))
}
plans := exec.NewPlanBuilder()
err = parser.Parse(ft, plans, *red)
err := parser.Parse(ft, plans)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -540,25 +547,16 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD
}, nil
}

func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec")
}

getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID}))
if err != nil {
return nil, fmt.Errorf("requesting to get storages: %w", err)
}
if getStgs.Storages[0] == nil {
srcStg, ok := allStgs[obj.Blocks[0].StorageID]
if !ok {
return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID)
}
if getStgs.Storages[0].MasterHub == nil {
if srcStg.Storage.MasterHub == nil {
return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID)
}

@@ -568,7 +566,7 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object
}

plans := exec.NewPlanBuilder()
err = lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, -1), toes, plans)
err := lrcparser.Encode(ioswitchlrc.NewFromStorage(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, -1), toes, plans)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -597,25 +595,70 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object
}, nil
}

func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.SegmentRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep")
}

coorCli, err := stgglb.CoordinatorMQPool.Acquire()
srcStg, ok := allStgs[obj.Blocks[0].StorageID]
if !ok {
return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID)
}
if srcStg.Storage.MasterHub == nil {
return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID)
}

// 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID })

ft := ioswitch2.NewFromTo()
ft.SegmentParam = red
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream()))
for i, stg := range uploadStgs {
ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.SegmentStream(i), fmt.Sprintf("%d", i)))
}

plans := exec.NewPlanBuilder()
err := parser.Parse(ft, plans)
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
return nil, fmt.Errorf("parsing plan: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

getStgs, err := coorCli.GetStorageDetails(coormq.ReqGetStorageDetails([]cdssdk.StorageID{obj.Blocks[0].StorageID}))
// TODO 添加依赖
execCtx := exec.NewExecContext()
exec.SetValueByType(execCtx, ctx.Args.StgMgr)
ret, err := plans.Execute(execCtx).Wait(context.Background())
if err != nil {
return nil, fmt.Errorf("requesting to get storages: %w", err)
return nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []stgmod.ObjectBlock
for i, stg := range uploadStgs {
blocks = append(blocks, stgmod.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
StorageID: stg.Storage.Storage.StorageID,
FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash,
})
}

return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: red,
Blocks: blocks,
}, nil
}

func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to rep")
}
if getStgs.Storages[0] == nil {

srcStg, ok := allStgs[obj.Blocks[0].StorageID]
if !ok {
return nil, fmt.Errorf("storage %v not found", obj.Blocks[0].StorageID)
}
if getStgs.Storages[0].MasterHub == nil {
if srcStg.Storage.MasterHub == nil {
return nil, fmt.Errorf("storage %v has no master hub", obj.Blocks[0].StorageID)
}

@@ -623,13 +666,13 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD
uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID })

ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *getStgs.Storages[0].MasterHub, getStgs.Storages[0].Storage, ioswitch2.RawStream()))
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.Storage.MasterHub, srcStg.Storage.Storage, ioswitch2.RawStream()))
for i, stg := range uploadStgs {
ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i)))
}

plans := exec.NewPlanBuilder()
err = parser.Parse(ft, plans, cdssdk.DefaultECRedundancy)
err := parser.Parse(ft, plans)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -659,23 +702,28 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD
}, nil
}

func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
return t.noneToEC(ctx, obj, red, uploadStorages)
func (t *CheckPackageRedundancy) repToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
return t.noneToEC(ctx, obj, red, uploadStorages, allStgs)
}

func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
var chosenBlocks []stgmod.GrouppedObjectBlock
var chosenBlockIndexes []int
var chosenBlockStg []stgmod.StorageDetail
for _, block := range obj.GroupBlocks() {
if len(block.StorageIDs) > 0 {
// TODO 考虑选择最优的节点
stg, ok := allStgs[block.StorageIDs[0]]
if !ok {
continue
}
if stg.Storage.MasterHub == nil {
continue
}

chosenBlocks = append(chosenBlocks, block)
chosenBlockIndexes = append(chosenBlockIndexes, block.Index)
chosenBlockStg = append(chosenBlockStg, stg.Storage)
}

if len(chosenBlocks) == srcRed.K {
@@ -694,9 +742,10 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe
planBlder := exec.NewPlanBuilder()
for i := range uploadStgs {
ft := ioswitch2.NewFromTo()
ft.ECParam = srcRed

for _, block := range chosenBlocks {
ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, ioswitch2.ECSrteam(block.Index)))
for i2, block := range chosenBlocks {
ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index)))
}

len := obj.Object.Size
@@ -705,7 +754,7 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe
Length: &len,
}))

err := parser.Parse(ft, planBlder, *srcRed)
err := parser.Parse(ft, planBlder)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -736,19 +785,23 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe
}, nil
}

func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
grpBlocks := obj.GroupBlocks()

var chosenBlocks []stgmod.GrouppedObjectBlock
var chosenBlockStg []stgmod.StorageDetail
for _, block := range grpBlocks {
if len(block.StorageIDs) > 0 {
stg, ok := allStgs[block.StorageIDs[0]]
if !ok {
continue
}
if stg.Storage.MasterHub == nil {
continue
}

chosenBlocks = append(chosenBlocks, block)
chosenBlockStg = append(chosenBlockStg, stg.Storage)
}

if len(chosenBlocks) == srcRed.K {
@@ -786,15 +839,15 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet
// 否则就要重建出这个节点需要的块

ft := ioswitch2.NewFromTo()
for _, block := range chosenBlocks {
stg := stg.Storage
ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *stg.MasterHub, stg.Storage, ioswitch2.ECSrteam(block.Index)))
ft.ECParam = srcRed
for i2, block := range chosenBlocks {
ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, ioswitch2.ECSrteam(block.Index)))
}

// 输出只需要自己要保存的那一块
ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage.Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d", i)))

err := parser.Parse(ft, planBlder, *srcRed)
err := parser.Parse(ft, planBlder)
if err != nil {
return nil, fmt.Errorf("parsing plan: %w", err)
}
@@ -830,12 +883,7 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet
}, nil
}

func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)
func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, srcRed *cdssdk.LRCRedundancy, tarRed *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {

blocksGrpByIndex := obj.GroupBlocks()

@@ -870,7 +918,7 @@ func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectD
// return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadStorages)
}

return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages)
return t.reconstructLRC(ctx, obj, blocksGrpByIndex, srcRed, uploadStorages, allStgs)
}

/*
@@ -939,11 +987,22 @@ TODO2 修复这一块的代码
}, nil
}
*/
func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, grpBlocks []stgmod.GrouppedObjectBlock, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
var chosenBlocks []stgmod.GrouppedObjectBlock
var chosenBlockStg []stgmod.StorageDetail

for _, block := range grpBlocks {
if len(block.StorageIDs) > 0 && block.Index < red.M() {
stg, ok := allStgs[block.StorageIDs[0]]
if !ok {
continue
}
if stg.Storage.MasterHub == nil {
continue
}

chosenBlocks = append(chosenBlocks, block)
chosenBlockStg = append(chosenBlockStg, stg.Storage)
}

if len(chosenBlocks) == red.K {
@@ -982,10 +1041,8 @@ func (t *CheckPackageRedundancy) reconstructLRC(ctx ExecuteContext, obj stgmod.O

// 否则就要重建出这个节点需要的块

for _, block := range chosenBlocks {
fmt.Printf("b: %v\n", block.Index)
stg := storage.Storage
froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *stg.MasterHub, stg.Storage, block.Index))
for i2, block := range chosenBlocks {
froms = append(froms, ioswitchlrc.NewFromStorage(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2].Storage, block.Index))
}

// 输出只需要自己要保存的那一块


+ 3
- 2
scanner/internal/event/clean_pinned.go View File

@@ -745,7 +745,7 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st
toStg := allStgInfos[solu.blockList[i].StorageID]
ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, toStg.Storage, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID)))

err := parser.Parse(ft, planBld, cdssdk.DefaultECRedundancy)
err := parser.Parse(ft, planBld)
if err != nil {
// TODO 错误处理
continue
@@ -798,13 +798,14 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg

for id, idxs := range reconstrct {
ft := ioswitch2.NewFromTo()
ft.ECParam = ecRed
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.RawStream()))

for _, i := range *idxs {
ft.AddTo(ioswitch2.NewToShardStore(*allStgInfos[id].MasterHub, allStgInfos[id].Storage, ioswitch2.ECSrteam(i), fmt.Sprintf("%d.%d", obj.Object.ObjectID, i)))
}

err := parser.Parse(ft, planBld, *ecRed)
err := parser.Parse(ft, planBld)
if err != nil {
// TODO 错误处理
continue


Loading…
Cancel
Save