Browse Source

重构ioswitch

gitlink
Sydonian 1 year ago
parent
commit
aed2080163
24 changed files with 579 additions and 509 deletions
  1. +9
    -20
      agent/internal/grpc/io.go
  2. +83
    -53
      common/pkgs/grpc/agent/agent.pb.go
  3. +8
    -5
      common/pkgs/grpc/agent/agent.proto
  4. +20
    -27
      common/pkgs/grpc/agent/client.go
  5. +8
    -8
      common/pkgs/ioswitch2/agent_worker.go
  6. +8
    -9
      common/pkgs/ioswitch2/http_hub_worker.go
  7. +35
    -33
      common/pkgs/ioswitch2/ops2/chunked.go
  8. +28
    -30
      common/pkgs/ioswitch2/ops2/clone.go
  9. +84
    -83
      common/pkgs/ioswitch2/ops2/ec.go
  10. +21
    -20
      common/pkgs/ioswitch2/ops2/file.go
  11. +19
    -13
      common/pkgs/ioswitch2/ops2/join.go
  12. +16
    -11
      common/pkgs/ioswitch2/ops2/length.go
  13. +3
    -3
      common/pkgs/ioswitch2/ops2/ops.go
  14. +22
    -21
      common/pkgs/ioswitch2/ops2/range.go
  15. +37
    -27
      common/pkgs/ioswitch2/ops2/shard_store.go
  16. +4
    -4
      common/pkgs/ioswitch2/parser/parser.go
  17. +8
    -8
      common/pkgs/ioswitchlrc/agent_worker.go
  18. +56
    -33
      common/pkgs/ioswitchlrc/ops2/chunked.go
  19. +28
    -30
      common/pkgs/ioswitchlrc/ops2/clone.go
  20. +19
    -19
      common/pkgs/ioswitchlrc/ops2/ec.go
  21. +3
    -3
      common/pkgs/ioswitchlrc/ops2/ops.go
  22. +22
    -21
      common/pkgs/ioswitchlrc/ops2/range.go
  23. +37
    -27
      common/pkgs/ioswitchlrc/ops2/shard_store.go
  24. +1
    -1
      common/pkgs/ioswitchlrc/parser/generator.go

+ 9
- 20
agent/internal/grpc/io.go View File

@@ -66,10 +66,7 @@ func (s *Service) SendStream(server agtrpc.Agent_SendStreamServer) error {
pr, pw := io.Pipe() pr, pw := io.Pipe()


varID := exec.VarID(msg.VarID) varID := exec.VarID(msg.VarID)
sw.PutVars(&exec.StreamVar{
ID: varID,
Stream: pr,
})
sw.PutVar(varID, &exec.StreamValue{Stream: pr})


// 然后读取文件数据 // 然后读取文件数据
var recvSize int64 var recvSize int64
@@ -132,17 +129,14 @@ func (s *Service) GetStream(req *agtrpc.GetStreamReq, server agtrpc.Agent_GetStr
return fmt.Errorf("plan not found") return fmt.Errorf("plan not found")
} }


signal, err := serder.JSONToObjectEx[*exec.SignalVar]([]byte(req.Signal))
signal, err := serder.JSONToObjectEx[exec.VarValue]([]byte(req.Signal))
if err != nil { if err != nil {
return fmt.Errorf("deserializing var: %w", err) return fmt.Errorf("deserializing var: %w", err)
} }


sw.PutVars(signal)
sw.PutVar(exec.VarID(req.SignalID), signal)


strVar := &exec.StreamVar{
ID: exec.VarID(req.VarID),
}
err = sw.BindVars(server.Context(), strVar)
strVar, err := exec.BindVar[*exec.StreamValue](sw, server.Context(), exec.VarID(req.VarID))
if err != nil { if err != nil {
return fmt.Errorf("binding vars: %w", err) return fmt.Errorf("binding vars: %w", err)
} }
@@ -205,12 +199,12 @@ func (s *Service) SendVar(ctx context.Context, req *agtrpc.SendVarReq) (*agtrpc.
return nil, fmt.Errorf("plan not found") return nil, fmt.Errorf("plan not found")
} }


v, err := serder.JSONToObjectEx[exec.Var]([]byte(req.Var))
v, err := serder.JSONToObjectEx[exec.VarValue]([]byte(req.VarValue))
if err != nil { if err != nil {
return nil, fmt.Errorf("deserializing var: %w", err) return nil, fmt.Errorf("deserializing var: %w", err)
} }


sw.PutVars(v)
sw.PutVar(exec.VarID(req.VarID), v)
return &agtrpc.SendVarResp{}, nil return &agtrpc.SendVarResp{}, nil
} }


@@ -223,19 +217,14 @@ func (s *Service) GetVar(ctx context.Context, req *agtrpc.GetVarReq) (*agtrpc.Ge
return nil, fmt.Errorf("plan not found") return nil, fmt.Errorf("plan not found")
} }


v, err := serder.JSONToObjectEx[exec.Var]([]byte(req.Var))
if err != nil {
return nil, fmt.Errorf("deserializing var: %w", err)
}

signal, err := serder.JSONToObjectEx[*exec.SignalVar]([]byte(req.Signal))
signal, err := serder.JSONToObjectEx[exec.VarValue]([]byte(req.Signal))
if err != nil { if err != nil {
return nil, fmt.Errorf("deserializing var: %w", err) return nil, fmt.Errorf("deserializing var: %w", err)
} }


sw.PutVars(signal)
sw.PutVar(exec.VarID(req.SignalID), signal)


err = sw.BindVars(ctx, v)
v, err := sw.BindVar(ctx, exec.VarID(req.VarID))
if err != nil { if err != nil {
return nil, fmt.Errorf("binding vars: %w", err) return nil, fmt.Errorf("binding vars: %w", err)
} }


+ 83
- 53
common/pkgs/grpc/agent/agent.pb.go View File

@@ -327,9 +327,10 @@ type GetStreamReq struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields


PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
VarID int32 `protobuf:"varint,2,opt,name=VarID,proto3" json:"VarID,omitempty"`
Signal string `protobuf:"bytes,3,opt,name=Signal,proto3" json:"Signal,omitempty"`
PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
VarID int32 `protobuf:"varint,2,opt,name=VarID,proto3" json:"VarID,omitempty"`
SignalID int32 `protobuf:"varint,3,opt,name=SignalID,proto3" json:"SignalID,omitempty"`
Signal string `protobuf:"bytes,4,opt,name=Signal,proto3" json:"Signal,omitempty"`
} }


func (x *GetStreamReq) Reset() { func (x *GetStreamReq) Reset() {
@@ -378,6 +379,13 @@ func (x *GetStreamReq) GetVarID() int32 {
return 0 return 0
} }


func (x *GetStreamReq) GetSignalID() int32 {
if x != nil {
return x.SignalID
}
return 0
}

func (x *GetStreamReq) GetSignal() string { func (x *GetStreamReq) GetSignal() string {
if x != nil { if x != nil {
return x.Signal return x.Signal
@@ -390,8 +398,9 @@ type SendVarReq struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields


PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
Var string `protobuf:"bytes,2,opt,name=Var,proto3" json:"Var,omitempty"`
PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
VarID int32 `protobuf:"varint,2,opt,name=VarID,proto3" json:"VarID,omitempty"`
VarValue string `protobuf:"bytes,3,opt,name=VarValue,proto3" json:"VarValue,omitempty"`
} }


func (x *SendVarReq) Reset() { func (x *SendVarReq) Reset() {
@@ -433,9 +442,16 @@ func (x *SendVarReq) GetPlanID() string {
return "" return ""
} }


func (x *SendVarReq) GetVar() string {
func (x *SendVarReq) GetVarID() int32 {
if x != nil { if x != nil {
return x.Var
return x.VarID
}
return 0
}

func (x *SendVarReq) GetVarValue() string {
if x != nil {
return x.VarValue
} }
return "" return ""
} }
@@ -483,9 +499,10 @@ type GetVarReq struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields


PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
Var string `protobuf:"bytes,2,opt,name=Var,proto3" json:"Var,omitempty"`
Signal string `protobuf:"bytes,3,opt,name=Signal,proto3" json:"Signal,omitempty"`
PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
VarID int32 `protobuf:"varint,2,opt,name=VarID,proto3" json:"VarID,omitempty"`
SignalID int32 `protobuf:"varint,3,opt,name=SignalID,proto3" json:"SignalID,omitempty"`
Signal string `protobuf:"bytes,4,opt,name=Signal,proto3" json:"Signal,omitempty"`
} }


func (x *GetVarReq) Reset() { func (x *GetVarReq) Reset() {
@@ -527,11 +544,18 @@ func (x *GetVarReq) GetPlanID() string {
return "" return ""
} }


func (x *GetVarReq) GetVar() string {
func (x *GetVarReq) GetVarID() int32 {
if x != nil { if x != nil {
return x.Var
return x.VarID
} }
return ""
return 0
}

func (x *GetVarReq) GetSignalID() int32 {
if x != nil {
return x.SignalID
}
return 0
} }


func (x *GetVarReq) GetSignal() string { func (x *GetVarReq) GetSignal() string {
@@ -546,7 +570,7 @@ type GetVarResp struct {
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields unknownFields protoimpl.UnknownFields


Var string `protobuf:"bytes,1,opt,name=Var,proto3" json:"Var,omitempty"` // 此处不使用VarID的原因是,Switch的BindVars函数还需要知道Var的类型
Var string `protobuf:"bytes,1,opt,name=Var,proto3" json:"Var,omitempty"`
} }


func (x *GetVarResp) Reset() { func (x *GetVarResp) Reset() {
@@ -686,48 +710,54 @@ var file_pkgs_grpc_agent_agent_proto_rawDesc = []byte{
0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28,
0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61,
0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x22, 0x10, 0x0a, 0x0e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x22, 0x10, 0x0a, 0x0e,
0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x54,
0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x70,
0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x12, 0x16,
0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18,
0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69,
0x67, 0x6e, 0x61, 0x6c, 0x22, 0x36, 0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52,
0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x61,
0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56, 0x61, 0x72, 0x22, 0x0d, 0x0a, 0x0b,
0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x4d, 0x0a, 0x09, 0x47,
0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e,
0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44,
0x12, 0x10, 0x0a, 0x03, 0x56, 0x61, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56,
0x61, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01,
0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x1e, 0x0a, 0x0a, 0x47, 0x65,
0x74, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x61, 0x72, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56, 0x61, 0x72, 0x22, 0x09, 0x0a, 0x07, 0x50, 0x69,
0x6e, 0x67, 0x52, 0x65, 0x71, 0x22, 0x0a, 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73,
0x70, 0x2a, 0x37, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50,
0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x45, 0x4f, 0x46,
0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08,
0x53, 0x65, 0x6e, 0x64, 0x41, 0x72, 0x67, 0x73, 0x10, 0x02, 0x32, 0x96, 0x02, 0x0a, 0x05, 0x41,
0x67, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x0d, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49,
0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x12, 0x11, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49,
0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75,
0x74, 0x65, 0x49, 0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34,
0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x11, 0x2e, 0x53,
0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a,
0x0f, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70,
0x22, 0x00, 0x28, 0x01, 0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61,
0x6d, 0x12, 0x0d, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71,
0x1a, 0x11, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63,
0x6b, 0x65, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x26, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x56,
0x61, 0x72, 0x12, 0x0b, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x1a,
0x0c, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12,
0x23, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x12, 0x0a, 0x2e, 0x47, 0x65, 0x74, 0x56,
0x61, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0b, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65,
0x73, 0x70, 0x22, 0x00, 0x12, 0x1d, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x08, 0x2e, 0x50,
0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x1a, 0x09, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73,
0x70, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e,
0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c,
0x22, 0x56, 0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x12, 0x16,
0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18,
0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08,
0x56, 0x61, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
0x56, 0x61, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x0d, 0x0a, 0x0b, 0x53, 0x65, 0x6e, 0x64,
0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x6d, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x56, 0x61,
0x72, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05,
0x56, 0x61, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72,
0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x49, 0x44, 0x18, 0x03,
0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x49, 0x44, 0x12, 0x16,
0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x1e, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72,
0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x61, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x03, 0x56, 0x61, 0x72, 0x22, 0x09, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65,
0x71, 0x22, 0x0a, 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x2a, 0x37, 0x0a,
0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65,
0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x45, 0x4f, 0x46, 0x10, 0x00, 0x12, 0x08,
0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x65, 0x6e, 0x64,
0x41, 0x72, 0x67, 0x73, 0x10, 0x02, 0x32, 0x96, 0x02, 0x0a, 0x05, 0x41, 0x67, 0x65, 0x6e, 0x74,
0x12, 0x38, 0x0a, 0x0d, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49, 0x4f, 0x50, 0x6c, 0x61,
0x6e, 0x12, 0x11, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49, 0x4f, 0x50, 0x6c, 0x61,
0x6e, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49, 0x4f,
0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34, 0x0a, 0x0a, 0x53, 0x65,
0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x11, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61,
0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x0f, 0x2e, 0x53, 0x65,
0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x28, 0x01,
0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x0d, 0x2e,
0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x1a, 0x11, 0x2e, 0x53,
0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22,
0x00, 0x30, 0x01, 0x12, 0x26, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x12, 0x0b,
0x2e, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0c, 0x2e, 0x53, 0x65,
0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x23, 0x0a, 0x06, 0x47,
0x65, 0x74, 0x56, 0x61, 0x72, 0x12, 0x0a, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65,
0x71, 0x1a, 0x0b, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00,
0x12, 0x1d, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x08, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52,
0x65, 0x71, 0x1a, 0x09, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42,
0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
} }


var ( var (


+ 8
- 5
common/pkgs/grpc/agent/agent.proto View File

@@ -37,22 +37,25 @@ message SendStreamResp {}
message GetStreamReq { message GetStreamReq {
string PlanID = 1; string PlanID = 1;
int32 VarID = 2; int32 VarID = 2;
string Signal = 3;
int32 SignalID = 3;
string Signal = 4;
} }


message SendVarReq { message SendVarReq {
string PlanID = 1; string PlanID = 1;
string Var = 2;
int32 VarID = 2;
string VarValue = 3;
} }
message SendVarResp {} message SendVarResp {}


message GetVarReq { message GetVarReq {
string PlanID = 1; string PlanID = 1;
string Var = 2;
string Signal = 3;
int32 VarID = 2;
int32 SignalID = 3;
string Signal = 4;
} }
message GetVarResp { message GetVarResp {
string Var = 1; // 此处不使用VarID的原因是,Switch的BindVars函数还需要知道Var的类型
string Var = 1;
} }


message PingReq {} message PingReq {}


+ 20
- 27
common/pkgs/grpc/agent/client.go View File

@@ -132,7 +132,7 @@ func (c *Client) SendStream(ctx context.Context, planID exec.PlanID, varID exec.
} }
} }


func (c *Client) GetStream(ctx context.Context, planID exec.PlanID, varID exec.VarID, signal *exec.SignalVar) (io.ReadCloser, error) {
func (c *Client) GetStream(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) {
ctx, cancel := context.WithCancel(ctx) ctx, cancel := context.WithCancel(ctx)


sdata, err := serder.ObjectToJSONEx(signal) sdata, err := serder.ObjectToJSONEx(signal)
@@ -142,9 +142,10 @@ func (c *Client) GetStream(ctx context.Context, planID exec.PlanID, varID exec.V
} }


stream, err := c.cli.GetStream(ctx, &GetStreamReq{ stream, err := c.cli.GetStream(ctx, &GetStreamReq{
PlanID: string(planID),
VarID: int32(varID),
Signal: string(sdata),
PlanID: string(planID),
VarID: int32(varID),
SignalID: int32(signalID),
Signal: string(sdata),
}) })
if err != nil { if err != nil {
cancel() cancel()
@@ -157,50 +158,42 @@ func (c *Client) GetStream(ctx context.Context, planID exec.PlanID, varID exec.V
}, nil }, nil
} }


func (c *Client) SendVar(ctx context.Context, planID exec.PlanID, v exec.Var) error {
data, err := serder.ObjectToJSONEx(v)
func (c *Client) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error {
data, err := serder.ObjectToJSONEx(value)
if err != nil { if err != nil {
return err return err
} }


_, err = c.cli.SendVar(ctx, &SendVarReq{ _, err = c.cli.SendVar(ctx, &SendVarReq{
PlanID: string(planID),
Var: string(data),
PlanID: string(planID),
VarID: int32(id),
VarValue: string(data),
}) })
return err return err
} }


func (c *Client) GetVar(ctx context.Context, planID exec.PlanID, v exec.Var, signal *exec.SignalVar) error {
vdata, err := serder.ObjectToJSONEx(v)
if err != nil {
return err
}

func (c *Client) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) {
sdata, err := serder.ObjectToJSONEx(signal) sdata, err := serder.ObjectToJSONEx(signal)
if err != nil { if err != nil {
return err
return nil, err
} }


resp, err := c.cli.GetVar(ctx, &GetVarReq{ resp, err := c.cli.GetVar(ctx, &GetVarReq{
PlanID: string(planID),
Var: string(vdata),
Signal: string(sdata),
PlanID: string(planID),
VarID: int32(varID),
SignalID: int32(signalID),
Signal: string(sdata),
}) })
if err != nil { if err != nil {
return err
}

v2, err := serder.JSONToObjectEx[exec.Var]([]byte(resp.Var))
if err != nil {
return err
return nil, err
} }


err = exec.AssignVar(v2, v)
getVar, err := serder.JSONToObjectEx[exec.VarValue]([]byte(resp.Var))
if err != nil { if err != nil {
return err
return nil, err
} }


return nil
return getVar, nil
} }


func (c *Client) Ping() error { func (c *Client) Ping() error {


+ 8
- 8
common/pkgs/ioswitch2/agent_worker.go View File

@@ -49,17 +49,17 @@ type AgentWorkerClient struct {
func (c *AgentWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error { func (c *AgentWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error {
return c.cli.ExecuteIOPlan(ctx, plan) return c.cli.ExecuteIOPlan(ctx, plan)
} }
func (c *AgentWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, str io.ReadCloser) error {
return c.cli.SendStream(ctx, planID, v.ID, str)
func (c *AgentWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, id exec.VarID, stream io.ReadCloser) error {
return c.cli.SendStream(ctx, planID, id, stream)
} }
func (c *AgentWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, v exec.Var) error {
return c.cli.SendVar(ctx, planID, v)
func (c *AgentWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error {
return c.cli.SendVar(ctx, planID, id, value)
} }
func (c *AgentWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, signal *exec.SignalVar) (io.ReadCloser, error) {
return c.cli.GetStream(ctx, planID, v.ID, signal)
func (c *AgentWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, streamID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) {
return c.cli.GetStream(ctx, planID, streamID, signalID, signal)
} }
func (c *AgentWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, v exec.Var, signal *exec.SignalVar) error {
return c.cli.GetVar(ctx, planID, v, signal)
func (c *AgentWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) {
return c.cli.GetVar(ctx, planID, varID, signalID, signal)
} }
func (c *AgentWorkerClient) Close() error { func (c *AgentWorkerClient) Close() error {
stgglb.AgentRPCPool.Release(c.cli) stgglb.AgentRPCPool.Release(c.cli)


+ 8
- 9
common/pkgs/ioswitch2/http_hub_worker.go View File

@@ -56,18 +56,17 @@ type HttpHubWorkerClient struct {
func (c *HttpHubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error { func (c *HttpHubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error {
return c.cli.ExecuteIOPlan(plan) return c.cli.ExecuteIOPlan(plan)
} }
func (c *HttpHubWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, str io.ReadCloser) error {
return c.cli.SendStream(planID, v.ID, str)
func (c *HttpHubWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, id exec.VarID, stream io.ReadCloser) error {
return c.cli.SendStream(planID, id, stream)
} }
func (c *HttpHubWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, v exec.Var) error {
return c.cli.SendVar(planID, v)
func (c *HttpHubWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error {
return c.cli.SendVar(planID, id, value)
} }
func (c *HttpHubWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, signal *exec.SignalVar) (io.ReadCloser, error) {
return c.cli.GetStream(planID, v.ID, signal)
func (c *HttpHubWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, streamID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) {
return c.cli.GetStream(planID, streamID, signalID, signal)
} }
func (c *HttpHubWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, v exec.Var, signal *exec.SignalVar) error {
return c.cli.GetVar(planID, v, signal)
//return nil
func (c *HttpHubWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) {
return c.cli.GetVar(planID, varID, signalID, signal)
} }
func (c *HttpHubWorkerClient) Close() error { func (c *HttpHubWorkerClient) Close() error {
//stgglb.AgentRPCPool.Release(c.cli) //stgglb.AgentRPCPool.Release(c.cli)


+ 35
- 33
common/pkgs/ioswitch2/ops2/chunked.go View File

@@ -19,20 +19,20 @@ func init() {
} }


type ChunkedSplit struct { type ChunkedSplit struct {
Input *exec.StreamVar `json:"input"`
Outputs []*exec.StreamVar `json:"outputs"`
ChunkSize int `json:"chunkSize"`
PaddingZeros bool `json:"paddingZeros"`
Input exec.VarID `json:"input"`
Outputs []exec.VarID `json:"outputs"`
ChunkSize int `json:"chunkSize"`
PaddingZeros bool `json:"paddingZeros"`
} }


func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


outputs := io2.ChunkedSplit(o.Input.Stream, o.ChunkSize, len(o.Outputs), io2.ChunkedSplitOption{
outputs := io2.ChunkedSplit(input.Stream, o.ChunkSize, len(o.Outputs), io2.ChunkedSplitOption{
PaddingZeros: o.PaddingZeros, PaddingZeros: o.PaddingZeros,
}) })


@@ -40,11 +40,12 @@ func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
for i := range outputs { for i := range outputs {
sem.Acquire(ctx.Context, 1) sem.Acquire(ctx.Context, 1)


o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
e.PutVar(o.Outputs[i], &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
}),
}) })
} }
exec.PutArrayVars(e, o.Outputs)


return sem.Acquire(ctx.Context, int64(len(outputs))) return sem.Acquire(ctx.Context, int64(len(outputs)))
} }
@@ -54,38 +55,39 @@ func (o *ChunkedSplit) String() string {
"ChunkedSplit(chunkSize=%v, paddingZeros=%v), %v -> (%v)", "ChunkedSplit(chunkSize=%v, paddingZeros=%v), %v -> (%v)",
o.ChunkSize, o.ChunkSize,
o.PaddingZeros, o.PaddingZeros,
o.Input.ID,
o.Input,
utils.FormatVarIDs(o.Outputs), utils.FormatVarIDs(o.Outputs),
) )
} }


type ChunkedJoin struct { type ChunkedJoin struct {
Inputs []*exec.StreamVar `json:"inputs"`
Output *exec.StreamVar `json:"output"`
ChunkSize int `json:"chunkSize"`
Inputs []exec.VarID `json:"inputs"`
Output exec.VarID `json:"output"`
ChunkSize int `json:"chunkSize"`
} }


func (o *ChunkedJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ChunkedJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := exec.BindArrayVars(e, ctx.Context, o.Inputs)
inputs, err := exec.BindArray[*exec.StreamValue](e, ctx.Context, o.Inputs)
if err != nil { if err != nil {
return err return err
} }


var strReaders []io.Reader var strReaders []io.Reader
for _, s := range o.Inputs {
for _, s := range inputs {
strReaders = append(strReaders, s.Stream) strReaders = append(strReaders, s.Stream)
} }
defer func() { defer func() {
for _, str := range o.Inputs {
for _, str := range inputs {
str.Stream.Close() str.Stream.Close()
} }
}() }()


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(io2.BufferedChunkedJoin(strReaders, o.ChunkSize), func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(io2.BufferedChunkedJoin(strReaders, o.ChunkSize), func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }
@@ -95,7 +97,7 @@ func (o *ChunkedJoin) String() string {
"ChunkedJoin(chunkSize=%v), (%v) -> %v", "ChunkedJoin(chunkSize=%v), (%v) -> %v",
o.ChunkSize, o.ChunkSize,
utils.FormatVarIDs(o.Inputs), utils.FormatVarIDs(o.Inputs),
o.Output.ID,
o.Output,
) )
} }


@@ -112,16 +114,16 @@ func (b *GraphNodeBuilder) NewChunkedSplit(chunkSize int) *ChunkedSplitNode {
return node return node
} }


func (t *ChunkedSplitNode) Split(input *dag.StreamVar, cnt int) {
func (t *ChunkedSplitNode) Split(input *dag.Var, cnt int) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.OutputStreams().Resize(cnt) t.OutputStreams().Resize(cnt)
for i := 0; i < cnt; i++ { for i := 0; i < cnt; i++ {
t.OutputStreams().Setup(t, t.Graph().NewStreamVar(), i)
t.OutputStreams().Setup(t, t.Graph().NewVar(), i)
} }
} }


func (t *ChunkedSplitNode) SubStream(idx int) *dag.StreamVar {
func (t *ChunkedSplitNode) SubStream(idx int) *dag.Var {
return t.OutputStreams().Get(idx) return t.OutputStreams().Get(idx)
} }


@@ -145,9 +147,9 @@ func (t *ChunkedSplitNode) Clear() {


func (t *ChunkedSplitNode) GenerateOp() (exec.Op, error) { func (t *ChunkedSplitNode) GenerateOp() (exec.Op, error) {
return &ChunkedSplit{ return &ChunkedSplit{
Input: t.InputStreams().Get(0).Var,
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Input: t.InputStreams().Get(0).VarID,
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
ChunkSize: t.ChunkSize, ChunkSize: t.ChunkSize,
PaddingZeros: true, PaddingZeros: true,
@@ -168,16 +170,16 @@ func (b *GraphNodeBuilder) NewChunkedJoin(chunkSize int) *ChunkedJoinNode {
ChunkSize: chunkSize, ChunkSize: chunkSize,
} }
b.AddNode(node) b.AddNode(node)
node.OutputStreams().SetupNew(node, b.Graph.NewStreamVar())
node.OutputStreams().SetupNew(node, b.Graph.NewVar())
return node return node
} }


func (t *ChunkedJoinNode) AddInput(str *dag.StreamVar) {
func (t *ChunkedJoinNode) AddInput(str *dag.Var) {
idx := t.InputStreams().EnlargeOne() idx := t.InputStreams().EnlargeOne()
str.Connect(t, idx) str.Connect(t, idx)
} }


func (t *ChunkedJoinNode) Joined() *dag.StreamVar {
func (t *ChunkedJoinNode) Joined() *dag.Var {
return t.OutputStreams().Get(0) return t.OutputStreams().Get(0)
} }


@@ -190,10 +192,10 @@ func (t *ChunkedJoinNode) RemoveAllInputs() {


func (t *ChunkedJoinNode) GenerateOp() (exec.Op, error) { func (t *ChunkedJoinNode) GenerateOp() (exec.Op, error) {
return &ChunkedJoin{ return &ChunkedJoin{
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
Output: t.OutputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).VarID,
ChunkSize: t.ChunkSize, ChunkSize: t.ChunkSize,
}, nil }, nil
} }


+ 28
- 30
common/pkgs/ioswitch2/ops2/clone.go View File

@@ -18,59 +18,57 @@ func init() {
} }


type CloneStream struct { type CloneStream struct {
Raw *exec.StreamVar `json:"raw"`
Cloneds []*exec.StreamVar `json:"cloneds"`
Raw exec.VarID `json:"raw"`
Cloneds []exec.VarID `json:"cloneds"`
} }


func (o *CloneStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *CloneStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Raw)
raw, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Raw)
if err != nil { if err != nil {
return err return err
} }
defer o.Raw.Stream.Close()
defer raw.Stream.Close()


cloned := io2.Clone(o.Raw.Stream, len(o.Cloneds))
cloned := io2.Clone(raw.Stream, len(o.Cloneds))


sem := semaphore.NewWeighted(int64(len(o.Cloneds))) sem := semaphore.NewWeighted(int64(len(o.Cloneds)))
for i, s := range cloned { for i, s := range cloned {
sem.Acquire(ctx.Context, 1) sem.Acquire(ctx.Context, 1)


o.Cloneds[i].Stream = io2.AfterReadClosedOnce(s, func(closer io.ReadCloser) {
sem.Release(1)
e.PutVar(o.Cloneds[i], &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(s, func(closer io.ReadCloser) {
sem.Release(1)
}),
}) })
} }
exec.PutArrayVars(e, o.Cloneds)


return sem.Acquire(ctx.Context, int64(len(o.Cloneds))) return sem.Acquire(ctx.Context, int64(len(o.Cloneds)))
} }


func (o *CloneStream) String() string { func (o *CloneStream) String() string {
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw.ID, utils.FormatVarIDs(o.Cloneds))
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw, utils.FormatVarIDs(o.Cloneds))
} }


type CloneVar struct { type CloneVar struct {
Raw exec.Var `json:"raw"`
Cloneds []exec.Var `json:"cloneds"`
Raw exec.VarID `json:"raw"`
Cloneds []exec.VarID `json:"cloneds"`
} }


func (o *CloneVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *CloneVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Raw)
raw, err := e.BindVar(ctx.Context, o.Raw)
if err != nil { if err != nil {
return err return err
} }


for _, v := range o.Cloneds {
if err := exec.AssignVar(o.Raw, v); err != nil {
return fmt.Errorf("clone var: %w", err)
}
for i := range o.Cloneds {
e.PutVar(o.Cloneds[i], raw.Clone())
} }
e.PutVars(o.Cloneds...)


return nil return nil
} }


func (o *CloneVar) String() string { func (o *CloneVar) String() string {
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw.GetID(), utils.FormatVarIDs(o.Cloneds))
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw, utils.FormatVarIDs(o.Cloneds))
} }


type CloneStreamType struct { type CloneStreamType struct {
@@ -83,22 +81,22 @@ func (b *GraphNodeBuilder) NewCloneStream() *CloneStreamType {
return node return node
} }


func (t *CloneStreamType) SetInput(raw *dag.StreamVar) {
func (t *CloneStreamType) SetInput(raw *dag.Var) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
raw.Connect(t, 0) raw.Connect(t, 0)
} }


func (t *CloneStreamType) NewOutput() *dag.StreamVar {
output := t.Graph().NewStreamVar()
func (t *CloneStreamType) NewOutput() *dag.Var {
output := t.Graph().NewVar()
t.OutputStreams().SetupNew(t, output) t.OutputStreams().SetupNew(t, output)
return output return output
} }


func (t *CloneStreamType) GenerateOp() (exec.Op, error) { func (t *CloneStreamType) GenerateOp() (exec.Op, error) {
return &CloneStream{ return &CloneStream{
Raw: t.InputStreams().Get(0).Var,
Cloneds: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Raw: t.InputStreams().Get(0).VarID,
Cloneds: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
}, nil }, nil
} }
@@ -117,22 +115,22 @@ func (b *GraphNodeBuilder) NewCloneValue() *CloneVarType {
return node return node
} }


func (t *CloneVarType) SetInput(raw *dag.ValueVar) {
func (t *CloneVarType) SetInput(raw *dag.Var) {
t.InputValues().EnsureSize(1) t.InputValues().EnsureSize(1)
raw.Connect(t, 0) raw.Connect(t, 0)
} }


func (t *CloneVarType) NewOutput() *dag.ValueVar {
output := t.Graph().NewValueVar(t.InputValues().Get(0).Type)
func (t *CloneVarType) NewOutput() *dag.Var {
output := t.Graph().NewVar()
t.OutputValues().SetupNew(t, output) t.OutputValues().SetupNew(t, output)
return output return output
} }


func (t *CloneVarType) GenerateOp() (exec.Op, error) { func (t *CloneVarType) GenerateOp() (exec.Op, error) {
return &CloneVar{ return &CloneVar{
Raw: t.InputValues().Get(0).Var,
Cloneds: lo.Map(t.OutputValues().RawArray(), func(v *dag.ValueVar, idx int) exec.Var {
return v.Var
Raw: t.InputValues().Get(0).VarID,
Cloneds: lo.Map(t.OutputValues().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
}, nil }, nil
} }


+ 84
- 83
common/pkgs/ioswitch2/ops2/ec.go View File

@@ -1,7 +1,6 @@
package ops2 package ops2


import ( import (
"context"
"fmt" "fmt"
"io" "io"


@@ -14,7 +13,6 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/sync2" "gitlink.org.cn/cloudream/common/utils/sync2"
"gitlink.org.cn/cloudream/storage/common/pkgs/ec" "gitlink.org.cn/cloudream/storage/common/pkgs/ec"
"golang.org/x/sync/semaphore"
) )


func init() { func init() {
@@ -23,116 +21,118 @@ func init() {
exec.UseOp[*ECMultiply]() exec.UseOp[*ECMultiply]()
} }


type ECReconstructAny struct {
EC cdssdk.ECRedundancy `json:"ec"`
Inputs []*exec.StreamVar `json:"inputs"`
Outputs []*exec.StreamVar `json:"outputs"`
InputBlockIndexes []int `json:"inputBlockIndexes"`
OutputBlockIndexes []int `json:"outputBlockIndexes"`
}

func (o *ECReconstructAny) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
if err != nil {
return fmt.Errorf("new ec: %w", err)
/*
type ECReconstructAny struct {
EC cdssdk.ECRedundancy `json:"ec"`
Inputs []exec.VarID `json:"inputs"`
Outputs []exec.VarID `json:"outputs"`
InputBlockIndexes []int `json:"inputBlockIndexes"`
OutputBlockIndexes []int `json:"outputBlockIndexes"`
} }


err = exec.BindArrayVars(e, ctx.Context, o.Inputs)
if err != nil {
return err
}
defer func() {
func (o *ECReconstructAny) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
if err != nil {
return fmt.Errorf("new ec: %w", err)
}

err = exec.BindArrayVars(e, ctx.Context, inputs)
if err != nil {
return err
}
defer func() {
for _, s := range o.Inputs {
s.Stream.Close()
}
}()

var inputs []io.Reader
for _, s := range o.Inputs { for _, s := range o.Inputs {
s.Stream.Close()
inputs = append(inputs, s.Stream)
} }
}()


var inputs []io.Reader
for _, s := range o.Inputs {
inputs = append(inputs, s.Stream)
}
outputs := rs.ReconstructAny(inputs, o.InputBlockIndexes, o.OutputBlockIndexes)


outputs := rs.ReconstructAny(inputs, o.InputBlockIndexes, o.OutputBlockIndexes)
sem := semaphore.NewWeighted(int64(len(o.Outputs)))
for i := range o.Outputs {
sem.Acquire(ctx.Context, 1)


sem := semaphore.NewWeighted(int64(len(o.Outputs)))
for i := range o.Outputs {
sem.Acquire(ctx.Context, 1)
o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
})
}
e.PutVar(o.Outputs)


o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
})
return sem.Acquire(ctx.Context, int64(len(o.Outputs)))
} }
exec.PutArrayVars(e, o.Outputs)


return sem.Acquire(ctx.Context, int64(len(o.Outputs)))
}
type ECReconstruct struct {
EC cdssdk.ECRedundancy `json:"ec"`
Inputs []exec.VarID `json:"inputs"`
Outputs []exec.VarID `json:"outputs"`
InputBlockIndexes []int `json:"inputBlockIndexes"`
}


type ECReconstruct struct {
EC cdssdk.ECRedundancy `json:"ec"`
Inputs []*exec.StreamVar `json:"inputs"`
Outputs []*exec.StreamVar `json:"outputs"`
InputBlockIndexes []int `json:"inputBlockIndexes"`
}
func (o *ECReconstruct) Execute(ctx context.Context, e *exec.Executor) error {
rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
if err != nil {
return fmt.Errorf("new ec: %w", err)
}


func (o *ECReconstruct) Execute(ctx context.Context, e *exec.Executor) error {
rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
if err != nil {
return fmt.Errorf("new ec: %w", err)
}
err = exec.BindArrayVars(e, ctx, o.Inputs)
if err != nil {
return err
}
defer func() {
for _, s := range o.Inputs {
s.Stream.Close()
}
}()


err = exec.BindArrayVars(e, ctx, o.Inputs)
if err != nil {
return err
}
defer func() {
var inputs []io.Reader
for _, s := range o.Inputs { for _, s := range o.Inputs {
s.Stream.Close()
inputs = append(inputs, s.Stream)
} }
}()


var inputs []io.Reader
for _, s := range o.Inputs {
inputs = append(inputs, s.Stream)
}
outputs := rs.ReconstructData(inputs, o.InputBlockIndexes)


outputs := rs.ReconstructData(inputs, o.InputBlockIndexes)
sem := semaphore.NewWeighted(int64(len(o.Outputs)))
for i := range o.Outputs {
sem.Acquire(ctx, 1)


sem := semaphore.NewWeighted(int64(len(o.Outputs)))
for i := range o.Outputs {
sem.Acquire(ctx, 1)
o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
})
}
e.PutVar(o.Outputs)


o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
})
return sem.Acquire(ctx, int64(len(o.Outputs)))
} }
exec.PutArrayVars(e, o.Outputs)

return sem.Acquire(ctx, int64(len(o.Outputs)))
}

*/
type ECMultiply struct { type ECMultiply struct {
Coef [][]byte `json:"coef"`
Inputs []*exec.StreamVar `json:"inputs"`
Outputs []*exec.StreamVar `json:"outputs"`
ChunkSize int `json:"chunkSize"`
Coef [][]byte `json:"coef"`
Inputs []exec.VarID `json:"inputs"`
Outputs []exec.VarID `json:"outputs"`
ChunkSize int `json:"chunkSize"`
} }


func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := exec.BindArrayVars(e, ctx.Context, o.Inputs)
inputs, err := exec.BindArray[*exec.StreamValue](e, ctx.Context, o.Inputs)
if err != nil { if err != nil {
return err return err
} }
defer func() { defer func() {
for _, s := range o.Inputs {
for _, s := range inputs {
s.Stream.Close() s.Stream.Close()
} }
}() }()


outputWrs := make([]*io.PipeWriter, len(o.Outputs)) outputWrs := make([]*io.PipeWriter, len(o.Outputs))
outputVars := make([]*exec.StreamValue, len(o.Outputs))


for i := range o.Outputs { for i := range o.Outputs {
rd, wr := io.Pipe() rd, wr := io.Pipe()
o.Outputs[i].Stream = rd
outputVars[i].Stream = rd
outputWrs[i] = wr outputWrs[i] = wr
} }


@@ -150,7 +150,7 @@ func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


for { for {
err := sync2.ParallelDo(o.Inputs, func(s *exec.StreamVar, i int) error {
err := sync2.ParallelDo(inputs, func(s *exec.StreamValue, i int) error {
_, err := io.ReadFull(s.Stream, inputChunks[i]) _, err := io.ReadFull(s.Stream, inputChunks[i])
return err return err
}) })
@@ -179,7 +179,8 @@ func (o *ECMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }
}() }()


exec.PutArrayVars(e, o.Outputs)
exec.PutArray(e, o.Outputs, outputVars)

err = fut.Wait(ctx.Context) err = fut.Wait(ctx.Context)
if err != nil { if err != nil {
for _, wr := range outputWrs { for _, wr := range outputWrs {
@@ -218,7 +219,7 @@ func (b *GraphNodeBuilder) NewECMultiply(ec cdssdk.ECRedundancy) *ECMultiplyNode
return node return node
} }


func (t *ECMultiplyNode) AddInput(str *dag.StreamVar, dataIndex int) {
func (t *ECMultiplyNode) AddInput(str *dag.Var, dataIndex int) {
t.InputIndexes = append(t.InputIndexes, dataIndex) t.InputIndexes = append(t.InputIndexes, dataIndex)
idx := t.InputStreams().EnlargeOne() idx := t.InputStreams().EnlargeOne()
str.Connect(t, idx) str.Connect(t, idx)
@@ -232,9 +233,9 @@ func (t *ECMultiplyNode) RemoveAllInputs() {
t.InputIndexes = nil t.InputIndexes = nil
} }


func (t *ECMultiplyNode) NewOutput(dataIndex int) *dag.StreamVar {
func (t *ECMultiplyNode) NewOutput(dataIndex int) *dag.Var {
t.OutputIndexes = append(t.OutputIndexes, dataIndex) t.OutputIndexes = append(t.OutputIndexes, dataIndex)
output := t.Graph().NewStreamVar()
output := t.Graph().NewVar()
t.OutputStreams().SetupNew(t, output) t.OutputStreams().SetupNew(t, output)
return output return output
} }
@@ -251,8 +252,8 @@ func (t *ECMultiplyNode) GenerateOp() (exec.Op, error) {


return &ECMultiply{ return &ECMultiply{
Coef: coef, Coef: coef,
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
ChunkSize: t.EC.ChunkSize, ChunkSize: t.EC.ChunkSize,
}, nil }, nil
} }


+ 21
- 20
common/pkgs/ioswitch2/ops2/file.go View File

@@ -18,16 +18,16 @@ func init() {
} }


type FileWrite struct { type FileWrite struct {
Input *exec.StreamVar `json:"input"`
FilePath string `json:"filePath"`
Input exec.VarID `json:"input"`
FilePath string `json:"filePath"`
} }


func (o *FileWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *FileWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


dir := path.Dir(o.FilePath) dir := path.Dir(o.FilePath)
err = os.MkdirAll(dir, 0777) err = os.MkdirAll(dir, 0777)
@@ -41,7 +41,7 @@ func (o *FileWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }
defer file.Close() defer file.Close()


_, err = io.Copy(file, o.Input.Stream)
_, err = io.Copy(file, input.Stream)
if err != nil { if err != nil {
return fmt.Errorf("copying data to file: %w", err) return fmt.Errorf("copying data to file: %w", err)
} }
@@ -50,12 +50,12 @@ func (o *FileWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


func (o *FileWrite) String() string { func (o *FileWrite) String() string {
return fmt.Sprintf("FileWrite %v -> %s", o.Input.ID, o.FilePath)
return fmt.Sprintf("FileWrite %v -> %s", o.Input, o.FilePath)
} }


type FileRead struct { type FileRead struct {
Output *exec.StreamVar `json:"output"`
FilePath string `json:"filePath"`
Output exec.VarID `json:"output"`
FilePath string `json:"filePath"`
} }


func (o *FileRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *FileRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@@ -65,17 +65,18 @@ func (o *FileRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosed(file, func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosed(file, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)
fut.Wait(ctx.Context) fut.Wait(ctx.Context)


return nil return nil
} }


func (o *FileRead) String() string { func (o *FileRead) String() string {
return fmt.Sprintf("FileRead %s -> %v", o.FilePath, o.Output.ID)
return fmt.Sprintf("FileRead %s -> %v", o.FilePath, o.Output)
} }


type FileReadNode struct { type FileReadNode struct {
@@ -88,12 +89,12 @@ func (b *GraphNodeBuilder) NewFileRead(filePath string) *FileReadNode {
FilePath: filePath, FilePath: filePath,
} }
b.AddNode(node) b.AddNode(node)
node.OutputStreams().SetupNew(node, b.NewStreamVar())
node.OutputStreams().SetupNew(node, b.NewVar())
return node return node
} }


func (t *FileReadNode) Output() dag.StreamSlot {
return dag.StreamSlot{
func (t *FileReadNode) Output() dag.Slot {
return dag.Slot{
Var: t.OutputStreams().Get(0), Var: t.OutputStreams().Get(0),
Index: 0, Index: 0,
} }
@@ -101,7 +102,7 @@ func (t *FileReadNode) Output() dag.StreamSlot {


func (t *FileReadNode) GenerateOp() (exec.Op, error) { func (t *FileReadNode) GenerateOp() (exec.Op, error) {
return &FileRead{ return &FileRead{
Output: t.OutputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).VarID,
FilePath: t.FilePath, FilePath: t.FilePath,
}, nil }, nil
} }
@@ -123,21 +124,21 @@ func (b *GraphNodeBuilder) NewFileWrite(filePath string) *FileWriteNode {
return node return node
} }


func (t *FileWriteNode) Input() dag.StreamSlot {
return dag.StreamSlot{
func (t *FileWriteNode) Input() dag.Slot {
return dag.Slot{
Var: t.InputStreams().Get(0), Var: t.InputStreams().Get(0),
Index: 0, Index: 0,
} }
} }


func (t *FileWriteNode) SetInput(str *dag.StreamVar) {
func (t *FileWriteNode) SetInput(str *dag.Var) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
str.Connect(t, 0) str.Connect(t, 0)
} }


func (t *FileWriteNode) GenerateOp() (exec.Op, error) { func (t *FileWriteNode) GenerateOp() (exec.Op, error) {
return &FileWrite{ return &FileWrite{
Input: t.InputStreams().Get(0).Var,
Input: t.InputStreams().Get(0).VarID,
FilePath: t.FilePath, FilePath: t.FilePath,
}, nil }, nil
} }

+ 19
- 13
common/pkgs/ioswitch2/ops2/join.go View File

@@ -1,45 +1,51 @@
package ops2 package ops2


import ( import (
"context"
"fmt"
"io" "io"


"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
) )


func init() { func init() {
// OpUnion.AddT((*Join)(nil))
exec.UseOp[*Join]()
} }


type Join struct { type Join struct {
Inputs []*exec.StreamVar `json:"inputs"`
Output *exec.StreamVar `json:"output"`
Length int64 `json:"length"`
Inputs []exec.VarID `json:"inputs"`
Output exec.VarID `json:"output"`
Length int64 `json:"length"`
} }


func (o *Join) Execute(ctx context.Context, e *exec.Executor) error {
err := exec.BindArrayVars(e, ctx, o.Inputs)
func (o *Join) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
inputs, err := exec.BindArray[*exec.StreamValue](e, ctx.Context, o.Inputs)
if err != nil { if err != nil {
return err return err
} }


var strReaders []io.Reader var strReaders []io.Reader
for _, s := range o.Inputs {
for _, s := range inputs {
strReaders = append(strReaders, s.Stream) strReaders = append(strReaders, s.Stream)
} }
defer func() { defer func() {
for _, str := range o.Inputs {
for _, str := range inputs {
str.Stream.Close() str.Stream.Close()
} }
}() }()


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(io2.Length(io2.Join(strReaders), o.Length), func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(io2.Length(io2.Join(strReaders), o.Length), func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx)
return fut.Wait(ctx.Context)
}

func (o *Join) String() string {
return fmt.Sprintf("Join %v->%v", utils.FormatVarIDs(o.Inputs), o.Output)
} }

+ 16
- 11
common/pkgs/ioswitch2/ops2/length.go View File

@@ -1,7 +1,7 @@
package ops2 package ops2


import ( import (
"context"
"fmt"
"io" "io"


"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
@@ -14,23 +14,28 @@ func init() {
} }


type Length struct { type Length struct {
Input *exec.StreamVar `json:"input"`
Output *exec.StreamVar `json:"output"`
Length int64 `json:"length"`
Input exec.VarID `json:"input"`
Output exec.VarID `json:"output"`
Length int64 `json:"length"`
} }


func (o *Length) Execute(ctx context.Context, e *exec.Executor) error {
err := e.BindVars(ctx, o.Input)
func (o *Length) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
str, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer str.Stream.Close()


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(io2.Length(o.Input.Stream, o.Length), func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(io2.Length(str.Stream, o.Length), func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx)
return fut.Wait(ctx.Context)
}

func (o *Length) String() string {
return fmt.Sprintf("Length(length=%v) %v->%v", o.Length, o.Input, o.Output)
} }

+ 3
- 3
common/pkgs/ioswitch2/ops2/ops.go View File

@@ -15,13 +15,13 @@ func NewGraphNodeBuilder() *GraphNodeBuilder {


type FromNode interface { type FromNode interface {
dag.Node dag.Node
Output() dag.StreamSlot
Output() dag.Slot
} }


type ToNode interface { type ToNode interface {
dag.Node dag.Node
Input() dag.StreamSlot
SetInput(input *dag.StreamVar)
Input() dag.Slot
SetInput(input *dag.Var)
} }


// func formatStreamIO(node *dag.Node) string { // func formatStreamIO(node *dag.Node) string {


+ 22
- 21
common/pkgs/ioswitch2/ops2/range.go View File

@@ -16,25 +16,25 @@ func init() {
} }


type Range struct { type Range struct {
Input *exec.StreamVar `json:"input"`
Output *exec.StreamVar `json:"output"`
Offset int64 `json:"offset"`
Length *int64 `json:"length"`
Input exec.VarID `json:"input"`
Output exec.VarID `json:"output"`
Offset int64 `json:"offset"`
Length *int64 `json:"length"`
} }


func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


buf := make([]byte, 1024*16) buf := make([]byte, 1024*16)


// 跳过前Offset个字节 // 跳过前Offset个字节
for o.Offset > 0 { for o.Offset > 0 {
rdCnt := math2.Min(o.Offset, int64(len(buf))) rdCnt := math2.Min(o.Offset, int64(len(buf)))
rd, err := o.Input.Stream.Read(buf[:rdCnt])
rd, err := input.Stream.Read(buf[:rdCnt])
if err == io.EOF { if err == io.EOF {
// 输入流不够长度也不报错,只是产生一个空的流 // 输入流不够长度也不报错,只是产生一个空的流
break break
@@ -48,30 +48,31 @@ func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
fut := future.NewSetVoid() fut := future.NewSetVoid()


if o.Length == nil { if o.Length == nil {
o.Output.Stream = io2.AfterEOF(o.Input.Stream, func(closer io.ReadCloser, err error) {
fut.SetVoid()
})


e.PutVars(o.Output)
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterEOF(input.Stream, func(closer io.ReadCloser, err error) {
fut.SetVoid()
}),
})
return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }


o.Output.Stream = io2.AfterEOF(io2.Length(o.Input.Stream, *o.Length), func(closer io.ReadCloser, err error) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterEOF(io2.Length(input.Stream, *o.Length), func(closer io.ReadCloser, err error) {
fut.SetVoid()
}),
}) })

e.PutVars(o.Output)
err = fut.Wait(ctx.Context) err = fut.Wait(ctx.Context)
if err != nil { if err != nil {
return err return err
} }


io2.DropWithBuf(o.Input.Stream, buf)
io2.DropWithBuf(input.Stream, buf)
return nil return nil
} }


func (o *Range) String() string { func (o *Range) String() string {
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input.ID, o.Output.ID)
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input, o.Output)
} }


type RangeNode struct { type RangeNode struct {
@@ -85,19 +86,19 @@ func (b *GraphNodeBuilder) NewRange() *RangeNode {
return node return node
} }


func (t *RangeNode) RangeStream(input *dag.StreamVar, rng exec.Range) *dag.StreamVar {
func (t *RangeNode) RangeStream(input *dag.Var, rng exec.Range) *dag.Var {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.Range = rng t.Range = rng
output := t.Graph().NewStreamVar()
output := t.Graph().NewVar()
t.OutputStreams().Setup(t, output, 0) t.OutputStreams().Setup(t, output, 0)
return output return output
} }


func (t *RangeNode) GenerateOp() (exec.Op, error) { func (t *RangeNode) GenerateOp() (exec.Op, error) {
return &Range{ return &Range{
Input: t.InputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).Var,
Input: t.InputStreams().Get(0).VarID,
Output: t.OutputStreams().Get(0).VarID,
Offset: t.Range.Offset, Offset: t.Range.Offset,
Length: t.Range.Length, Length: t.Range.Length,
}, nil }, nil


+ 37
- 27
common/pkgs/ioswitch2/ops2/shard_store.go View File

@@ -17,10 +17,19 @@ import (
func init() { func init() {
exec.UseOp[*ShardRead]() exec.UseOp[*ShardRead]()
exec.UseOp[*ShardWrite]() exec.UseOp[*ShardWrite]()
exec.UseVarValue[*FileHashValue]()
}

type FileHashValue struct {
Hash types.FileHash `json:"hash"`
}

func (v *FileHashValue) Clone() exec.VarValue {
return &FileHashValue{Hash: v.Hash}
} }


type ShardRead struct { type ShardRead struct {
Output *exec.StreamVar `json:"output"`
Output exec.VarID `json:"output"`
StorageID cdssdk.StorageID `json:"storageID"` StorageID cdssdk.StorageID `json:"storageID"`
Open types.OpenOption `json:"option"` Open types.OpenOption `json:"option"`
} }
@@ -47,28 +56,29 @@ func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }


func (o *ShardRead) String() string { func (o *ShardRead) String() string {
return fmt.Sprintf("ShardRead %v -> %v", o.Open, o.Output.ID)
return fmt.Sprintf("ShardRead %v -> %v", o.Open, o.Output)
} }


type ShardWrite struct { type ShardWrite struct {
Input *exec.StreamVar `json:"input"`
FileHash *exec.StringVar `json:"fileHash"`
Input exec.VarID `json:"input"`
FileHash exec.VarID `json:"fileHash"`
StorageID cdssdk.StorageID `json:"storageID"` StorageID cdssdk.StorageID `json:"storageID"`
} }


func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger. logger.
WithField("Input", o.Input.ID).
WithField("FileHashVar", o.FileHash.ID).
WithField("Input", o.Input).
WithField("FileHash", o.FileHash).
Debugf("writting file to shard store") Debugf("writting file to shard store")
defer logger.Debugf("write to shard store finished") defer logger.Debugf("write to shard store finished")


@@ -82,16 +92,16 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
return fmt.Errorf("getting shard store %v: %w", o.StorageID, err) return fmt.Errorf("getting shard store %v: %w", o.StorageID, err)
} }


err = e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


writer := store.New() writer := store.New()
defer writer.Abort() defer writer.Abort()


_, err = io.Copy(writer, o.Input.Stream)
_, err = io.Copy(writer, input.Stream)
if err != nil { if err != nil {
return fmt.Errorf("writing file to shard store: %w", err) return fmt.Errorf("writing file to shard store: %w", err)
} }
@@ -101,14 +111,14 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
return fmt.Errorf("finishing writing file to shard store: %w", err) return fmt.Errorf("finishing writing file to shard store: %w", err)
} }


o.FileHash.Value = string(fileInfo.Hash)
e.PutVars(o.FileHash)
e.PutVar(o.FileHash, &FileHashValue{
Hash: fileInfo.Hash,
})
return nil return nil
} }


func (o *ShardWrite) String() string { func (o *ShardWrite) String() string {
return fmt.Sprintf("ShardWrite %v -> %v", o.Input.ID, o.FileHash.ID)
return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHash)
} }


type ShardReadNode struct { type ShardReadNode struct {
@@ -123,12 +133,12 @@ func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenO
Open: open, Open: open,
} }
b.AddNode(node) b.AddNode(node)
node.OutputStreams().SetupNew(node, b.NewStreamVar())
node.OutputStreams().SetupNew(node, b.NewVar())
return node return node
} }


func (t *ShardReadNode) Output() dag.StreamSlot {
return dag.StreamSlot{
func (t *ShardReadNode) Output() dag.Slot {
return dag.Slot{
Var: t.OutputStreams().Get(0), Var: t.OutputStreams().Get(0),
Index: 0, Index: 0,
} }
@@ -136,7 +146,7 @@ func (t *ShardReadNode) Output() dag.StreamSlot {


func (t *ShardReadNode) GenerateOp() (exec.Op, error) { func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
return &ShardRead{ return &ShardRead{
Output: t.OutputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).VarID,
StorageID: t.StorageID, StorageID: t.StorageID,
Open: t.Open, Open: t.Open,
}, nil }, nil
@@ -159,27 +169,27 @@ func (b *GraphNodeBuilder) NewShardWrite(fileHashStoreKey string) *ShardWriteNod
return node return node
} }


func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
func (t *ShardWriteNode) SetInput(input *dag.Var) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.OutputValues().SetupNew(t, t.Graph().NewValueVar(dag.StringValueVar))
t.OutputValues().SetupNew(t, t.Graph().NewVar())
} }


func (t *ShardWriteNode) Input() dag.StreamSlot {
return dag.StreamSlot{
func (t *ShardWriteNode) Input() dag.Slot {
return dag.Slot{
Var: t.InputStreams().Get(0), Var: t.InputStreams().Get(0),
Index: 0, Index: 0,
} }
} }


func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
func (t *ShardWriteNode) FileHashVar() *dag.Var {
return t.OutputValues().Get(0) return t.OutputValues().Get(0)
} }


func (t *ShardWriteNode) GenerateOp() (exec.Op, error) { func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
return &ShardWrite{ return &ShardWrite{
Input: t.InputStreams().Get(0).Var,
FileHash: t.OutputValues().Get(0).Var.(*exec.StringVar),
Input: t.InputStreams().Get(0).VarID,
FileHash: t.OutputValues().Get(0).VarID,
}, nil }, nil
} }




+ 4
- 4
common/pkgs/ioswitch2/parser/parser.go View File

@@ -26,7 +26,7 @@ func NewParser(ec cdssdk.ECRedundancy) *DefaultParser {
} }


type IndexedStream struct { type IndexedStream struct {
Stream *dag.StreamVar
Stream *dag.Var
DataIndex int DataIndex int
} }


@@ -94,8 +94,8 @@ func (p *DefaultParser) Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) erro


return plan.Generate(ctx.DAG.Graph, blder) return plan.Generate(ctx.DAG.Graph, blder)
} }
func (p *DefaultParser) findOutputStream(ctx *ParseContext, streamIndex int) *dag.StreamVar {
var ret *dag.StreamVar
func (p *DefaultParser) findOutputStream(ctx *ParseContext, streamIndex int) *dag.Var {
var ret *dag.Var
for _, s := range ctx.IndexedStreams { for _, s := range ctx.IndexedStreams {
if s.DataIndex == streamIndex { if s.DataIndex == streamIndex {
ret = s.Stream ret = s.Stream
@@ -166,7 +166,7 @@ func (p *DefaultParser) extend(ctx *ParseContext) error {
} }


// 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令 // 如果有K个不同的文件块流,则生成Multiply指令,同时针对其生成的流,生成Join指令
ecInputStrs := make(map[int]*dag.StreamVar)
ecInputStrs := make(map[int]*dag.Var)
for _, s := range ctx.IndexedStreams { for _, s := range ctx.IndexedStreams {
if s.DataIndex >= 0 && ecInputStrs[s.DataIndex] == nil { if s.DataIndex >= 0 && ecInputStrs[s.DataIndex] == nil {
ecInputStrs[s.DataIndex] = s.Stream ecInputStrs[s.DataIndex] = s.Stream


+ 8
- 8
common/pkgs/ioswitchlrc/agent_worker.go View File

@@ -47,17 +47,17 @@ type AgentWorkerClient struct {
func (c *AgentWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error { func (c *AgentWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error {
return c.cli.ExecuteIOPlan(ctx, plan) return c.cli.ExecuteIOPlan(ctx, plan)
} }
func (c *AgentWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, str io.ReadCloser) error {
return c.cli.SendStream(ctx, planID, v.ID, str)
func (c *AgentWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, id exec.VarID, stream io.ReadCloser) error {
return c.cli.SendStream(ctx, planID, id, stream)
} }
func (c *AgentWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, v exec.Var) error {
return c.cli.SendVar(ctx, planID, v)
func (c *AgentWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error {
return c.cli.SendVar(ctx, planID, id, value)
} }
func (c *AgentWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, v *exec.StreamVar, signal *exec.SignalVar) (io.ReadCloser, error) {
return c.cli.GetStream(ctx, planID, v.ID, signal)
func (c *AgentWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, streamID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) {
return c.cli.GetStream(ctx, planID, streamID, signalID, signal)
} }
func (c *AgentWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, v exec.Var, signal *exec.SignalVar) error {
return c.cli.GetVar(ctx, planID, v, signal)
func (c *AgentWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) {
return c.cli.GetVar(ctx, planID, varID, signalID, signal)
} }
func (c *AgentWorkerClient) Close() error { func (c *AgentWorkerClient) Close() error {
stgglb.AgentRPCPool.Release(c.cli) stgglb.AgentRPCPool.Release(c.cli)


+ 56
- 33
common/pkgs/ioswitchlrc/ops2/chunked.go View File

@@ -19,20 +19,20 @@ func init() {
} }


type ChunkedSplit struct { type ChunkedSplit struct {
Input *exec.StreamVar `json:"input"`
Outputs []*exec.StreamVar `json:"outputs"`
ChunkSize int `json:"chunkSize"`
PaddingZeros bool `json:"paddingZeros"`
Input exec.VarID `json:"input"`
Outputs []exec.VarID `json:"outputs"`
ChunkSize int `json:"chunkSize"`
PaddingZeros bool `json:"paddingZeros"`
} }


func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


outputs := io2.ChunkedSplit(o.Input.Stream, o.ChunkSize, len(o.Outputs), io2.ChunkedSplitOption{
outputs := io2.ChunkedSplit(input.Stream, o.ChunkSize, len(o.Outputs), io2.ChunkedSplitOption{
PaddingZeros: o.PaddingZeros, PaddingZeros: o.PaddingZeros,
}) })


@@ -40,11 +40,12 @@ func (o *ChunkedSplit) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
for i := range outputs { for i := range outputs {
sem.Acquire(ctx.Context, 1) sem.Acquire(ctx.Context, 1)


o.Outputs[i].Stream = io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
e.PutVar(o.Outputs[i], &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
sem.Release(1)
}),
}) })
} }
exec.PutArrayVars(e, o.Outputs)


return sem.Acquire(ctx.Context, int64(len(outputs))) return sem.Acquire(ctx.Context, int64(len(outputs)))
} }
@@ -54,38 +55,39 @@ func (o *ChunkedSplit) String() string {
"ChunkedSplit(chunkSize=%v, paddingZeros=%v), %v -> (%v)", "ChunkedSplit(chunkSize=%v, paddingZeros=%v), %v -> (%v)",
o.ChunkSize, o.ChunkSize,
o.PaddingZeros, o.PaddingZeros,
o.Input.ID,
o.Input,
utils.FormatVarIDs(o.Outputs), utils.FormatVarIDs(o.Outputs),
) )
} }


type ChunkedJoin struct { type ChunkedJoin struct {
Inputs []*exec.StreamVar `json:"inputs"`
Output *exec.StreamVar `json:"output"`
ChunkSize int `json:"chunkSize"`
Inputs []exec.VarID `json:"inputs"`
Output exec.VarID `json:"output"`
ChunkSize int `json:"chunkSize"`
} }


func (o *ChunkedJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ChunkedJoin) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := exec.BindArrayVars(e, ctx.Context, o.Inputs)
inputs, err := exec.BindArray[*exec.StreamValue](e, ctx.Context, o.Inputs)
if err != nil { if err != nil {
return err return err
} }


var strReaders []io.Reader var strReaders []io.Reader
for _, s := range o.Inputs {
for _, s := range inputs {
strReaders = append(strReaders, s.Stream) strReaders = append(strReaders, s.Stream)
} }
defer func() { defer func() {
for _, str := range o.Inputs {
for _, str := range inputs {
str.Stream.Close() str.Stream.Close()
} }
}() }()


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(io2.BufferedChunkedJoin(strReaders, o.ChunkSize), func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(io2.BufferedChunkedJoin(strReaders, o.ChunkSize), func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }
@@ -95,7 +97,7 @@ func (o *ChunkedJoin) String() string {
"ChunkedJoin(chunkSize=%v), (%v) -> %v", "ChunkedJoin(chunkSize=%v), (%v) -> %v",
o.ChunkSize, o.ChunkSize,
utils.FormatVarIDs(o.Inputs), utils.FormatVarIDs(o.Inputs),
o.Output.ID,
o.Output,
) )
} }


@@ -112,16 +114,16 @@ func (b *GraphNodeBuilder) NewChunkedSplit(chunkSize int) *ChunkedSplitNode {
return node return node
} }


func (t *ChunkedSplitNode) Split(input *dag.StreamVar, cnt int) {
func (t *ChunkedSplitNode) Split(input *dag.Var, cnt int) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.OutputStreams().Resize(cnt) t.OutputStreams().Resize(cnt)
for i := 0; i < cnt; i++ { for i := 0; i < cnt; i++ {
t.OutputStreams().Setup(t, t.Graph().NewStreamVar(), i)
t.OutputStreams().Setup(t, t.Graph().NewVar(), i)
} }
} }


func (t *ChunkedSplitNode) SubStream(idx int) *dag.StreamVar {
func (t *ChunkedSplitNode) SubStream(idx int) *dag.Var {
return t.OutputStreams().Get(idx) return t.OutputStreams().Get(idx)
} }


@@ -129,11 +131,25 @@ func (t *ChunkedSplitNode) SplitCount() int {
return t.OutputStreams().Len() return t.OutputStreams().Len()
} }


func (t *ChunkedSplitNode) Clear() {
if t.InputStreams().Len() == 0 {
return
}

t.InputStreams().Get(0).Disconnect(t, 0)
t.InputStreams().Resize(0)

for _, out := range t.OutputStreams().RawArray() {
out.DisconnectAll()
}
t.OutputStreams().Resize(0)
}

func (t *ChunkedSplitNode) GenerateOp() (exec.Op, error) { func (t *ChunkedSplitNode) GenerateOp() (exec.Op, error) {
return &ChunkedSplit{ return &ChunkedSplit{
Input: t.InputStreams().Get(0).Var,
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Input: t.InputStreams().Get(0).VarID,
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
ChunkSize: t.ChunkSize, ChunkSize: t.ChunkSize,
PaddingZeros: true, PaddingZeros: true,
@@ -154,25 +170,32 @@ func (b *GraphNodeBuilder) NewChunkedJoin(chunkSize int) *ChunkedJoinNode {
ChunkSize: chunkSize, ChunkSize: chunkSize,
} }
b.AddNode(node) b.AddNode(node)
node.OutputStreams().SetupNew(node, b.Graph.NewStreamVar())
node.OutputStreams().SetupNew(node, b.Graph.NewVar())
return node return node
} }


func (t *ChunkedJoinNode) AddInput(str *dag.StreamVar) {
func (t *ChunkedJoinNode) AddInput(str *dag.Var) {
idx := t.InputStreams().EnlargeOne() idx := t.InputStreams().EnlargeOne()
str.Connect(t, idx) str.Connect(t, idx)
} }


func (t *ChunkedJoinNode) Joined() *dag.StreamVar {
func (t *ChunkedJoinNode) Joined() *dag.Var {
return t.OutputStreams().Get(0) return t.OutputStreams().Get(0)
} }


func (t *ChunkedJoinNode) RemoveAllInputs() {
for i, in := range t.InputStreams().RawArray() {
in.Disconnect(t, i)
}
t.InputStreams().Resize(0)
}

func (t *ChunkedJoinNode) GenerateOp() (exec.Op, error) { func (t *ChunkedJoinNode) GenerateOp() (exec.Op, error) {
return &ChunkedJoin{ return &ChunkedJoin{
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
Output: t.OutputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).VarID,
ChunkSize: t.ChunkSize, ChunkSize: t.ChunkSize,
}, nil }, nil
} }


+ 28
- 30
common/pkgs/ioswitchlrc/ops2/clone.go View File

@@ -18,59 +18,57 @@ func init() {
} }


type CloneStream struct { type CloneStream struct {
Raw *exec.StreamVar `json:"raw"`
Cloneds []*exec.StreamVar `json:"cloneds"`
Raw exec.VarID `json:"raw"`
Cloneds []exec.VarID `json:"cloneds"`
} }


func (o *CloneStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *CloneStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Raw)
raw, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Raw)
if err != nil { if err != nil {
return err return err
} }
defer o.Raw.Stream.Close()
defer raw.Stream.Close()


cloned := io2.Clone(o.Raw.Stream, len(o.Cloneds))
cloned := io2.Clone(raw.Stream, len(o.Cloneds))


sem := semaphore.NewWeighted(int64(len(o.Cloneds))) sem := semaphore.NewWeighted(int64(len(o.Cloneds)))
for i, s := range cloned { for i, s := range cloned {
sem.Acquire(ctx.Context, 1) sem.Acquire(ctx.Context, 1)


o.Cloneds[i].Stream = io2.AfterReadClosedOnce(s, func(closer io.ReadCloser) {
sem.Release(1)
e.PutVar(o.Cloneds[i], &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(s, func(closer io.ReadCloser) {
sem.Release(1)
}),
}) })
} }
exec.PutArrayVars(e, o.Cloneds)


return sem.Acquire(ctx.Context, int64(len(o.Cloneds))) return sem.Acquire(ctx.Context, int64(len(o.Cloneds)))
} }


func (o *CloneStream) String() string { func (o *CloneStream) String() string {
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw.ID, utils.FormatVarIDs(o.Cloneds))
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw, utils.FormatVarIDs(o.Cloneds))
} }


type CloneVar struct { type CloneVar struct {
Raw exec.Var `json:"raw"`
Cloneds []exec.Var `json:"cloneds"`
Raw exec.VarID `json:"raw"`
Cloneds []exec.VarID `json:"cloneds"`
} }


func (o *CloneVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *CloneVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Raw)
raw, err := e.BindVar(ctx.Context, o.Raw)
if err != nil { if err != nil {
return err return err
} }


for _, v := range o.Cloneds {
if err := exec.AssignVar(o.Raw, v); err != nil {
return fmt.Errorf("clone var: %w", err)
}
for i := range o.Cloneds {
e.PutVar(o.Cloneds[i], raw.Clone())
} }
e.PutVars(o.Cloneds...)


return nil return nil
} }


func (o *CloneVar) String() string { func (o *CloneVar) String() string {
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw.GetID(), utils.FormatVarIDs(o.Cloneds))
return fmt.Sprintf("CloneStream %v -> (%v)", o.Raw, utils.FormatVarIDs(o.Cloneds))
} }


type CloneStreamType struct { type CloneStreamType struct {
@@ -83,22 +81,22 @@ func (b *GraphNodeBuilder) NewCloneStream() *CloneStreamType {
return node return node
} }


func (t *CloneStreamType) SetInput(raw *dag.StreamVar) {
func (t *CloneStreamType) SetInput(raw *dag.Var) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
raw.Connect(t, 0) raw.Connect(t, 0)
} }


func (t *CloneStreamType) NewOutput() *dag.StreamVar {
output := t.Graph().NewStreamVar()
func (t *CloneStreamType) NewOutput() *dag.Var {
output := t.Graph().NewVar()
t.OutputStreams().SetupNew(t, output) t.OutputStreams().SetupNew(t, output)
return output return output
} }


func (t *CloneStreamType) GenerateOp() (exec.Op, error) { func (t *CloneStreamType) GenerateOp() (exec.Op, error) {
return &CloneStream{ return &CloneStream{
Raw: t.InputStreams().Get(0).Var,
Cloneds: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar {
return v.Var
Raw: t.InputStreams().Get(0).VarID,
Cloneds: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
}, nil }, nil
} }
@@ -117,22 +115,22 @@ func (b *GraphNodeBuilder) NewCloneValue() *CloneVarType {
return node return node
} }


func (t *CloneVarType) SetInput(raw *dag.ValueVar) {
func (t *CloneVarType) SetInput(raw *dag.Var) {
t.InputValues().EnsureSize(1) t.InputValues().EnsureSize(1)
raw.Connect(t, 0) raw.Connect(t, 0)
} }


func (t *CloneVarType) NewOutput() *dag.ValueVar {
output := t.Graph().NewValueVar(t.InputValues().Get(0).Type)
func (t *CloneVarType) NewOutput() *dag.Var {
output := t.Graph().NewVar()
t.OutputValues().SetupNew(t, output) t.OutputValues().SetupNew(t, output)
return output return output
} }


func (t *CloneVarType) GenerateOp() (exec.Op, error) { func (t *CloneVarType) GenerateOp() (exec.Op, error) {
return &CloneVar{ return &CloneVar{
Raw: t.InputValues().Get(0).Var,
Cloneds: lo.Map(t.OutputValues().RawArray(), func(v *dag.ValueVar, idx int) exec.Var {
return v.Var
Raw: t.InputValues().Get(0).VarID,
Cloneds: lo.Map(t.OutputValues().RawArray(), func(v *dag.Var, idx int) exec.VarID {
return v.VarID
}), }),
}, nil }, nil
} }


+ 19
- 19
common/pkgs/ioswitchlrc/ops2/ec.go View File

@@ -21,28 +21,28 @@ func init() {
} }


type GalMultiply struct { type GalMultiply struct {
Coef [][]byte `json:"coef"`
Inputs []*exec.StreamVar `json:"inputs"`
Outputs []*exec.StreamVar `json:"outputs"`
ChunkSize int `json:"chunkSize"`
Coef [][]byte `json:"coef"`
Inputs []exec.VarID `json:"inputs"`
Outputs []exec.VarID `json:"outputs"`
ChunkSize int `json:"chunkSize"`
} }


func (o *GalMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *GalMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := exec.BindArrayVars(e, ctx.Context, o.Inputs)
inputs, err := exec.BindArray[*exec.StreamValue](e, ctx.Context, o.Inputs)
if err != nil { if err != nil {
return err return err
} }
defer func() { defer func() {
for _, s := range o.Inputs {
for _, s := range inputs {
s.Stream.Close() s.Stream.Close()
} }
}() }()


outputWrs := make([]*io.PipeWriter, len(o.Outputs)) outputWrs := make([]*io.PipeWriter, len(o.Outputs))
outputVars := make([]*exec.StreamValue, len(o.Outputs))
for i := range o.Outputs { for i := range o.Outputs {
rd, wr := io.Pipe() rd, wr := io.Pipe()
o.Outputs[i].Stream = rd
outputVars[i].Stream = rd
outputWrs[i] = wr outputWrs[i] = wr
} }


@@ -60,7 +60,7 @@ func (o *GalMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


for { for {
err := sync2.ParallelDo(o.Inputs, func(s *exec.StreamVar, i int) error {
err := sync2.ParallelDo(inputs, func(s *exec.StreamValue, i int) error {
_, err := io.ReadFull(s.Stream, inputChunks[i]) _, err := io.ReadFull(s.Stream, inputChunks[i])
return err return err
}) })
@@ -89,7 +89,7 @@ func (o *GalMultiply) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }
}() }()


exec.PutArrayVars(e, o.Outputs)
exec.PutArray(e, o.Outputs, outputVars)
err = fut.Wait(ctx.Context) err = fut.Wait(ctx.Context)
if err != nil { if err != nil {
for _, wr := range outputWrs { for _, wr := range outputWrs {
@@ -128,7 +128,7 @@ func (b *GraphNodeBuilder) NewLRCConstructAny(lrc cdssdk.LRCRedundancy) *LRCCons
return node return node
} }


func (t *LRCConstructAnyNode) AddInput(str *dag.StreamVar, dataIndex int) {
func (t *LRCConstructAnyNode) AddInput(str *dag.Var, dataIndex int) {
t.InputIndexes = append(t.InputIndexes, dataIndex) t.InputIndexes = append(t.InputIndexes, dataIndex)
idx := t.InputStreams().EnlargeOne() idx := t.InputStreams().EnlargeOne()
str.Connect(t, idx) str.Connect(t, idx)
@@ -142,9 +142,9 @@ func (t *LRCConstructAnyNode) RemoveAllInputs() {
t.InputIndexes = nil t.InputIndexes = nil
} }


func (t *LRCConstructAnyNode) NewOutput(dataIndex int) *dag.StreamVar {
func (t *LRCConstructAnyNode) NewOutput(dataIndex int) *dag.Var {
t.OutputIndexes = append(t.OutputIndexes, dataIndex) t.OutputIndexes = append(t.OutputIndexes, dataIndex)
output := t.Graph().NewStreamVar()
output := t.Graph().NewVar()
t.OutputStreams().SetupNew(t, output) t.OutputStreams().SetupNew(t, output)
return output return output
} }
@@ -161,8 +161,8 @@ func (t *LRCConstructAnyNode) GenerateOp() (exec.Op, error) {


return &GalMultiply{ return &GalMultiply{
Coef: coef, Coef: coef,
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
ChunkSize: t.LRC.ChunkSize, ChunkSize: t.LRC.ChunkSize,
}, nil }, nil
} }
@@ -185,7 +185,7 @@ func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc cdssdk.LRCRedundancy) *LRCCo
return node return node
} }


func (t *LRCConstructGroupNode) SetupForTarget(blockIdx int, inputs []*dag.StreamVar) *dag.StreamVar {
func (t *LRCConstructGroupNode) SetupForTarget(blockIdx int, inputs []*dag.Var) *dag.Var {
t.TargetBlockIndex = blockIdx t.TargetBlockIndex = blockIdx


t.InputStreams().Resize(0) t.InputStreams().Resize(0)
@@ -194,7 +194,7 @@ func (t *LRCConstructGroupNode) SetupForTarget(blockIdx int, inputs []*dag.Strea
in.Connect(t, idx) in.Connect(t, idx)
} }


output := t.Graph().NewStreamVar()
output := t.Graph().NewVar()
t.OutputStreams().Setup(t, output, 0) t.OutputStreams().Setup(t, output, 0)
return output return output
} }
@@ -211,8 +211,8 @@ func (t *LRCConstructGroupNode) GenerateOp() (exec.Op, error) {


return &GalMultiply{ return &GalMultiply{
Coef: coef, Coef: coef,
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.StreamVar, idx int) *exec.StreamVar { return v.Var }),
Inputs: lo.Map(t.InputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
Outputs: lo.Map(t.OutputStreams().RawArray(), func(v *dag.Var, idx int) exec.VarID { return v.VarID }),
ChunkSize: t.LRC.ChunkSize, ChunkSize: t.LRC.ChunkSize,
}, nil }, nil
} }


+ 3
- 3
common/pkgs/ioswitchlrc/ops2/ops.go View File

@@ -15,13 +15,13 @@ func NewGraphNodeBuilder() *GraphNodeBuilder {


type FromNode interface { type FromNode interface {
dag.Node dag.Node
Output() dag.StreamSlot
Output() dag.Slot
} }


type ToNode interface { type ToNode interface {
dag.Node dag.Node
Input() dag.StreamSlot
SetInput(input *dag.StreamVar)
Input() dag.Slot
SetInput(input *dag.Var)
} }


// func formatStreamIO(node *dag.Node) string { // func formatStreamIO(node *dag.Node) string {


+ 22
- 21
common/pkgs/ioswitchlrc/ops2/range.go View File

@@ -16,25 +16,25 @@ func init() {
} }


type Range struct { type Range struct {
Input *exec.StreamVar `json:"input"`
Output *exec.StreamVar `json:"output"`
Offset int64 `json:"offset"`
Length *int64 `json:"length"`
Input exec.VarID `json:"input"`
Output exec.VarID `json:"output"`
Offset int64 `json:"offset"`
Length *int64 `json:"length"`
} }


func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
err := e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


buf := make([]byte, 1024*16) buf := make([]byte, 1024*16)


// 跳过前Offset个字节 // 跳过前Offset个字节
for o.Offset > 0 { for o.Offset > 0 {
rdCnt := math2.Min(o.Offset, int64(len(buf))) rdCnt := math2.Min(o.Offset, int64(len(buf)))
rd, err := o.Input.Stream.Read(buf[:rdCnt])
rd, err := input.Stream.Read(buf[:rdCnt])
if err == io.EOF { if err == io.EOF {
// 输入流不够长度也不报错,只是产生一个空的流 // 输入流不够长度也不报错,只是产生一个空的流
break break
@@ -48,30 +48,31 @@ func (o *Range) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
fut := future.NewSetVoid() fut := future.NewSetVoid()


if o.Length == nil { if o.Length == nil {
o.Output.Stream = io2.AfterEOF(o.Input.Stream, func(closer io.ReadCloser, err error) {
fut.SetVoid()
})


e.PutVars(o.Output)
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterEOF(input.Stream, func(closer io.ReadCloser, err error) {
fut.SetVoid()
}),
})
return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }


o.Output.Stream = io2.AfterEOF(io2.Length(o.Input.Stream, *o.Length), func(closer io.ReadCloser, err error) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterEOF(io2.Length(input.Stream, *o.Length), func(closer io.ReadCloser, err error) {
fut.SetVoid()
}),
}) })

e.PutVars(o.Output)
err = fut.Wait(ctx.Context) err = fut.Wait(ctx.Context)
if err != nil { if err != nil {
return err return err
} }


io2.DropWithBuf(o.Input.Stream, buf)
io2.DropWithBuf(input.Stream, buf)
return nil return nil
} }


func (o *Range) String() string { func (o *Range) String() string {
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input.ID, o.Output.ID)
return fmt.Sprintf("Range(%v+%v) %v -> %v", o.Offset, o.Length, o.Input, o.Output)
} }


type RangeNode struct { type RangeNode struct {
@@ -85,19 +86,19 @@ func (b *GraphNodeBuilder) NewRange() *RangeNode {
return node return node
} }


func (t *RangeNode) RangeStream(input *dag.StreamVar, rng exec.Range) *dag.StreamVar {
func (t *RangeNode) RangeStream(input *dag.Var, rng exec.Range) *dag.Var {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.Range = rng t.Range = rng
output := t.Graph().NewStreamVar()
output := t.Graph().NewVar()
t.OutputStreams().Setup(t, output, 0) t.OutputStreams().Setup(t, output, 0)
return output return output
} }


func (t *RangeNode) GenerateOp() (exec.Op, error) { func (t *RangeNode) GenerateOp() (exec.Op, error) {
return &Range{ return &Range{
Input: t.InputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).Var,
Input: t.InputStreams().Get(0).VarID,
Output: t.OutputStreams().Get(0).VarID,
Offset: t.Range.Offset, Offset: t.Range.Offset,
Length: t.Range.Length, Length: t.Range.Length,
}, nil }, nil


+ 37
- 27
common/pkgs/ioswitchlrc/ops2/shard_store.go View File

@@ -17,10 +17,19 @@ import (
func init() { func init() {
exec.UseOp[*ShardRead]() exec.UseOp[*ShardRead]()
exec.UseOp[*ShardWrite]() exec.UseOp[*ShardWrite]()
exec.UseVarValue[*FileHashValue]()
}

type FileHashValue struct {
Hash types.FileHash `json:"hash"`
}

func (v *FileHashValue) Clone() exec.VarValue {
return &FileHashValue{Hash: v.Hash}
} }


type ShardRead struct { type ShardRead struct {
Output *exec.StreamVar `json:"output"`
Output exec.VarID `json:"output"`
StorageID cdssdk.StorageID `json:"storageID"` StorageID cdssdk.StorageID `json:"storageID"`
Open types.OpenOption `json:"option"` Open types.OpenOption `json:"option"`
} }
@@ -47,28 +56,29 @@ func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
} }


fut := future.NewSetVoid() fut := future.NewSetVoid()
o.Output.Stream = io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
e.PutVar(o.Output, &exec.StreamValue{
Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
fut.SetVoid()
}),
}) })
e.PutVars(o.Output)


return fut.Wait(ctx.Context) return fut.Wait(ctx.Context)
} }


func (o *ShardRead) String() string { func (o *ShardRead) String() string {
return fmt.Sprintf("ShardRead %v -> %v", o.Open, o.Output.ID)
return fmt.Sprintf("ShardRead %v -> %v", o.Open, o.Output)
} }


type ShardWrite struct { type ShardWrite struct {
Input *exec.StreamVar `json:"input"`
FileHash *exec.StringVar `json:"fileHash"`
Input exec.VarID `json:"input"`
FileHash exec.VarID `json:"fileHash"`
StorageID cdssdk.StorageID `json:"storageID"` StorageID cdssdk.StorageID `json:"storageID"`
} }


func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error { func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
logger. logger.
WithField("Input", o.Input.ID).
WithField("FileHashVar", o.FileHash.ID).
WithField("Input", o.Input).
WithField("FileHash", o.FileHash).
Debugf("writting file to shard store") Debugf("writting file to shard store")
defer logger.Debugf("write to shard store finished") defer logger.Debugf("write to shard store finished")


@@ -82,16 +92,16 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
return fmt.Errorf("getting shard store %v: %w", o.StorageID, err) return fmt.Errorf("getting shard store %v: %w", o.StorageID, err)
} }


err = e.BindVars(ctx.Context, o.Input)
input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
if err != nil { if err != nil {
return err return err
} }
defer o.Input.Stream.Close()
defer input.Stream.Close()


writer := store.New() writer := store.New()
defer writer.Abort() defer writer.Abort()


_, err = io.Copy(writer, o.Input.Stream)
_, err = io.Copy(writer, input.Stream)
if err != nil { if err != nil {
return fmt.Errorf("writing file to shard store: %w", err) return fmt.Errorf("writing file to shard store: %w", err)
} }
@@ -101,14 +111,14 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
return fmt.Errorf("finishing writing file to shard store: %w", err) return fmt.Errorf("finishing writing file to shard store: %w", err)
} }


o.FileHash.Value = string(fileInfo.Hash)
e.PutVars(o.FileHash)
e.PutVar(o.FileHash, &FileHashValue{
Hash: fileInfo.Hash,
})
return nil return nil
} }


func (o *ShardWrite) String() string { func (o *ShardWrite) String() string {
return fmt.Sprintf("ShardWrite %v -> %v", o.Input.ID, o.FileHash.ID)
return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHash)
} }


type ShardReadNode struct { type ShardReadNode struct {
@@ -123,12 +133,12 @@ func (b *GraphNodeBuilder) NewShardRead(stgID cdssdk.StorageID, open types.OpenO
Open: open, Open: open,
} }
b.AddNode(node) b.AddNode(node)
node.OutputStreams().SetupNew(node, b.NewStreamVar())
node.OutputStreams().SetupNew(node, b.NewVar())
return node return node
} }


func (t *ShardReadNode) Output() dag.StreamSlot {
return dag.StreamSlot{
func (t *ShardReadNode) Output() dag.Slot {
return dag.Slot{
Var: t.OutputStreams().Get(0), Var: t.OutputStreams().Get(0),
Index: 0, Index: 0,
} }
@@ -136,7 +146,7 @@ func (t *ShardReadNode) Output() dag.StreamSlot {


func (t *ShardReadNode) GenerateOp() (exec.Op, error) { func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
return &ShardRead{ return &ShardRead{
Output: t.OutputStreams().Get(0).Var,
Output: t.OutputStreams().Get(0).VarID,
StorageID: t.StorageID, StorageID: t.StorageID,
Open: t.Open, Open: t.Open,
}, nil }, nil
@@ -159,27 +169,27 @@ func (b *GraphNodeBuilder) NewShardWrite(fileHashStoreKey string) *ShardWriteNod
return node return node
} }


func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
func (t *ShardWriteNode) SetInput(input *dag.Var) {
t.InputStreams().EnsureSize(1) t.InputStreams().EnsureSize(1)
input.Connect(t, 0) input.Connect(t, 0)
t.OutputValues().SetupNew(t, t.Graph().NewValueVar(dag.StringValueVar))
t.OutputValues().SetupNew(t, t.Graph().NewVar())
} }


func (t *ShardWriteNode) Input() dag.StreamSlot {
return dag.StreamSlot{
func (t *ShardWriteNode) Input() dag.Slot {
return dag.Slot{
Var: t.InputStreams().Get(0), Var: t.InputStreams().Get(0),
Index: 0, Index: 0,
} }
} }


func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
func (t *ShardWriteNode) FileHashVar() *dag.Var {
return t.OutputValues().Get(0) return t.OutputValues().Get(0)
} }


func (t *ShardWriteNode) GenerateOp() (exec.Op, error) { func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
return &ShardWrite{ return &ShardWrite{
Input: t.InputStreams().Get(0).Var,
FileHash: t.OutputValues().Get(0).Var.(*exec.StringVar),
Input: t.InputStreams().Get(0).VarID,
FileHash: t.OutputValues().Get(0).VarID,
}, nil }, nil
} }




+ 1
- 1
common/pkgs/ioswitchlrc/parser/generator.go View File

@@ -271,7 +271,7 @@ func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec


func buildDAGReconstructGroup(ctx *GenerateContext, frs []ioswitchlrc.From, toes []ioswitchlrc.To) error { func buildDAGReconstructGroup(ctx *GenerateContext, frs []ioswitchlrc.From, toes []ioswitchlrc.To) error {


var inputs []*dag.StreamVar
var inputs []*dag.Var
for _, fr := range frs { for _, fr := range frs {
frNode, err := buildFromNode(ctx, fr) frNode, err := buildFromNode(ctx, fr)
if err != nil { if err != nil {


Loading…
Cancel
Save